From 16ae3deb1c491dc91dd8cb25f6555fb343692516 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:25:35 +0100 Subject: [PATCH 1/3] Check mirrored files against their own SHA-256 Mirroring a PyPI version passed each file's sha256 as the integrity GetOrFetchArtifactFromURLWithIntegrity records with the version. Wheels of one version share the name and version they are cached under, so the last file's checksum replaced the others', and serving the other wheels from the cache failed the integrity check. The mirror now passes the sha256 as a digest, which is checked against that file's content hash alone. --- internal/mirror/mirror.go | 15 ++++++++++++++- internal/server/mirror_pypi_test.go | 21 +++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ddf97364..24d98ff4 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "sync/atomic" "time" @@ -284,7 +285,7 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr if name == "" { name, version = pv.Name, pv.Version } - result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity) + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, fileDigest(file.Integrity)) if err != nil { fail(fmt.Errorf("%s: %w", file.Filename, err)) return @@ -315,6 +316,18 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr } } +// fileDigest returns the SHA-256 of one file as a digest, which is checked +// against that file alone. The integrity GetOrFetchArtifactFromURLWithIntegrity +// takes is kept with the version instead, and the files of a version that +// share a name and version would overwrite each other's. Other checksums are +// not checked, as the download handlers don't check them either. +func fileDigest(integrity string) string { + if sum, ok := strings.CutPrefix(integrity, "sha256-"); ok { + return "sha256:" + sum + } + return "" +} + // fileLister lists files from where Versions lists versions, so both come // from the configured upstream. func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do diff --git a/internal/server/mirror_pypi_test.go b/internal/server/mirror_pypi_test.go index 103e0f56..a5919579 100644 --- a/internal/server/mirror_pypi_test.go +++ b/internal/server/mirror_pypi_test.go @@ -16,13 +16,14 @@ import ( ) const ( - mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" - mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" + mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" + mirrorLinuxWheelPath = "/packages/23/45/example_pkg-1.0-cp312-cp312-manylinux_2_17_x86_64.whl" + mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" ) -// pypiUpstream serves the JSON API for example-pkg 1.0 with a wheel and an -// sdist, whose filenames spell the name differently, and counts file -// downloads by path. +// pypiUpstream serves the JSON API for example-pkg 1.0 with two wheels, which +// are cached under the same name and version, and an sdist, whose filename +// spells the name differently. It counts file downloads by path. type pypiUpstream struct { *httptest.Server mu sync.Mutex @@ -33,13 +34,13 @@ type pypiUpstream struct { func newPyPIUpstream(t *testing.T, digests map[string]string) *pypiUpstream { t.Helper() u := &pypiUpstream{ - files: map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"}, + files: map[string]string{mirrorWheelPath: "wheel", mirrorLinuxWheelPath: "linux wheel", mirrorSdistPath: "sdist"}, hits: map[string]int{}, } u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/pypi/example-pkg/json" { var files []map[string]any - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for _, path := range []string{mirrorWheelPath, mirrorLinuxWheelPath, mirrorSdistPath} { sum := sha256.Sum256([]byte(u.files[path])) digest := hex.EncodeToString(sum[:]) if d, ok := digests[path]; ok { @@ -93,16 +94,16 @@ func TestMirrorPyPIFetchesEveryFile(t *testing.T) { if progress.Completed != 1 || progress.Failed != 0 { t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) } - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for path := range upstream.files { if got := upstream.hitsFor(path); got != 1 { t.Errorf("%s fetched %d times, want 1", path, got) } } - // pip gets both files from the cache once the upstream is down. + // pip gets every file from the cache once the upstream is down. upstream.Close() pypi := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.PyPI, cfg.Upstream.PyPIDownload) - for path, want := range map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"} { + for path, want := range upstream.files { rec := httptest.NewRecorder() pypi.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) if rec.Code != http.StatusOK || rec.Body.String() != want { From 749934eb6c161f1f43905b1b00c14edfb7d5d118 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:25:40 +0100 Subject: [PATCH 2/3] Mirror conda packages A conda version is mirrored with every build anaconda.org lists for it, each cached under the channel, name, version and subdir/filename the conda download handler reads from its path. The channel comes from the namespace or the channel qualifier; a package without one is refused, as retention keys without one are. The subdir, build and type qualifiers select builds. Which files a version has, which qualifiers select them and where the handler caches them is now described per ecosystem, so PyPI and conda share mirrorFiles. Part of #414. --- docs/configuration.md | 2 + internal/handler/conda.go | 8 +- internal/mirror/files.go | 132 ++++++++++++++++++++++ internal/mirror/files_test.go | 38 +++++++ internal/mirror/mirror.go | 37 +++--- internal/mirror/source.go | 21 ++-- internal/server/mirror_conda_test.go | 161 +++++++++++++++++++++++++++ internal/server/mirror_fetch_test.go | 12 +- internal/server/mirror_versions.go | 8 +- 9 files changed, 391 insertions(+), 28 deletions(-) create mode 100644 internal/mirror/files.go create mode 100644 internal/mirror/files_test.go create mode 100644 internal/server/mirror_conda_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e38..49994ad0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1510,6 +1510,8 @@ Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`. +A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1142ccf2..b372468c 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -142,9 +142,15 @@ func (h *CondaHandler) isPackageFile(filename string) bool { } // parseFilename extracts name and version from a Conda package filename. +func (h *CondaHandler) parseFilename(filename string) (name, version string) { + return CondaFilenameIdentity(filename) +} + +// CondaFilenameIdentity returns the name and version a download of filename +// is cached under, or empty strings when it is not a package file name. // Conda filenames are: {name}-{version}-{build}.{ext} // e.g., "numpy-1.24.0-py311h64a7726_0.conda" -func (h *CondaHandler) parseFilename(filename string) (name, version string) { +func CondaFilenameIdentity(filename string) (name, version string) { // Remove extension base := filename for _, ext := range []string{".conda", ".tar.bz2"} { diff --git a/internal/mirror/files.go b/internal/mirror/files.go new file mode 100644 index 00000000..8cf61fe7 --- /dev/null +++ b/internal/mirror/files.go @@ -0,0 +1,132 @@ +package mirror + +import ( + "fmt" + "slices" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// fileEcosystem describes an ecosystem whose versions publish several files, +// each mirrored on its own. +type fileEcosystem struct { + // qualifiers are the PURL qualifiers that select files. A file is + // mirrored when each one given matches the value attrs reports for it. + qualifiers []string + attrs func(file registries.Artifact) map[string]string + // cacheKey returns the name, version and filename the download handler + // caches file under, or false when the handler would not cache it. + cacheKey func(pv PackageVersion, file registries.Artifact) (name, version, filename string, ok bool) + // check refuses a package the ecosystem can't mirror, before its files + // are listed. + check func(pv PackageVersion) error +} + +var fileEcosystems = map[string]fileEcosystem{ + "pypi": { + qualifiers: []string{"file_name"}, + attrs: func(file registries.Artifact) map[string]string { + return map[string]string{"file_name": file.Filename} + }, + cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { + name, version := handler.PyPIFilenameIdentity(file.Filename) + if name == "" { + name, version = pv.Name, pv.Version + } + return name, version, file.Filename, true + }, + }, + "conda": { + qualifiers: []string{"subdir", "build", "type"}, + attrs: func(file registries.Artifact) map[string]string { + subdir, _ := file.Metadata["subdir"].(string) + build, _ := file.Metadata["build"].(string) + return map[string]string{"subdir": subdir, "build": build, "type": condaFileType(file.Filename)} + }, + cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { + channel, _, _ := strings.Cut(pv.Name, "/") + subdir, _ := file.Metadata["subdir"].(string) + name, version := handler.CondaFilenameIdentity(file.Filename) + if name == "" || subdir == "" || condaFileType(file.Filename) == "" { + return "", "", "", false + } + return channel + "/" + name, version, subdir + "/" + file.Filename, true + }, + check: func(pv PackageVersion) error { + if !strings.Contains(pv.Name, "/") { + return fmt.Errorf("conda packages need a channel, such as pkg:conda/conda-forge/%s", pv.Name) + } + return nil + }, + }, +} + +// condaFileType returns the PURL type qualifier of a conda package file. +func condaFileType(filename string) string { + switch { + case strings.HasSuffix(filename, ".conda"): + return "conda" + case strings.HasSuffix(filename, ".tar.bz2"): + return "tar.bz2" + default: + return "" + } +} + +// selection returns the qualifiers of pv that select files, as +// "key=value" pairs, for error messages. +func (e fileEcosystem) selection(pv PackageVersion) string { + var pairs []string + for _, key := range e.qualifiers { + if value := pv.Qualifiers[key]; value != "" { + pairs = append(pairs, key+"="+value) + } + } + return strings.Join(pairs, " ") +} + +// selects reports whether file matches every selecting qualifier of pv. +func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { + attrs := e.attrs(file) + for _, key := range e.qualifiers { + if want := pv.Qualifiers[key]; want != "" && attrs[key] != want { + return false + } + } + return true +} + +// fileQualifiers keeps the qualifiers of a PURL that select files of its +// ecosystem, or returns nil. +func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string { + e, ok := fileEcosystems[ecosystem] + if !ok { + return nil + } + var kept map[string]string + for key, value := range qualifiers { + if value != "" && slices.Contains(e.qualifiers, key) { + if kept == nil { + kept = map[string]string{} + } + kept[key] = value + } + } + return kept +} + +// condaName returns a conda package name with its channel, taken from the +// namespace or the channel qualifier. Without either it stays bare, and +// mirroring refuses it. +func condaName(namespace, name, channel string) string { + switch { + case namespace != "": + return namespace + "/" + name + case channel != "": + return channel + "/" + name + default: + return name + } +} diff --git a/internal/mirror/files_test.go b/internal/mirror/files_test.go new file mode 100644 index 00000000..d84af723 --- /dev/null +++ b/internal/mirror/files_test.go @@ -0,0 +1,38 @@ +package mirror + +import ( + "reflect" + "testing" +) + +func TestPURLSourceCondaChannelAndQualifiers(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0", "1.1"}} + source := &PURLSource{ + PURLs: []string{ + "pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&build=py312h_0&type=conda", + "pkg:conda/numpy@2.1.0?channel=conda-forge&repository_url=https://example.com", + "pkg:conda/numpy@2.1.0", + "pkg:conda/numpy", + "pkg:conda/numpy?channel=bioconda&subdir=noarch", + }, + Versions: versions, + } + + got := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0", + Qualifiers: map[string]string{"subdir": "linux-64", "build": "py312h_0", "type": "conda"}}, + {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0"}, + {Ecosystem: "conda", Name: "numpy", Version: "2.1.0"}, + {Ecosystem: "conda", Name: "numpy"}, + {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.0", Qualifiers: map[string]string{"subdir": "noarch"}}, + {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.1", Qualifiers: map[string]string{"subdir": "noarch"}}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("items = %+v\nwant %+v", got, want) + } + if !reflect.DeepEqual(versions.asked, []string{"conda/bioconda/numpy"}) { + t.Errorf("listed versions of %v, want only the package with a channel", versions.asked) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 24d98ff4..27691615 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -228,8 +228,8 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } - if pv.Ecosystem == "pypi" { - m.mirrorFiles(ctx, pv, tracker) + if eco, ok := fileEcosystems[pv.Ecosystem]; ok { + m.mirrorFiles(ctx, pv, eco, tracker) return } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") @@ -258,34 +258,39 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog } } -// mirrorFiles mirrors every file of a PyPI version, or only the one named by -// the file_name qualifier. Each file is cached under the name and version -// the download handler reads from its filename, so clients are served it. -func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *progressTracker) { +// mirrorFiles mirrors every file of a version, or those its qualifiers +// select. Each file is cached under the name and version the download +// handler reads from its filename, so clients are served it. +func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) { fail := func(err error) { tracker.failed.Add(1) tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) m.logger.Warn("mirror failed", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) } + if eco.check != nil { + if err := eco.check(pv); err != nil { + fail(err) + return + } + } files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) if err != nil { fail(fmt.Errorf("listing files: %w", err)) return } - wantFile := pv.Qualifiers["file_name"] var matched, fetched int var size int64 for _, file := range files { - if wantFile != "" && file.Filename != wantFile { + if !eco.selects(pv, file) { continue } - matched++ - name, version := handler.PyPIFilenameIdentity(file.Filename) - if name == "" { - name, version = pv.Name, pv.Version + name, version, filename, ok := eco.cacheKey(pv, file) + if !ok { + continue } - result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, fileDigest(file.Integrity)) + matched++ + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filename, file.URL, fileDigest(file.Integrity)) if err != nil { fail(fmt.Errorf("%s: %w", file.Filename, err)) return @@ -298,9 +303,9 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr size += result.Artifact.Size } } - switch { - case matched == 0 && wantFile != "": - fail(fmt.Errorf("no file named %q", wantFile)) + switch selection := eco.selection(pv); { + case matched == 0 && selection != "": + fail(fmt.Errorf("no file matches %s", selection)) case matched == 0: fail(errors.New("version has no files")) case fetched == 0: diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 89596ade..d8778b5c 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "strings" "github.com/git-pkgs/purl" "github.com/git-pkgs/sbom" @@ -14,7 +15,8 @@ type PackageVersion struct { Name string Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and - // arch; PyPI uses file_name to mirror one file of the version. + // arch; PyPI uses file_name, and conda subdir, build and type, to select + // which files of the version are mirrored. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -67,19 +69,20 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro ecosystem := handlerEcosystem(p.Type) name := p.Name - if p.Namespace != "" { + if ecosystem == "conda" { + name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) + } else if p.Namespace != "" { name = p.Namespace + "/" + p.Name } + qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) - if p.Version != "" || isImageEcosystem(ecosystem) { - // Images are never expanded to all tags; an unversioned image - // fails when it is mirrored. - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version} + // Images are never expanded to all tags, and conda packages without + // a channel aren't listed; both fail when they are mirrored. + if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() pv.Platforms = s.Platforms - } else if fileName := p.Qualifier("file_name"); fileName != "" && ecosystem == "pypi" { - pv.Qualifiers = map[string]string{"file_name": fileName} } if err := fn(pv); err != nil { return err @@ -93,7 +96,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { + if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers}); err != nil { return err } } diff --git a/internal/server/mirror_conda_test.go b/internal/server/mirror_conda_test.go new file mode 100644 index 00000000..e75ac9c6 --- /dev/null +++ b/internal/server/mirror_conda_test.go @@ -0,0 +1,161 @@ +package server + +import ( + "crypto/md5" //nolint:gosec // anaconda.org gives older files only an MD5 + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +const ( + condaLinuxFile = "linux-64/example-pkg-1.0-h123_0.conda" + condaNoarchFile = "noarch/example-pkg-1.0-pyh_0.tar.bz2" +) + +// condaUpstream serves the anaconda.org package API for +// conda-forge/example-pkg 1.0, with a linux-64 build that has a SHA-256 and +// a noarch build that, like older uploads, has only an MD5. It counts file +// downloads by subdir and filename. +type condaUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func newCondaUpstream(t *testing.T) *condaUpstream { + t.Helper() + u := &condaUpstream{ + files: map[string]string{condaLinuxFile: "linux build", condaNoarchFile: "noarch build"}, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/package/conda-forge/example-pkg" { + linuxSum := sha256.Sum256([]byte(u.files[condaLinuxFile])) + noarchSum := md5.Sum([]byte(u.files[condaNoarchFile])) //nolint:gosec // see import + file := func(basename, build string) map[string]any { + subdir, _, _ := strings.Cut(basename, "/") + return map[string]any{ + "version": "1.0", + "basename": basename, + "download_url": u.URL + "/download/conda-forge/example-pkg/1.0/" + basename, + "attrs": map[string]any{"subdir": subdir, "build": build}, + "size": len(u.files[basename]), + } + } + linux := file(condaLinuxFile, "h123_0") + linux["sha256"] = hex.EncodeToString(linuxSum[:]) + noarch := file(condaNoarchFile, "pyh_0") + noarch["md5"] = hex.EncodeToString(noarchSum[:]) + _ = json.NewEncoder(w).Encode(map[string]any{ + "name": "example-pkg", + "versions": []string{"1.0"}, + "files": []any{linux, noarch}, + }) + return + } + basename := strings.TrimPrefix(r.URL.Path, "/download/conda-forge/example-pkg/1.0/") + body, ok := u.files[basename] + if !ok { + http.NotFound(w, r) + return + } + u.mu.Lock() + u.hits[basename]++ + u.mu.Unlock() + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *condaUpstream) hitsFor(basename string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[basename] +} + +// mirrorConda mirrors purls with files listed from upstream. The proxy only +// lists conda files from anaconda.org itself, so the test points the lister +// at the fake directly. +func mirrorConda(t *testing.T, upstream *condaUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + versions := mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(upstream.Client())), + URL: func(string, string) (string, error) { return upstream.URL, nil }, + } + return mirrorWithVersions(t, config.Default(), versions, purls...) +} + +func TestMirrorCondaFetchesEveryFile(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, proxy := mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, basename := range []string{condaLinuxFile, condaNoarchFile} { + if got := upstream.hitsFor(basename); got != 1 { + t.Errorf("%s fetched %d times, want 1", basename, got) + } + } + + // conda gets both files from the cache once the upstream is down. + upstream.Close() + conda := handler.NewCondaHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + for basename, want := range upstream.files { + rec := httptest.NewRecorder() + conda.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/conda-forge/"+basename, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", basename, rec.Code, rec.Body, want) + } + } + + progress, _ = mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0") + if progress.Failed != 1 { + t.Errorf("failed = %d after the upstream closed, want 1", progress.Failed) + } +} + +func TestMirrorCondaQualifiersSelectFiles(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, _ := mirrorConda(t, upstream, "pkg:conda/example-pkg@1.0?channel=conda-forge&subdir=noarch") + + if progress.Completed != 1 || upstream.hitsFor(condaNoarchFile) != 1 { + t.Errorf("completed = %d, noarch hits = %d, errors = %v; want the noarch build", progress.Completed, upstream.hitsFor(condaNoarchFile), progress.Errors) + } + if upstream.hitsFor(condaLinuxFile) != 0 { + t.Error("the linux-64 build was mirrored although subdir names noarch") + } + + progress, _ = mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0?subdir=linux-64&type=tar.bz2") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "subdir=linux-64 type=tar.bz2") { + t.Errorf("failed = %d, errors = %v; want a failure naming the selection", progress.Failed, progress.Errors) + } +} + +func TestMirrorCondaNeedsChannel(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, _ := mirrorConda(t, upstream, "pkg:conda/example-pkg@1.0") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "channel") { + t.Errorf("failed = %d, errors = %v; want a failure asking for a channel", progress.Failed, progress.Errors) + } + if upstream.hitsFor(condaLinuxFile)+upstream.hitsFor(condaNoarchFile) != 0 { + t.Error("files were fetched for a package without a channel") + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 66f8d576..142594c9 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -36,6 +36,13 @@ func newTarballUpstream(t *testing.T, path, body string) (*httptest.Server, *ato // mirrorWithConfig mirrors purls with the proxy proxy mirror builds for cfg. func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + return mirrorWithVersions(t, cfg, nil, purls...) +} + +// mirrorWithVersions mirrors purls, listing versions and files with versions, +// or from the configured upstreams when it is nil. +func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.VersionLister, purls ...string) (*mirror.Progress, *handler.Proxy) { t.Helper() cfg.Upstream.AllowLoopback = true db, err := database.Create(t.TempDir() + "/test.db") @@ -53,7 +60,10 @@ func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirro logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy, _ := NewMirrorProxy(cfg, db, store, logger) m := mirror.New(proxy, db, store, logger, 1) - m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + m.Versions = versions + if versions == nil { + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + } progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls}) if err != nil { t.Fatalf("Run() error = %v", err) diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 9b981fdc..a7b0689d 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -2,6 +2,7 @@ package server import ( "cmp" + "errors" "fmt" "net/http" "strings" @@ -71,7 +72,12 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, case "cargo": return publicOnly("upstream.cargo", u.Cargo, d.Cargo) case "conda": - return publicOnly("upstream.conda", u.Conda, d.Conda) + // Files are listed from anaconda.org even for a given version, so a + // version doesn't help here. + if upstreamOr(u.Conda, d.Conda) != strings.TrimSuffix(d.Conda, "/") { + return "", errors.New("mirroring conda packages from a custom upstream.conda isn't supported") + } + return "", nil case "julia": return publicOnly("upstream.julia", u.Julia, d.Julia) default: From ae9cf36f2623837d3d23718cce68b8e4d9f25465 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:36:43 +0100 Subject: [PATCH 3/3] Mirror Composer releases Mirroring pkg:composer failed with unsupported ecosystem. A tagged release is now mirrored with the dist archive upstream.composer lists, under the file name the Composer handler serves it as and checked against the listed shasum. The release's publish time is stored as the handler stores it, so the archive is served from the cache without the metadata, with cooldown on as well. Branches are refused, since the handler reads the metadata for every branch download, and so are archive URLs with a query and packages in upstream.composer_routes. The mirror now finds the whole version rather than its files, for the dist type and publish time. Part of #414. --- docs/configuration.md | 2 + internal/handler/composer.go | 30 ++++++ internal/mirror/mirror.go | 74 ++++++++++---- internal/mirror/versions.go | 18 ++-- internal/server/mirror_composer_test.go | 126 ++++++++++++++++++++++++ internal/server/mirror_versions.go | 2 +- 6 files changed, 225 insertions(+), 27 deletions(-) create mode 100644 internal/server/mirror_composer_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 49994ad0..217d4f89 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1512,6 +1512,8 @@ A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdi A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. +A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 92eae0ae..bc3e17c7 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1011,6 +1011,36 @@ func composerIntegrity(shasum, version string, listed bool) string { return sha1Checksum(shasum) } +// ComposerReleaseArchive returns the file name the Composer handler caches a +// tagged release's archive under, and the integrity it checks it against, +// given the dist the release's metadata lists. It refuses versions the +// handler only serves after reading the metadata again: branches, whose +// archive changes with every push, and archives whose URL has a query, which +// clients ask for with one. +func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename, declared string, err error) { + switch { + case isDevVersion(version): + return "", "", fmt.Errorf("%s is a branch; Composer branches are fetched when requested, not mirrored", version) + case distURL == "": + return "", "", fmt.Errorf("%s has no dist archive", version) + case strings.ContainsAny(distURL, "?#"): + return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version) + } + return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil +} + +// RecordComposerPublishTime stores the publish time of a tagged release from +// the default repository, as the Composer handler does when it resolves the +// release's dist, so the archive is served from the cache without the +// metadata, with or without cooldown. +func (p *Proxy) RecordComposerPublishTime(packageName, version string, publishedAt time.Time) error { + if p.DB == nil || isDevVersion(version) || publishedAt.IsZero() { + return nil + } + return p.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version), + canonicalPackagePURL("composer", packageName), publishedAt) +} + // proxyUpstream forwards a request to packagist.org without caching. func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 27691615..6bd8d183 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -232,15 +232,29 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorFiles(ctx, pv, eco, tracker) return } + if pv.Ecosystem == "composer" { + m.mirrorComposer(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) + m.recordFailure(tracker, pv, err) return } + m.recordResult(tracker, pv, result) +} + +// recordFailure counts and logs a package that could not be mirrored. +func (m *Mirror) recordFailure(tracker *progressTracker, pv PackageVersion, err error) { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) +} +// recordResult closes the fetched artifact and counts it as mirrored, or as +// skipped when it was already cached. +func (m *Mirror) recordResult(tracker *progressTracker, pv PackageVersion, result *handler.CacheResult) { if result.Reader != nil { _ = result.Reader.Close() } @@ -258,30 +272,56 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog } } +// mirrorComposer mirrors the archive of a tagged Composer release under the +// file name the Composer handler serves it as, checked against the listed +// shasum, and records the release's publish time, which the handler needs to +// serve the archive from the cache without the metadata. +func (m *Mirror) mirrorComposer(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + // Packagist names are lowercase, and so are the paths Composer requests. + name := strings.ToLower(pv.Name) + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + distURL, _ := v.Metadata["dist_url"].(string) + distType, _ := v.Metadata["dist_type"].(string) + filename, declared, err := handler.ComposerReleaseArchive(v.Number, distURL, distType, strings.TrimPrefix(v.Integrity, "sha1-")) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, v.Number, filename, distURL, declared) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + if err := m.proxy.RecordComposerPublishTime(name, v.Number, v.PublishedAt); err != nil { + m.logger.Warn("failed to store composer publish time", + "name", name, "version", v.Number, "error", err) + } + m.recordResult(tracker, pv, result) +} + // mirrorFiles mirrors every file of a version, or those its qualifiers // select. Each file is cached under the name and version the download // handler reads from its filename, so clients are served it. func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) { - fail := func(err error) { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) - } + fail := func(err error) { m.recordFailure(tracker, pv, err) } if eco.check != nil { if err := eco.check(pv); err != nil { fail(err) return } } - files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, pv.Name, pv.Version) if err != nil { fail(fmt.Errorf("listing files: %w", err)) return } var matched, fetched int var size int64 - for _, file := range files { + for _, file := range v.Artifacts { if !eco.selects(pv, file) { continue } @@ -333,11 +373,11 @@ func fileDigest(integrity string) string { return "" } -// fileLister lists files from where Versions lists versions, so both come -// from the configured upstream. -func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do - if lister, ok := m.Versions.(FileLister); ok { - return lister +// versionFinder finds versions where Versions lists them, so both come from +// the configured upstream. +func (m *Mirror) versionFinder() VersionFinder { //nolint:ireturn // any finder will do + if finder, ok := m.Versions.(VersionFinder); ok { + return finder } return RegistryVersions{} } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8eab11fa..1b68a6dd 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -23,10 +23,10 @@ type RegistryVersions struct { URL func(purlType, name string) (string, error) } -// FileLister lists the files of one package version, for ecosystems that -// publish several files per version. -type FileLister interface { - ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) +// VersionFinder finds one version of a package, with the files it +// publishes, for ecosystems the mirror fetches from the version's metadata. +type VersionFinder interface { + FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } // ListVersions implements VersionLister. @@ -42,18 +42,18 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri return result, nil } -// ListFiles implements FileLister. -func (r RegistryVersions) ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) { +// FindVersion implements VersionFinder. +func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) { versions, err := r.fetchVersions(ctx, ecosystem, name) if err != nil { - return nil, err + return registries.Version{}, err } for _, v := range versions { if v.Number == version { - return v.Artifacts, nil + return v, nil } } - return nil, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) + return registries.Version{}, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { diff --git a/internal/server/mirror_composer_test.go b/internal/server/mirror_composer_test.go new file mode 100644 index 00000000..b51aadf7 --- /dev/null +++ b/internal/server/mirror_composer_test.go @@ -0,0 +1,126 @@ +package server + +import ( + "crypto/sha1" //nolint:gosec // Packagist lists SHA-1 shasums + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const composerCommit = "0123456789abcdef0123456789abcdef01234567" + +// composerUpstream serves the Packagist API for acme/widget with release +// 1.0.0, published in 2020, and branch dev-main, and the 1.0.0 archive. +type composerUpstream struct { + *httptest.Server + archive string + shasum string + apiHits atomic.Int32 + distHits atomic.Int32 +} + +func newComposerUpstream(t *testing.T) *composerUpstream { + t.Helper() + u := &composerUpstream{archive: composerZip(`{"name":"acme/widget"}`)} + sum := sha1.Sum([]byte(u.archive)) //nolint:gosec // see import + u.shasum = hex.EncodeToString(sum[:]) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/packages/acme/widget.json": + u.apiHits.Add(1) + dist := func(shasum string) map[string]string { + return map[string]string{"type": "zip", "url": u.URL + "/zipball/" + composerCommit, "shasum": shasum} + } + _ = json.NewEncoder(w).Encode(map[string]any{"package": map[string]any{ + "name": "acme/widget", + "versions": map[string]any{ + "1.0.0": map[string]any{"version": "1.0.0", "time": "2020-01-02T03:04:05+00:00", "dist": dist(u.shasum)}, + "dev-main": map[string]any{"version": "dev-main", "time": "2026-10-01T00:00:00+00:00", "dist": dist("")}, + }, + }}) + case "/zipball/" + composerCommit: + u.distHits.Add(1) + _, _ = io.WriteString(w, u.archive) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func composerMirrorConfig(upstream *composerUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.Composer = upstream.URL + cfg.Upstream.ComposerRepository = upstream.URL + return cfg +} + +func TestMirrorComposerReleaseIsServedFromCache(t *testing.T) { + upstream := newComposerUpstream(t) + cfg := composerMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:composer/Acme/Widget@1.0.0") + + if progress.Completed != 1 || upstream.distHits.Load() != 1 { + t.Fatalf("completed = %d, dist hits = %d, errors = %v", progress.Completed, upstream.distHits.Load(), progress.Errors) + } + + // Composer gets the archive from the cache once the upstream is down, + // with cooldown on as well: the mirror recorded the publish time. + upstream.Close() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + composer := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.Composer, cfg.Upstream.ComposerRepository) + rec := httptest.NewRecorder() + composer.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, + "/files/acme/widget/1.0.0/"+composerCommit+".zip", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != upstream.archive { + t.Errorf("archive from cache: status = %d, body %q; want 200 and the mirrored archive", rec.Code, strings.TrimSpace(rec.Body.String())) + } +} + +func TestMirrorComposerRefusesShasumMismatch(t *testing.T) { + upstream := newComposerUpstream(t) + upstream.shasum = strings.Repeat("0", 40) + + progress, _ := mirrorWithConfig(t, composerMirrorConfig(upstream), "pkg:composer/acme/widget@1.0.0") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("completed = %d, failed = %d, errors = %v; want the archive refused", progress.Completed, progress.Failed, progress.Errors) + } +} + +func TestMirrorComposerRefusesBranches(t *testing.T) { + upstream := newComposerUpstream(t) + + progress, _ := mirrorWithConfig(t, composerMirrorConfig(upstream), "pkg:composer/acme/widget@dev-main") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "branch") { + t.Errorf("failed = %d, errors = %v; want the branch refused", progress.Failed, progress.Errors) + } + if upstream.distHits.Load() != 0 { + t.Error("the branch archive was fetched") + } +} + +func TestMirrorComposerRefusesRoutedPackages(t *testing.T) { + upstream := newComposerUpstream(t) + cfg := composerMirrorConfig(upstream) + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:composer/acme/widget@1.0.0") + + if progress.Failed != 1 || upstream.apiHits.Load() != 0 || upstream.distHits.Load() != 0 { + t.Errorf("failed = %d, api hits = %d, dist hits = %d; want a routed package refused without asking Packagist", + progress.Failed, upstream.apiHits.Load(), upstream.distHits.Load()) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index a7b0689d..404388d7 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -62,7 +62,7 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, return upstreamOr(u.NuGet, d.NuGet) + "/v3", nil case "composer": if _, ok := handler.RoutedPackageURL(u.ComposerRoutes, name); ok { - return "", fmt.Errorf("listing versions of packages in upstream.composer_routes isn't supported; give a version") + return "", errors.New("mirroring packages in upstream.composer_routes isn't supported") } return upstreamOr(u.Composer, d.Composer), nil case "cran":