diff --git a/docs/configuration.md b/docs/configuration.md index 5a877e38..217d4f89 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1510,6 +1510,10 @@ Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`. +A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else. + +A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 92eae0ae..bc3e17c7 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1011,6 +1011,36 @@ func composerIntegrity(shasum, version string, listed bool) string { return sha1Checksum(shasum) } +// ComposerReleaseArchive returns the file name the Composer handler caches a +// tagged release's archive under, and the integrity it checks it against, +// given the dist the release's metadata lists. It refuses versions the +// handler only serves after reading the metadata again: branches, whose +// archive changes with every push, and archives whose URL has a query, which +// clients ask for with one. +func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename, declared string, err error) { + switch { + case isDevVersion(version): + return "", "", fmt.Errorf("%s is a branch; Composer branches are fetched when requested, not mirrored", version) + case distURL == "": + return "", "", fmt.Errorf("%s has no dist archive", version) + case strings.ContainsAny(distURL, "?#"): + return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version) + } + return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil +} + +// RecordComposerPublishTime stores the publish time of a tagged release from +// the default repository, as the Composer handler does when it resolves the +// release's dist, so the archive is served from the cache without the +// metadata, with or without cooldown. +func (p *Proxy) RecordComposerPublishTime(packageName, version string, publishedAt time.Time) error { + if p.DB == nil || isDevVersion(version) || publishedAt.IsZero() { + return nil + } + return p.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version), + canonicalPackagePURL("composer", packageName), publishedAt) +} + // proxyUpstream forwards a request to packagist.org without caching. func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1142ccf2..b372468c 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -142,9 +142,15 @@ func (h *CondaHandler) isPackageFile(filename string) bool { } // parseFilename extracts name and version from a Conda package filename. +func (h *CondaHandler) parseFilename(filename string) (name, version string) { + return CondaFilenameIdentity(filename) +} + +// CondaFilenameIdentity returns the name and version a download of filename +// is cached under, or empty strings when it is not a package file name. // Conda filenames are: {name}-{version}-{build}.{ext} // e.g., "numpy-1.24.0-py311h64a7726_0.conda" -func (h *CondaHandler) parseFilename(filename string) (name, version string) { +func CondaFilenameIdentity(filename string) (name, version string) { // Remove extension base := filename for _, ext := range []string{".conda", ".tar.bz2"} { diff --git a/internal/mirror/files.go b/internal/mirror/files.go new file mode 100644 index 00000000..8cf61fe7 --- /dev/null +++ b/internal/mirror/files.go @@ -0,0 +1,132 @@ +package mirror + +import ( + "fmt" + "slices" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// fileEcosystem describes an ecosystem whose versions publish several files, +// each mirrored on its own. +type fileEcosystem struct { + // qualifiers are the PURL qualifiers that select files. A file is + // mirrored when each one given matches the value attrs reports for it. + qualifiers []string + attrs func(file registries.Artifact) map[string]string + // cacheKey returns the name, version and filename the download handler + // caches file under, or false when the handler would not cache it. + cacheKey func(pv PackageVersion, file registries.Artifact) (name, version, filename string, ok bool) + // check refuses a package the ecosystem can't mirror, before its files + // are listed. + check func(pv PackageVersion) error +} + +var fileEcosystems = map[string]fileEcosystem{ + "pypi": { + qualifiers: []string{"file_name"}, + attrs: func(file registries.Artifact) map[string]string { + return map[string]string{"file_name": file.Filename} + }, + cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { + name, version := handler.PyPIFilenameIdentity(file.Filename) + if name == "" { + name, version = pv.Name, pv.Version + } + return name, version, file.Filename, true + }, + }, + "conda": { + qualifiers: []string{"subdir", "build", "type"}, + attrs: func(file registries.Artifact) map[string]string { + subdir, _ := file.Metadata["subdir"].(string) + build, _ := file.Metadata["build"].(string) + return map[string]string{"subdir": subdir, "build": build, "type": condaFileType(file.Filename)} + }, + cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) { + channel, _, _ := strings.Cut(pv.Name, "/") + subdir, _ := file.Metadata["subdir"].(string) + name, version := handler.CondaFilenameIdentity(file.Filename) + if name == "" || subdir == "" || condaFileType(file.Filename) == "" { + return "", "", "", false + } + return channel + "/" + name, version, subdir + "/" + file.Filename, true + }, + check: func(pv PackageVersion) error { + if !strings.Contains(pv.Name, "/") { + return fmt.Errorf("conda packages need a channel, such as pkg:conda/conda-forge/%s", pv.Name) + } + return nil + }, + }, +} + +// condaFileType returns the PURL type qualifier of a conda package file. +func condaFileType(filename string) string { + switch { + case strings.HasSuffix(filename, ".conda"): + return "conda" + case strings.HasSuffix(filename, ".tar.bz2"): + return "tar.bz2" + default: + return "" + } +} + +// selection returns the qualifiers of pv that select files, as +// "key=value" pairs, for error messages. +func (e fileEcosystem) selection(pv PackageVersion) string { + var pairs []string + for _, key := range e.qualifiers { + if value := pv.Qualifiers[key]; value != "" { + pairs = append(pairs, key+"="+value) + } + } + return strings.Join(pairs, " ") +} + +// selects reports whether file matches every selecting qualifier of pv. +func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { + attrs := e.attrs(file) + for _, key := range e.qualifiers { + if want := pv.Qualifiers[key]; want != "" && attrs[key] != want { + return false + } + } + return true +} + +// fileQualifiers keeps the qualifiers of a PURL that select files of its +// ecosystem, or returns nil. +func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string { + e, ok := fileEcosystems[ecosystem] + if !ok { + return nil + } + var kept map[string]string + for key, value := range qualifiers { + if value != "" && slices.Contains(e.qualifiers, key) { + if kept == nil { + kept = map[string]string{} + } + kept[key] = value + } + } + return kept +} + +// condaName returns a conda package name with its channel, taken from the +// namespace or the channel qualifier. Without either it stays bare, and +// mirroring refuses it. +func condaName(namespace, name, channel string) string { + switch { + case namespace != "": + return namespace + "/" + name + case channel != "": + return channel + "/" + name + default: + return name + } +} diff --git a/internal/mirror/files_test.go b/internal/mirror/files_test.go new file mode 100644 index 00000000..d84af723 --- /dev/null +++ b/internal/mirror/files_test.go @@ -0,0 +1,38 @@ +package mirror + +import ( + "reflect" + "testing" +) + +func TestPURLSourceCondaChannelAndQualifiers(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0", "1.1"}} + source := &PURLSource{ + PURLs: []string{ + "pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&build=py312h_0&type=conda", + "pkg:conda/numpy@2.1.0?channel=conda-forge&repository_url=https://example.com", + "pkg:conda/numpy@2.1.0", + "pkg:conda/numpy", + "pkg:conda/numpy?channel=bioconda&subdir=noarch", + }, + Versions: versions, + } + + got := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0", + Qualifiers: map[string]string{"subdir": "linux-64", "build": "py312h_0", "type": "conda"}}, + {Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0"}, + {Ecosystem: "conda", Name: "numpy", Version: "2.1.0"}, + {Ecosystem: "conda", Name: "numpy"}, + {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.0", Qualifiers: map[string]string{"subdir": "noarch"}}, + {Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.1", Qualifiers: map[string]string{"subdir": "noarch"}}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("items = %+v\nwant %+v", got, want) + } + if !reflect.DeepEqual(versions.asked, []string{"conda/bioconda/numpy"}) { + t.Errorf("listed versions of %v, want only the package with a channel", versions.asked) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ddf97364..6bd8d183 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "sync/atomic" "time" @@ -227,19 +228,33 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } - if pv.Ecosystem == "pypi" { - m.mirrorFiles(ctx, pv, tracker) + if eco, ok := fileEcosystems[pv.Ecosystem]; ok { + m.mirrorFiles(ctx, pv, eco, tracker) + return + } + if pv.Ecosystem == "composer" { + m.mirrorComposer(ctx, pv, tracker) return } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) + m.recordFailure(tracker, pv, err) return } + m.recordResult(tracker, pv, result) +} +// recordFailure counts and logs a package that could not be mirrored. +func (m *Mirror) recordFailure(tracker *progressTracker, pv PackageVersion, err error) { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) +} + +// recordResult closes the fetched artifact and counts it as mirrored, or as +// skipped when it was already cached. +func (m *Mirror) recordResult(tracker *progressTracker, pv PackageVersion, result *handler.CacheResult) { if result.Reader != nil { _ = result.Reader.Close() } @@ -257,34 +272,65 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog } } -// mirrorFiles mirrors every file of a PyPI version, or only the one named by -// the file_name qualifier. Each file is cached under the name and version -// the download handler reads from its filename, so clients are served it. -func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - fail := func(err error) { - tracker.failed.Add(1) - tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) - m.logger.Warn("mirror failed", - "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) +// mirrorComposer mirrors the archive of a tagged Composer release under the +// file name the Composer handler serves it as, checked against the listed +// shasum, and records the release's publish time, which the handler needs to +// serve the archive from the cache without the metadata. +func (m *Mirror) mirrorComposer(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + // Packagist names are lowercase, and so are the paths Composer requests. + name := strings.ToLower(pv.Name) + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return } - files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) + distURL, _ := v.Metadata["dist_url"].(string) + distType, _ := v.Metadata["dist_type"].(string) + filename, declared, err := handler.ComposerReleaseArchive(v.Number, distURL, distType, strings.TrimPrefix(v.Integrity, "sha1-")) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, v.Number, filename, distURL, declared) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + if err := m.proxy.RecordComposerPublishTime(name, v.Number, v.PublishedAt); err != nil { + m.logger.Warn("failed to store composer publish time", + "name", name, "version", v.Number, "error", err) + } + m.recordResult(tracker, pv, result) +} + +// mirrorFiles mirrors every file of a version, or those its qualifiers +// select. Each file is cached under the name and version the download +// handler reads from its filename, so clients are served it. +func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + if eco.check != nil { + if err := eco.check(pv); err != nil { + fail(err) + return + } + } + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, pv.Name, pv.Version) if err != nil { fail(fmt.Errorf("listing files: %w", err)) return } - wantFile := pv.Qualifiers["file_name"] var matched, fetched int var size int64 - for _, file := range files { - if wantFile != "" && file.Filename != wantFile { + for _, file := range v.Artifacts { + if !eco.selects(pv, file) { continue } - matched++ - name, version := handler.PyPIFilenameIdentity(file.Filename) - if name == "" { - name, version = pv.Name, pv.Version + name, version, filename, ok := eco.cacheKey(pv, file) + if !ok { + continue } - result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity) + matched++ + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filename, file.URL, fileDigest(file.Integrity)) if err != nil { fail(fmt.Errorf("%s: %w", file.Filename, err)) return @@ -297,9 +343,9 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr size += result.Artifact.Size } } - switch { - case matched == 0 && wantFile != "": - fail(fmt.Errorf("no file named %q", wantFile)) + switch selection := eco.selection(pv); { + case matched == 0 && selection != "": + fail(fmt.Errorf("no file matches %s", selection)) case matched == 0: fail(errors.New("version has no files")) case fetched == 0: @@ -315,11 +361,23 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr } } -// fileLister lists files from where Versions lists versions, so both come -// from the configured upstream. -func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do - if lister, ok := m.Versions.(FileLister); ok { - return lister +// fileDigest returns the SHA-256 of one file as a digest, which is checked +// against that file alone. The integrity GetOrFetchArtifactFromURLWithIntegrity +// takes is kept with the version instead, and the files of a version that +// share a name and version would overwrite each other's. Other checksums are +// not checked, as the download handlers don't check them either. +func fileDigest(integrity string) string { + if sum, ok := strings.CutPrefix(integrity, "sha256-"); ok { + return "sha256:" + sum + } + return "" +} + +// versionFinder finds versions where Versions lists them, so both come from +// the configured upstream. +func (m *Mirror) versionFinder() VersionFinder { //nolint:ireturn // any finder will do + if finder, ok := m.Versions.(VersionFinder); ok { + return finder } return RegistryVersions{} } diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 89596ade..d8778b5c 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "strings" "github.com/git-pkgs/purl" "github.com/git-pkgs/sbom" @@ -14,7 +15,8 @@ type PackageVersion struct { Name string Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and - // arch; PyPI uses file_name to mirror one file of the version. + // arch; PyPI uses file_name, and conda subdir, build and type, to select + // which files of the version are mirrored. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -67,19 +69,20 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro ecosystem := handlerEcosystem(p.Type) name := p.Name - if p.Namespace != "" { + if ecosystem == "conda" { + name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) + } else if p.Namespace != "" { name = p.Namespace + "/" + p.Name } + qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) - if p.Version != "" || isImageEcosystem(ecosystem) { - // Images are never expanded to all tags; an unversioned image - // fails when it is mirrored. - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version} + // Images are never expanded to all tags, and conda packages without + // a channel aren't listed; both fail when they are mirrored. + if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() pv.Platforms = s.Platforms - } else if fileName := p.Qualifier("file_name"); fileName != "" && ecosystem == "pypi" { - pv.Qualifiers = map[string]string{"file_name": fileName} } if err := fn(pv); err != nil { return err @@ -93,7 +96,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { + if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers}); err != nil { return err } } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8eab11fa..1b68a6dd 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -23,10 +23,10 @@ type RegistryVersions struct { URL func(purlType, name string) (string, error) } -// FileLister lists the files of one package version, for ecosystems that -// publish several files per version. -type FileLister interface { - ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) +// VersionFinder finds one version of a package, with the files it +// publishes, for ecosystems the mirror fetches from the version's metadata. +type VersionFinder interface { + FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } // ListVersions implements VersionLister. @@ -42,18 +42,18 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri return result, nil } -// ListFiles implements FileLister. -func (r RegistryVersions) ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) { +// FindVersion implements VersionFinder. +func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) { versions, err := r.fetchVersions(ctx, ecosystem, name) if err != nil { - return nil, err + return registries.Version{}, err } for _, v := range versions { if v.Number == version { - return v.Artifacts, nil + return v, nil } } - return nil, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) + return registries.Version{}, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { diff --git a/internal/server/mirror_composer_test.go b/internal/server/mirror_composer_test.go new file mode 100644 index 00000000..b51aadf7 --- /dev/null +++ b/internal/server/mirror_composer_test.go @@ -0,0 +1,126 @@ +package server + +import ( + "crypto/sha1" //nolint:gosec // Packagist lists SHA-1 shasums + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const composerCommit = "0123456789abcdef0123456789abcdef01234567" + +// composerUpstream serves the Packagist API for acme/widget with release +// 1.0.0, published in 2020, and branch dev-main, and the 1.0.0 archive. +type composerUpstream struct { + *httptest.Server + archive string + shasum string + apiHits atomic.Int32 + distHits atomic.Int32 +} + +func newComposerUpstream(t *testing.T) *composerUpstream { + t.Helper() + u := &composerUpstream{archive: composerZip(`{"name":"acme/widget"}`)} + sum := sha1.Sum([]byte(u.archive)) //nolint:gosec // see import + u.shasum = hex.EncodeToString(sum[:]) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/packages/acme/widget.json": + u.apiHits.Add(1) + dist := func(shasum string) map[string]string { + return map[string]string{"type": "zip", "url": u.URL + "/zipball/" + composerCommit, "shasum": shasum} + } + _ = json.NewEncoder(w).Encode(map[string]any{"package": map[string]any{ + "name": "acme/widget", + "versions": map[string]any{ + "1.0.0": map[string]any{"version": "1.0.0", "time": "2020-01-02T03:04:05+00:00", "dist": dist(u.shasum)}, + "dev-main": map[string]any{"version": "dev-main", "time": "2026-10-01T00:00:00+00:00", "dist": dist("")}, + }, + }}) + case "/zipball/" + composerCommit: + u.distHits.Add(1) + _, _ = io.WriteString(w, u.archive) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func composerMirrorConfig(upstream *composerUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.Composer = upstream.URL + cfg.Upstream.ComposerRepository = upstream.URL + return cfg +} + +func TestMirrorComposerReleaseIsServedFromCache(t *testing.T) { + upstream := newComposerUpstream(t) + cfg := composerMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:composer/Acme/Widget@1.0.0") + + if progress.Completed != 1 || upstream.distHits.Load() != 1 { + t.Fatalf("completed = %d, dist hits = %d, errors = %v", progress.Completed, upstream.distHits.Load(), progress.Errors) + } + + // Composer gets the archive from the cache once the upstream is down, + // with cooldown on as well: the mirror recorded the publish time. + upstream.Close() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + composer := handler.NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.Composer, cfg.Upstream.ComposerRepository) + rec := httptest.NewRecorder() + composer.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, + "/files/acme/widget/1.0.0/"+composerCommit+".zip", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != upstream.archive { + t.Errorf("archive from cache: status = %d, body %q; want 200 and the mirrored archive", rec.Code, strings.TrimSpace(rec.Body.String())) + } +} + +func TestMirrorComposerRefusesShasumMismatch(t *testing.T) { + upstream := newComposerUpstream(t) + upstream.shasum = strings.Repeat("0", 40) + + progress, _ := mirrorWithConfig(t, composerMirrorConfig(upstream), "pkg:composer/acme/widget@1.0.0") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("completed = %d, failed = %d, errors = %v; want the archive refused", progress.Completed, progress.Failed, progress.Errors) + } +} + +func TestMirrorComposerRefusesBranches(t *testing.T) { + upstream := newComposerUpstream(t) + + progress, _ := mirrorWithConfig(t, composerMirrorConfig(upstream), "pkg:composer/acme/widget@dev-main") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "branch") { + t.Errorf("failed = %d, errors = %v; want the branch refused", progress.Failed, progress.Errors) + } + if upstream.distHits.Load() != 0 { + t.Error("the branch archive was fetched") + } +} + +func TestMirrorComposerRefusesRoutedPackages(t *testing.T) { + upstream := newComposerUpstream(t) + cfg := composerMirrorConfig(upstream) + cfg.Upstream.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} + + progress, _ := mirrorWithConfig(t, cfg, "pkg:composer/acme/widget@1.0.0") + + if progress.Failed != 1 || upstream.apiHits.Load() != 0 || upstream.distHits.Load() != 0 { + t.Errorf("failed = %d, api hits = %d, dist hits = %d; want a routed package refused without asking Packagist", + progress.Failed, upstream.apiHits.Load(), upstream.distHits.Load()) + } +} diff --git a/internal/server/mirror_conda_test.go b/internal/server/mirror_conda_test.go new file mode 100644 index 00000000..e75ac9c6 --- /dev/null +++ b/internal/server/mirror_conda_test.go @@ -0,0 +1,161 @@ +package server + +import ( + "crypto/md5" //nolint:gosec // anaconda.org gives older files only an MD5 + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +const ( + condaLinuxFile = "linux-64/example-pkg-1.0-h123_0.conda" + condaNoarchFile = "noarch/example-pkg-1.0-pyh_0.tar.bz2" +) + +// condaUpstream serves the anaconda.org package API for +// conda-forge/example-pkg 1.0, with a linux-64 build that has a SHA-256 and +// a noarch build that, like older uploads, has only an MD5. It counts file +// downloads by subdir and filename. +type condaUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func newCondaUpstream(t *testing.T) *condaUpstream { + t.Helper() + u := &condaUpstream{ + files: map[string]string{condaLinuxFile: "linux build", condaNoarchFile: "noarch build"}, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/package/conda-forge/example-pkg" { + linuxSum := sha256.Sum256([]byte(u.files[condaLinuxFile])) + noarchSum := md5.Sum([]byte(u.files[condaNoarchFile])) //nolint:gosec // see import + file := func(basename, build string) map[string]any { + subdir, _, _ := strings.Cut(basename, "/") + return map[string]any{ + "version": "1.0", + "basename": basename, + "download_url": u.URL + "/download/conda-forge/example-pkg/1.0/" + basename, + "attrs": map[string]any{"subdir": subdir, "build": build}, + "size": len(u.files[basename]), + } + } + linux := file(condaLinuxFile, "h123_0") + linux["sha256"] = hex.EncodeToString(linuxSum[:]) + noarch := file(condaNoarchFile, "pyh_0") + noarch["md5"] = hex.EncodeToString(noarchSum[:]) + _ = json.NewEncoder(w).Encode(map[string]any{ + "name": "example-pkg", + "versions": []string{"1.0"}, + "files": []any{linux, noarch}, + }) + return + } + basename := strings.TrimPrefix(r.URL.Path, "/download/conda-forge/example-pkg/1.0/") + body, ok := u.files[basename] + if !ok { + http.NotFound(w, r) + return + } + u.mu.Lock() + u.hits[basename]++ + u.mu.Unlock() + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *condaUpstream) hitsFor(basename string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[basename] +} + +// mirrorConda mirrors purls with files listed from upstream. The proxy only +// lists conda files from anaconda.org itself, so the test points the lister +// at the fake directly. +func mirrorConda(t *testing.T, upstream *condaUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + versions := mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(upstream.Client())), + URL: func(string, string) (string, error) { return upstream.URL, nil }, + } + return mirrorWithVersions(t, config.Default(), versions, purls...) +} + +func TestMirrorCondaFetchesEveryFile(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, proxy := mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, basename := range []string{condaLinuxFile, condaNoarchFile} { + if got := upstream.hitsFor(basename); got != 1 { + t.Errorf("%s fetched %d times, want 1", basename, got) + } + } + + // conda gets both files from the cache once the upstream is down. + upstream.Close() + conda := handler.NewCondaHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + for basename, want := range upstream.files { + rec := httptest.NewRecorder() + conda.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/conda-forge/"+basename, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", basename, rec.Code, rec.Body, want) + } + } + + progress, _ = mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0") + if progress.Failed != 1 { + t.Errorf("failed = %d after the upstream closed, want 1", progress.Failed) + } +} + +func TestMirrorCondaQualifiersSelectFiles(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, _ := mirrorConda(t, upstream, "pkg:conda/example-pkg@1.0?channel=conda-forge&subdir=noarch") + + if progress.Completed != 1 || upstream.hitsFor(condaNoarchFile) != 1 { + t.Errorf("completed = %d, noarch hits = %d, errors = %v; want the noarch build", progress.Completed, upstream.hitsFor(condaNoarchFile), progress.Errors) + } + if upstream.hitsFor(condaLinuxFile) != 0 { + t.Error("the linux-64 build was mirrored although subdir names noarch") + } + + progress, _ = mirrorConda(t, upstream, "pkg:conda/conda-forge/example-pkg@1.0?subdir=linux-64&type=tar.bz2") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "subdir=linux-64 type=tar.bz2") { + t.Errorf("failed = %d, errors = %v; want a failure naming the selection", progress.Failed, progress.Errors) + } +} + +func TestMirrorCondaNeedsChannel(t *testing.T) { + upstream := newCondaUpstream(t) + + progress, _ := mirrorConda(t, upstream, "pkg:conda/example-pkg@1.0") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "channel") { + t.Errorf("failed = %d, errors = %v; want a failure asking for a channel", progress.Failed, progress.Errors) + } + if upstream.hitsFor(condaLinuxFile)+upstream.hitsFor(condaNoarchFile) != 0 { + t.Error("files were fetched for a package without a channel") + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 66f8d576..142594c9 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -36,6 +36,13 @@ func newTarballUpstream(t *testing.T, path, body string) (*httptest.Server, *ato // mirrorWithConfig mirrors purls with the proxy proxy mirror builds for cfg. func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + return mirrorWithVersions(t, cfg, nil, purls...) +} + +// mirrorWithVersions mirrors purls, listing versions and files with versions, +// or from the configured upstreams when it is nil. +func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.VersionLister, purls ...string) (*mirror.Progress, *handler.Proxy) { t.Helper() cfg.Upstream.AllowLoopback = true db, err := database.Create(t.TempDir() + "/test.db") @@ -53,7 +60,10 @@ func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirro logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy, _ := NewMirrorProxy(cfg, db, store, logger) m := mirror.New(proxy, db, store, logger, 1) - m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + m.Versions = versions + if versions == nil { + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + } progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls}) if err != nil { t.Fatalf("Run() error = %v", err) diff --git a/internal/server/mirror_pypi_test.go b/internal/server/mirror_pypi_test.go index 103e0f56..a5919579 100644 --- a/internal/server/mirror_pypi_test.go +++ b/internal/server/mirror_pypi_test.go @@ -16,13 +16,14 @@ import ( ) const ( - mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" - mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" + mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" + mirrorLinuxWheelPath = "/packages/23/45/example_pkg-1.0-cp312-cp312-manylinux_2_17_x86_64.whl" + mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" ) -// pypiUpstream serves the JSON API for example-pkg 1.0 with a wheel and an -// sdist, whose filenames spell the name differently, and counts file -// downloads by path. +// pypiUpstream serves the JSON API for example-pkg 1.0 with two wheels, which +// are cached under the same name and version, and an sdist, whose filename +// spells the name differently. It counts file downloads by path. type pypiUpstream struct { *httptest.Server mu sync.Mutex @@ -33,13 +34,13 @@ type pypiUpstream struct { func newPyPIUpstream(t *testing.T, digests map[string]string) *pypiUpstream { t.Helper() u := &pypiUpstream{ - files: map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"}, + files: map[string]string{mirrorWheelPath: "wheel", mirrorLinuxWheelPath: "linux wheel", mirrorSdistPath: "sdist"}, hits: map[string]int{}, } u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/pypi/example-pkg/json" { var files []map[string]any - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for _, path := range []string{mirrorWheelPath, mirrorLinuxWheelPath, mirrorSdistPath} { sum := sha256.Sum256([]byte(u.files[path])) digest := hex.EncodeToString(sum[:]) if d, ok := digests[path]; ok { @@ -93,16 +94,16 @@ func TestMirrorPyPIFetchesEveryFile(t *testing.T) { if progress.Completed != 1 || progress.Failed != 0 { t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) } - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for path := range upstream.files { if got := upstream.hitsFor(path); got != 1 { t.Errorf("%s fetched %d times, want 1", path, got) } } - // pip gets both files from the cache once the upstream is down. + // pip gets every file from the cache once the upstream is down. upstream.Close() pypi := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.PyPI, cfg.Upstream.PyPIDownload) - for path, want := range map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"} { + for path, want := range upstream.files { rec := httptest.NewRecorder() pypi.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) if rec.Code != http.StatusOK || rec.Body.String() != want { diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 9b981fdc..404388d7 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -2,6 +2,7 @@ package server import ( "cmp" + "errors" "fmt" "net/http" "strings" @@ -61,7 +62,7 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, return upstreamOr(u.NuGet, d.NuGet) + "/v3", nil case "composer": if _, ok := handler.RoutedPackageURL(u.ComposerRoutes, name); ok { - return "", fmt.Errorf("listing versions of packages in upstream.composer_routes isn't supported; give a version") + return "", errors.New("mirroring packages in upstream.composer_routes isn't supported") } return upstreamOr(u.Composer, d.Composer), nil case "cran": @@ -71,7 +72,12 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, case "cargo": return publicOnly("upstream.cargo", u.Cargo, d.Cargo) case "conda": - return publicOnly("upstream.conda", u.Conda, d.Conda) + // Files are listed from anaconda.org even for a given version, so a + // version doesn't help here. + if upstreamOr(u.Conda, d.Conda) != strings.TrimSuffix(d.Conda, "/") { + return "", errors.New("mirroring conda packages from a custom upstream.conda isn't supported") + } + return "", nil case "julia": return publicOnly("upstream.julia", u.Julia, d.Julia) default: