From 607a55e7464901c8644db56569cbe1d04f2803f6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 12:36:36 +0100 Subject: [PATCH] Mirror PyPI packages proxy mirror and mirror jobs failed for pypi with "unsupported ecosystem", since the resolver they use has no registry for it. A PyPI version has several files, so the mirror now lists them from upstream.pypi's JSON API, through the same client as version listing, and fetches each one, verified against its sha256. The file_name qualifier limits it to one file. Files are cached under the name and version the download handler reads from the filename, which can differ from the PURL (zope.interface's wheels are zope_interface), so pip is served them from the cache. --- docs/configuration.md | 2 + internal/handler/pypi.go | 12 ++- internal/mirror/mirror.go | 72 ++++++++++++++ internal/mirror/source.go | 4 +- internal/mirror/versions.go | 43 ++++++-- internal/server/mirror_fetch_test.go | 4 +- internal/server/mirror_pypi_test.go | 140 +++++++++++++++++++++++++++ 7 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 internal/server/mirror_pypi_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 70a18615..c64a196d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1506,6 +1506,8 @@ The `proxy mirror` command pre-populates the cache from various sources. It acce Packages are downloaded from the configured upstreams: npm, cargo, gem, golang, hex, pub, maven and nuget from `upstream.npm`, `upstream.cargo_download`, `upstream.gem`, `upstream.go`, `upstream.hex`, `upstream.pub`, `upstream.maven` and `upstream.nuget`. npm packages in `upstream.npm_routes` come from their route and are cached under the route's keys, so clients get them from the cache as they would after an install. Mirror API jobs do the same. +A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 021150f6..3c89b73c 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -649,13 +649,19 @@ var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".t // bdist_msi formats, which share a filename layout. var windowsInstallerExtensions = []string{".exe", ".msi"} -// parseFilename extracts package name and version from a PyPI filename. -// Handles wheels, sdists and legacy bdist formats: +func (h *PyPIHandler) parseFilename(filename string) (name, version string) { + return PyPIFilenameIdentity(filename) +} + +// PyPIFilenameIdentity extracts the package name and version a PyPI file is +// cached under from its filename, so proxy mirror stores files where the +// download handler looks for them. Handles wheels, sdists and legacy bdist +// formats: // - requests-2.31.0-py3-none-any.whl // - requests-2.31.0.tar.gz // - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg // - numpy-1.8.0.win32-py2.7.exe -func (h *PyPIHandler) parseFilename(filename string) (name, version string) { +func PyPIFilenameIdentity(filename string) (name, version string) { // PEP 658/714 core-metadata sidecars are the distribution filename plus // ".metadata"; they describe the same name and version. Without this, pip's // metadata-only fetches fall back to a hash-derived package identifier. diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 4b79ce9f..ddf97364 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -3,6 +3,7 @@ package mirror import ( "context" + "errors" "fmt" "log/slog" "sync" @@ -226,6 +227,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } + if pv.Ecosystem == "pypi" { + m.mirrorFiles(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { tracker.failed.Add(1) @@ -251,3 +256,70 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "size", result.Artifact.Size) } } + +// mirrorFiles mirrors every file of a PyPI version, or only the one named by +// the file_name qualifier. Each file is cached under the name and version +// the download handler reads from its filename, so clients are served it. +func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { + tracker.failed.Add(1) + tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error()) + m.logger.Warn("mirror failed", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err) + } + files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version) + if err != nil { + fail(fmt.Errorf("listing files: %w", err)) + return + } + wantFile := pv.Qualifiers["file_name"] + var matched, fetched int + var size int64 + for _, file := range files { + if wantFile != "" && file.Filename != wantFile { + continue + } + matched++ + name, version := handler.PyPIFilenameIdentity(file.Filename) + if name == "" { + name, version = pv.Name, pv.Version + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity) + if err != nil { + fail(fmt.Errorf("%s: %w", file.Filename, err)) + return + } + if result.Reader != nil { + _ = result.Reader.Close() + } + if !result.Cached { + fetched++ + size += result.Artifact.Size + } + } + switch { + case matched == 0 && wantFile != "": + fail(fmt.Errorf("no file named %q", wantFile)) + case matched == 0: + fail(errors.New("version has no files")) + case fetched == 0: + tracker.skipped.Add(1) + m.logger.Debug("already cached", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) + default: + tracker.completed.Add(1) + tracker.bytes.Add(size) + m.logger.Info("mirrored", + "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, + "files", fetched, "size", size) + } +} + +// fileLister lists files from where Versions lists versions, so both come +// from the configured upstream. +func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do + if lister, ok := m.Versions.(FileLister); ok { + return lister + } + return RegistryVersions{} +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index a88a3a8f..89596ade 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -14,7 +14,7 @@ type PackageVersion struct { Name string Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and - // arch. + // arch; PyPI uses file_name to mirror one file of the version. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -78,6 +78,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() pv.Platforms = s.Platforms + } else if fileName := p.Qualifier("file_name"); fileName != "" && ecosystem == "pypi" { + pv.Qualifiers = map[string]string{"file_name": fileName} } if err := fn(pv); err != nil { return err diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index f62102b9..8eab11fa 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -2,6 +2,7 @@ package mirror import ( "context" + "fmt" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" @@ -22,8 +23,40 @@ type RegistryVersions struct { URL func(purlType, name string) (string, error) } +// FileLister lists the files of one package version, for ecosystems that +// publish several files per version. +type FileLister interface { + ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) +} + // ListVersions implements VersionLister. func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + versions, err := r.fetchVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number + } + return result, nil +} + +// ListFiles implements FileLister. +func (r RegistryVersions) ListFiles(ctx context.Context, ecosystem, name, version string) ([]registries.Artifact, error) { + versions, err := r.fetchVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + for _, v := range versions { + if v.Number == version { + return v.Artifacts, nil + } + } + return nil, fmt.Errorf("%w: %s %s@%s", registries.ErrNotFound, ecosystem, name, version) +} + +func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -40,13 +73,5 @@ func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name stri if err != nil { return nil, err } - versions, err := reg.FetchVersions(ctx, name) - if err != nil { - return nil, err - } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number - } - return result, nil + return reg.FetchVersions(ctx, name) } diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 2b3970d0..66f8d576 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -52,7 +52,9 @@ func mirrorWithConfig(t *testing.T, cfg *config.Config, purls ...string) (*mirro } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) proxy, _ := NewMirrorProxy(cfg, db, store, logger) - progress, err := mirror.New(proxy, db, store, logger, 1).Run(context.Background(), &mirror.PURLSource{PURLs: purls}) + m := mirror.New(proxy, db, store, logger, 1) + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls}) if err != nil { t.Fatalf("Run() error = %v", err) } diff --git a/internal/server/mirror_pypi_test.go b/internal/server/mirror_pypi_test.go new file mode 100644 index 00000000..103e0f56 --- /dev/null +++ b/internal/server/mirror_pypi_test.go @@ -0,0 +1,140 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const ( + mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" + mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" +) + +// pypiUpstream serves the JSON API for example-pkg 1.0 with a wheel and an +// sdist, whose filenames spell the name differently, and counts file +// downloads by path. +type pypiUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func newPyPIUpstream(t *testing.T, digests map[string]string) *pypiUpstream { + t.Helper() + u := &pypiUpstream{ + files: map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"}, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/pypi/example-pkg/json" { + var files []map[string]any + for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + sum := sha256.Sum256([]byte(u.files[path])) + digest := hex.EncodeToString(sum[:]) + if d, ok := digests[path]; ok { + digest = d + } + files = append(files, map[string]any{ + "url": u.URL + path, + "digests": map[string]string{"sha256": digest}, + "size": len(u.files[path]), + }) + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "info": map[string]any{"name": "example-pkg"}, + "releases": map[string]any{"1.0": files}, + }) + return + } + body, ok := u.files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + u.mu.Lock() + u.hits[r.URL.Path]++ + u.mu.Unlock() + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *pypiUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func pypiMirrorConfig(upstream *pypiUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.PyPI = upstream.URL + cfg.Upstream.PyPIDownload = upstream.URL + return cfg +} + +func TestMirrorPyPIFetchesEveryFile(t *testing.T) { + upstream := newPyPIUpstream(t, nil) + cfg := pypiMirrorConfig(upstream) + + progress, proxy := mirrorWithConfig(t, cfg, "pkg:pypi/example-pkg@1.0") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // pip gets both files from the cache once the upstream is down. + upstream.Close() + pypi := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.PyPI, cfg.Upstream.PyPIDownload) + for path, want := range map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"} { + rec := httptest.NewRecorder() + pypi.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorPyPIFileNameQualifier(t *testing.T) { + upstream := newPyPIUpstream(t, nil) + + progress, _ := mirrorWithConfig(t, pypiMirrorConfig(upstream), "pkg:pypi/example-pkg@1.0?file_name=example.pkg-1.0.tar.gz") + + if progress.Completed != 1 || upstream.hitsFor(mirrorSdistPath) != 1 { + t.Errorf("completed = %d, sdist hits = %d, errors = %v; want the sdist", progress.Completed, upstream.hitsFor(mirrorSdistPath), progress.Errors) + } + if upstream.hitsFor(mirrorWheelPath) != 0 { + t.Error("the wheel was mirrored although file_name names the sdist") + } + + progress, _ = mirrorWithConfig(t, pypiMirrorConfig(upstream), "pkg:pypi/example-pkg@1.0?file_name=missing-1.0.tar.gz") + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "missing-1.0.tar.gz") { + t.Errorf("failed = %d, errors = %v; want a failure naming the missing file", progress.Failed, progress.Errors) + } +} + +func TestMirrorPyPIRefusesDigestMismatch(t *testing.T) { + upstream := newPyPIUpstream(t, map[string]string{mirrorWheelPath: strings.Repeat("0", 64)}) + + progress, _ := mirrorWithConfig(t, pypiMirrorConfig(upstream), "pkg:pypi/example-pkg@1.0?file_name=example_pkg-1.0-py3-none-any.whl") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("completed = %d, failed = %d, errors = %v; want the mismatching wheel refused", progress.Completed, progress.Failed, progress.Errors) + } +}