From 16ae3deb1c491dc91dd8cb25f6555fb343692516 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 14:25:35 +0100 Subject: [PATCH] Check mirrored files against their own SHA-256 Mirroring a PyPI version passed each file's sha256 as the integrity GetOrFetchArtifactFromURLWithIntegrity records with the version. Wheels of one version share the name and version they are cached under, so the last file's checksum replaced the others', and serving the other wheels from the cache failed the integrity check. The mirror now passes the sha256 as a digest, which is checked against that file's content hash alone. --- internal/mirror/mirror.go | 15 ++++++++++++++- internal/server/mirror_pypi_test.go | 21 +++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ddf97364..24d98ff4 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "sync" "sync/atomic" "time" @@ -284,7 +285,7 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr if name == "" { name, version = pv.Name, pv.Version } - result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity) + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, fileDigest(file.Integrity)) if err != nil { fail(fmt.Errorf("%s: %w", file.Filename, err)) return @@ -315,6 +316,18 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr } } +// fileDigest returns the SHA-256 of one file as a digest, which is checked +// against that file alone. The integrity GetOrFetchArtifactFromURLWithIntegrity +// takes is kept with the version instead, and the files of a version that +// share a name and version would overwrite each other's. Other checksums are +// not checked, as the download handlers don't check them either. +func fileDigest(integrity string) string { + if sum, ok := strings.CutPrefix(integrity, "sha256-"); ok { + return "sha256:" + sum + } + return "" +} + // fileLister lists files from where Versions lists versions, so both come // from the configured upstream. func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do diff --git a/internal/server/mirror_pypi_test.go b/internal/server/mirror_pypi_test.go index 103e0f56..a5919579 100644 --- a/internal/server/mirror_pypi_test.go +++ b/internal/server/mirror_pypi_test.go @@ -16,13 +16,14 @@ import ( ) const ( - mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" - mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" + mirrorWheelPath = "/packages/ab/cd/example_pkg-1.0-py3-none-any.whl" + mirrorLinuxWheelPath = "/packages/23/45/example_pkg-1.0-cp312-cp312-manylinux_2_17_x86_64.whl" + mirrorSdistPath = "/packages/ef/01/example.pkg-1.0.tar.gz" ) -// pypiUpstream serves the JSON API for example-pkg 1.0 with a wheel and an -// sdist, whose filenames spell the name differently, and counts file -// downloads by path. +// pypiUpstream serves the JSON API for example-pkg 1.0 with two wheels, which +// are cached under the same name and version, and an sdist, whose filename +// spells the name differently. It counts file downloads by path. type pypiUpstream struct { *httptest.Server mu sync.Mutex @@ -33,13 +34,13 @@ type pypiUpstream struct { func newPyPIUpstream(t *testing.T, digests map[string]string) *pypiUpstream { t.Helper() u := &pypiUpstream{ - files: map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"}, + files: map[string]string{mirrorWheelPath: "wheel", mirrorLinuxWheelPath: "linux wheel", mirrorSdistPath: "sdist"}, hits: map[string]int{}, } u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/pypi/example-pkg/json" { var files []map[string]any - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for _, path := range []string{mirrorWheelPath, mirrorLinuxWheelPath, mirrorSdistPath} { sum := sha256.Sum256([]byte(u.files[path])) digest := hex.EncodeToString(sum[:]) if d, ok := digests[path]; ok { @@ -93,16 +94,16 @@ func TestMirrorPyPIFetchesEveryFile(t *testing.T) { if progress.Completed != 1 || progress.Failed != 0 { t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) } - for _, path := range []string{mirrorWheelPath, mirrorSdistPath} { + for path := range upstream.files { if got := upstream.hitsFor(path); got != 1 { t.Errorf("%s fetched %d times, want 1", path, got) } } - // pip gets both files from the cache once the upstream is down. + // pip gets every file from the cache once the upstream is down. upstream.Close() pypi := handler.NewPyPIHandlerWithUpstreams(proxy, "http://proxy.test", cfg.Upstream.PyPI, cfg.Upstream.PyPIDownload) - for path, want := range map[string]string{mirrorWheelPath: "wheel", mirrorSdistPath: "sdist"} { + for path, want := range upstream.files { rec := httptest.NewRecorder() pypi.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) if rec.Code != http.StatusOK || rec.Body.String() != want {