From f44ce1e60fd94362928d5aee9bec83904d19adc9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 22 Oct 2026 15:38:35 +0200 Subject: [PATCH 1/3] List mirror versions from the configured upstream proxy mirror and mirror API jobs listed the versions of unversioned PURLs from the public registries, ignoring upstream.*. They now list them from the configured upstream through the proxy's upstream client, and npm packages from their npm_routes entry. Where the upstream has no listing API, or the ecosystem has no listing at all, the error asks for a version. Refs #524 --- cmd/proxy/main.go | 7 + docs/configuration.md | 4 + internal/handler/package_routes.go | 7 + internal/mirror/job.go | 4 +- internal/mirror/mirror.go | 4 + internal/mirror/schedule.go | 2 +- internal/mirror/source.go | 47 +++---- internal/mirror/versions.go | 52 +++++++ internal/mirror/versions_test.go | 98 +++++++++++++ internal/server/mirror_schedule.go | 3 +- internal/server/mirror_versions.go | 94 +++++++++++++ internal/server/mirror_versions_test.go | 174 ++++++++++++++++++++++++ internal/server/server.go | 3 +- 13 files changed, 462 insertions(+), 37 deletions(-) create mode 100644 internal/mirror/versions.go create mode 100644 internal/mirror/versions_test.go create mode 100644 internal/server/mirror_versions.go create mode 100644 internal/server/mirror_versions_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index e3505f29..55f48367 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -558,6 +558,13 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images + m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + switch s := source.(type) { + case *mirror.PURLSource: + s.Versions = m.Versions + case *mirror.SBOMSource: + s.Versions = m.Versions + } if cfg.Scanning.Enabled { refuseUnscannedMirroring(m) } diff --git a/docs/configuration.md b/docs/configuration.md index 6f7be456..e2e8f4fa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1403,6 +1403,10 @@ Positional arguments are treated as PURLs: proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ``` +### Unversioned PURLs + +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. + ### Container images `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. diff --git a/internal/handler/package_routes.go b/internal/handler/package_routes.go index 8af7099e..cb7763b4 100644 --- a/internal/handler/package_routes.go +++ b/internal/handler/package_routes.go @@ -72,3 +72,10 @@ func (r packageRoutes) cacheFilename(packageName, filename string) string { } return filename } + +// RoutedPackageURL returns the upstream URL a route in routes sends +// packageName to, as the npm and Composer handlers pick it. +func RoutedPackageURL(routes map[string]string, packageName string) (string, bool) { + route, ok := newPackageRoutes(routes).match(packageName) + return route.url, ok +} diff --git a/internal/mirror/job.go b/internal/mirror/job.go index 3d829667..70a2a2a4 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -201,9 +201,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Platforms: req.Platforms}, nil + return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index e9468253..4b79ce9f 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -27,6 +27,10 @@ type Mirror struct { // other unsupported ecosystem. Images ImageMirrorer + // Versions lists the versions of unversioned PURLs in mirror jobs and + // schedules. Nil lists them from the public registries. + Versions VersionLister + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 4ab0d2b2..b224ea20 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -76,7 +76,7 @@ func (s *Scheduler) loop(ctx context.Context, index int, schedule Schedule) { func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { started := time.Now() - progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms}) + progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) if ctx.Err() != nil { return } diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 8fcf11c2..3e408734 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -5,8 +5,6 @@ import ( "fmt" "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries" - _ "github.com/git-pkgs/registries/all" "github.com/git-pkgs/sbom" ) @@ -32,18 +30,21 @@ type Source interface { } // PURLSource yields packages from PURL strings. -// Versioned PURLs produce a single item. Unversioned PURLs look up all versions from the registry. +// Versioned PURLs produce a single item. Unversioned PURLs produce one item +// per version Versions lists. type PURLSource struct { - PURLs []string - RegClient *registries.Client + PURLs []string + // Versions lists the versions of unversioned PURLs. Nil lists them from + // the public registries. + Versions VersionLister // Platforms limit which platforms of container images are mirrored. Platforms []string } func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { - client := s.RegClient - if client == nil { - client = registries.DefaultClient() + versions := s.Versions + if versions == nil { + versions = RegistryVersions{} } for _, purlStr := range s.PURLs { @@ -73,11 +74,11 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } // Unversioned: enumerate all versions - versions, err := s.fetchVersions(ctx, client, ecosystem, name) + listed, err := versions.ListVersions(ctx, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } - for _, v := range versions { + for _, v := range listed { if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { return err } @@ -86,27 +87,13 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } -func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Client, ecosystem, name string) ([]string, error) { - reg, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", client) - if err != nil { - return nil, err - } - versions, err := reg.FetchVersions(ctx, name) - if err != nil { - return nil, err - } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number - } - return result, nil -} - // SBOMSource extracts package versions from CycloneDX or SPDX SBOM data. type SBOMSource struct { - Data []byte - Name string - RegClient *registries.Client + Data []byte + Name string + // Versions lists the versions of unversioned PURLs. Nil lists them from + // the public registries. + Versions VersionLister // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -120,7 +107,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, RegClient: s.RegClient, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go new file mode 100644 index 00000000..f62102b9 --- /dev/null +++ b/internal/mirror/versions.go @@ -0,0 +1,52 @@ +package mirror + +import ( + "context" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/all" +) + +// VersionLister lists the versions of a package, for PURLs without one. +type VersionLister interface { + ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) +} + +// RegistryVersions lists versions through the registries clients. +type RegistryVersions struct { + // Client makes the requests. Nil uses registries.DefaultClient. + Client *registries.Client + // URL returns the registry base URL to list a package from, by PURL + // type. Nil, or an empty URL, uses the public registry. + URL func(purlType, name string) (string, error) +} + +// ListVersions implements VersionLister. +func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + purlType := purl.EcosystemToPURLType(ecosystem) + var baseURL string + if r.URL != nil { + var err error + if baseURL, err = r.URL(purlType, name); err != nil { + return nil, err + } + } + client := r.Client + if client == nil { + client = registries.DefaultClient() + } + reg, err := registries.New(purlType, baseURL, client) + if err != nil { + return nil, err + } + versions, err := reg.FetchVersions(ctx, name) + if err != nil { + return nil, err + } + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number + } + return result, nil +} diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go new file mode 100644 index 00000000..9c04a210 --- /dev/null +++ b/internal/mirror/versions_test.go @@ -0,0 +1,98 @@ +package mirror + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" +) + +// fakeVersions lists fixed versions and records what it was asked for. +type fakeVersions struct { + versions []string + err error + asked []string +} + +func (f *fakeVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) { + f.asked = append(f.asked, ecosystem+"/"+name) + return f.versions, f.err +} + +func enumerate(t *testing.T, source Source) []PackageVersion { + t.Helper() + var items []PackageVersion + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + items = append(items, pv) + return nil + }); err != nil { + t.Fatalf("Enumerate() error = %v", err) + } + return items +} + +func TestPURLSourceListsVersionsThroughLister(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0.0", "1.1.0"}} + source := &PURLSource{PURLs: []string{"pkg:npm/%40acme/widget"}, Versions: versions} + + items := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "npm", Name: "@acme/widget", Version: "1.0.0"}, + {Ecosystem: "npm", Name: "@acme/widget", Version: "1.1.0"}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } + if !reflect.DeepEqual(versions.asked, []string{"npm/@acme/widget"}) { + t.Errorf("lister asked for %v, want npm/@acme/widget", versions.asked) + } +} + +func TestPURLSourceReportsListerError(t *testing.T) { + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{err: errors.New("upstream down")}} + err := source.Enumerate(context.Background(), func(PackageVersion) error { return nil }) + if err == nil { + t.Fatal("Enumerate() succeeded although listing versions failed") + } +} + +func TestSBOMSourceListsVersionsThroughLister(t *testing.T) { + bom := map[string]any{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, + } + versions := &fakeVersions{versions: []string{"1.3.0"}} + source := &SBOMSource{Data: marshalJSON(t, bom), Versions: versions} + + items := enumerate(t, source) + + if len(items) != 1 || items[0].Version != "1.3.0" { + t.Errorf("items = %v, want left-pad@1.3.0 from the lister", items) + } +} + +func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { + m := setupTestMirror(t, 1) + versions := &fakeVersions{} + m.Versions = versions + js := NewJobStore(context.Background(), m) + + source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if got := source.(*PURLSource).Versions; got != versions { + t.Errorf("PURL source lists versions through %v, want the mirror's lister", got) + } + + source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if got := source.(*SBOMSource).Versions; got != versions { + t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) + } +} diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index 3f69e7fa..3f0b77eb 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -17,8 +17,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) return } // One worker, so a schedule never fans out against the registry. - m := mirror.New(proxy, s.db, s.storage, s.logger, 1) - m.Images = s.newContainerHandler(proxy) + m := s.newMirror(proxy, 1) schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules)) for i, schedule := range s.cfg.Mirror.Schedules { schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go new file mode 100644 index 00000000..9b981fdc --- /dev/null +++ b/internal/server/mirror_versions.go @@ -0,0 +1,94 @@ +package server + +import ( + "cmp" + "fmt" + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +// NewMirrorVersions lists versions for proxy mirror from the configured +// upstreams, through client, so upstream.auth and the private address rules +// apply as they do when serving. +func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirror.VersionLister { //nolint:ireturn // the mirror takes any lister + return mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(client)), + URL: func(purlType, name string) (string, error) { + return mirrorVersionsURL(upstream, purlType, name) + }, + } +} + +// newMirror builds the mirror for mirror API jobs and schedules, which fetch +// and list versions from the configured upstreams. +func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { + m := mirror.New(proxy, s.db, s.storage, s.logger, workers) + m.Images = s.newContainerHandler(proxy) + m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) + return m +} + +// mirrorVersionsURL returns the registry base URL to list a package's +// versions from: the upstream the proxy serves it from, where that upstream +// has the API the registries client reads. An empty URL means the public +// registry, used only while the upstream is the public one too. +func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, error) { + d := config.Default().Upstream + switch purlType { + case "npm": + if routed, ok := handler.RoutedPackageURL(u.NPMRoutes, name); ok { + return routed, nil + } + return upstreamOr(u.NPM, d.NPM), nil + case "gem": + return upstreamOr(u.Gem, d.Gem), nil + case "golang": + return upstreamOr(u.Go, d.Go), nil + case "hex": + return upstreamOr(u.HexAPI, d.HexAPI), nil + case "pub": + return upstreamOr(u.Pub, d.Pub), nil + case "pypi": + return upstreamOr(u.PyPI, d.PyPI), nil + case "maven": + return upstreamOr(u.Maven, d.Maven), nil + case "nuget": + return upstreamOr(u.NuGet, d.NuGet) + "/v3", nil + case "composer": + if _, ok := handler.RoutedPackageURL(u.ComposerRoutes, name); ok { + return "", fmt.Errorf("listing versions of packages in upstream.composer_routes isn't supported; give a version") + } + return upstreamOr(u.Composer, d.Composer), nil + case "cran": + return upstreamOr(u.CRAN, d.CRAN), nil + case "brew": + return strings.TrimSuffix(upstreamOr(u.HomebrewAPI, d.HomebrewAPI), "/api"), nil + case "cargo": + return publicOnly("upstream.cargo", u.Cargo, d.Cargo) + case "conda": + return publicOnly("upstream.conda", u.Conda, d.Conda) + case "julia": + return publicOnly("upstream.julia", u.Julia, d.Julia) + default: + return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) + } +} + +func upstreamOr(configured, fallback string) string { + return strings.TrimSuffix(cmp.Or(configured, fallback), "/") +} + +// publicOnly is for upstreams that don't serve the API the registries +// client reads: versions come from the public registry while the upstream +// is the public one, and can't be listed otherwise. +func publicOnly(setting, configured, fallback string) (string, error) { + if upstreamOr(configured, fallback) != strings.TrimSuffix(fallback, "/") { + return "", fmt.Errorf("listing versions from a custom %s isn't supported; give a version", setting) + } + return "", nil +} diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go new file mode 100644 index 00000000..898c2128 --- /dev/null +++ b/internal/server/mirror_versions_test.go @@ -0,0 +1,174 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +// newPackumentUpstream serves an npm packument with the given versions for +// path and counts the requests it gets. +func newPackumentUpstream(t *testing.T, path string, versions ...string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + entries := make([]string, len(versions)) + for i, v := range versions { + entries[i] = `"` + v + `":{"version":"` + v + `"}` + } + body := `{"name":"x","versions":{` + strings.Join(entries, ",") + `}}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.URL.EscapedPath(), path) { + http.NotFound(w, r) + return + } + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + return upstream, &hits +} + +func listMirrorVersions(t *testing.T, cfg *config.Config, ecosystem, name string) ([]string, error) { + t.Helper() + cfg.Upstream.AllowLoopback = true + proxy, _ := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + versions, err := NewMirrorVersions(cfg.Upstream, proxy.HTTPClient).ListVersions(context.Background(), ecosystem, name) + slices.Sort(versions) + return versions, err +} + +func TestMirrorVersionsComeFromConfiguredUpstream(t *testing.T) { + upstream, hits := newPackumentUpstream(t, "/left-pad", "1.0.0", "1.1.0") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + + versions, err := listMirrorVersions(t, cfg, "npm", "left-pad") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0", "1.1.0"}) { + t.Errorf("versions = %v, want the configured upstream's 1.0.0 and 1.1.0", versions) + } + if hits.Load() == 0 { + t.Error("the configured npm upstream was never asked") + } +} + +func TestMirrorVersionsFollowNPMRoutes(t *testing.T) { + public, publicHits := newPackumentUpstream(t, "/@acme%2fwidget", "9.9.9") + private, _ := newPackumentUpstream(t, "/@acme%2fwidget", "1.0.0") + cfg := config.Default() + cfg.Upstream.NPM = public.URL + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": private.URL} + + versions, err := listMirrorVersions(t, cfg, "npm", "@acme/widget") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0"}) { + t.Errorf("versions = %v, want only the routed registry's 1.0.0", versions) + } + if publicHits.Load() != 0 { + t.Error("a routed package was listed from upstream.npm") + } +} + +func TestMirrorVersionsURL(t *testing.T) { + custom := config.Default().Upstream + custom.NPM = "https://npm.example.com/" + custom.Gem = "https://gems.example.com" + custom.Go = "https://goproxy.example.com" + custom.HexAPI = "https://hex.example.com" + custom.Pub = "https://pub.example.com" + custom.PyPI = "https://pypi.example.com" + custom.Maven = "https://maven.example.com/maven2" + custom.NuGet = "https://nuget.example.com" + custom.Composer = "https://packagist.example.com" + custom.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} + custom.CRAN = "https://cran.example.com" + custom.HomebrewAPI = "https://brew.example.com/api" + custom.Cargo = "https://index.example.com" + custom.Conda = "https://conda.example.com" + custom.Julia = "https://julia.example.com" + + tests := []struct { + name string + upstream config.UpstreamConfig + ecosystem string + pkg string + want string + wantErr string + }{ + {"npm default", config.Default().Upstream, "npm", "left-pad", "https://registry.npmjs.org", ""}, + {"npm unset", config.UpstreamConfig{}, "npm", "left-pad", "https://registry.npmjs.org", ""}, + {"npm", custom, "npm", "left-pad", "https://npm.example.com", ""}, + {"gem", custom, "gem", "rails", "https://gems.example.com", ""}, + {"golang", custom, "golang", "golang.org/x/mod", "https://goproxy.example.com", ""}, + {"hex uses the API URL", custom, "hex", "jason", "https://hex.example.com", ""}, + {"pub", custom, "pub", "http", "https://pub.example.com", ""}, + {"pypi", custom, "pypi", "requests", "https://pypi.example.com", ""}, + {"maven", custom, "maven", "org.example:lib", "https://maven.example.com/maven2", ""}, + {"nuget lists through v3", custom, "nuget", "Newtonsoft.Json", "https://nuget.example.com/v3", ""}, + {"composer", custom, "composer", "monolog/monolog", "https://packagist.example.com", ""}, + {"composer route", custom, "composer", "acme/widget", "", "upstream.composer_routes"}, + {"cran", custom, "cran", "ggplot2", "https://cran.example.com", ""}, + {"brew", custom, "brew", "wget", "https://brew.example.com", ""}, + {"cargo default lists from crates.io", config.Default().Upstream, "cargo", "serde", "", ""}, + {"cargo index", custom, "cargo", "serde", "", "upstream.cargo"}, + {"conda default", config.Default().Upstream, "conda", "numpy", "", ""}, + {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, + {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, + {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, + {"helm", custom, "helm", "nginx", "", "give a version"}, + {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, + {"apk", custom, "apk", "curl", "", "give a version"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mirrorVersionsURL(tt.upstream, tt.ecosystem, tt.pkg) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %v, want one mentioning %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("error = %v", err) + } + if got != tt.want { + t.Errorf("URL = %q, want %q", got, tt.want) + } + }) + } +} + +func TestServerMirrorListsFromConfiguredUpstream(t *testing.T) { + upstream, hits := newPackumentUpstream(t, "/left-pad", "1.0.0") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + cfg.Upstream.AllowLoopback = true + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, nil, nil, logger) + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + + m := s.newMirror(proxy, 1) + if m.Versions == nil { + t.Fatal("the server's mirror lists versions from the public registries") + } + versions, err := m.Versions.ListVersions(context.Background(), "npm", "left-pad") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0"}) || hits.Load() == 0 { + t.Errorf("versions = %v, want 1.0.0 from the configured upstream", versions) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 8a0a683a..96d3b242 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -380,8 +380,7 @@ func (s *Server) serve(listener net.Listener) error { // Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env) if s.cfg.MirrorAPI { - mirrorSvc := mirror.New(proxy, s.db, s.storage, s.logger, 4) //nolint:mnd // default concurrency - mirrorSvc.Images = s.newContainerHandler(proxy) + mirrorSvc := s.newMirror(proxy, 4) //nolint:mnd // default concurrency jobStore := mirror.NewJobStore(bgCtx, mirrorSvc) mirrorAPI := NewMirrorAPIHandler(jobStore) r.Post("/api/mirror", mirrorAPI.HandleCreate) From c3d0448e6ffe1af15d383c31fc0761b32ba75e9b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 19:03:38 +0200 Subject: [PATCH 2/3] Bump registries for streamed npm packuments registries main streams npm packuments (git-pkgs/registries#109), so listing the versions of packages whose packument is over 10 MiB, such as typescript, no longer fails with "unexpected end of JSON input". It also reads Maven versions from maven-metadata.xml (#114). go.mod points at a pseudo-version of registries main until those are released. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c1d4e2f..586ff0c7 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.22 - github.com/git-pkgs/registries v0.11.0 + github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 diff --git a/go.sum b/go.sum index f486b50a..b5be5d02 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.22 h1:qT9FNXT/3/RbWnQCxMHn1CFeuruMsq0RBH154oLt1Xs= github.com/git-pkgs/purl v0.1.22/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= -github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 h1:sg69ptO+sEegScnAmS+iiEVOXYtz5Tb5H2AjZBPV4Vo= +github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= From 42584c384522507cd019dfc80870be1723251d72 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 25 Oct 2026 10:29:20 +0000 Subject: [PATCH 3/3] Bump registries to v0.11.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9f27556a..5cf652be 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.23 - github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 + github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 diff --git a/go.sum b/go.sum index c9470905..52bda772 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.23 h1:RG7F73S4ZnbAnB/posWf2WS4OhdG0xUbyX8UIkEUhPw= github.com/git-pkgs/purl v0.1.23/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922 h1:sg69ptO+sEegScnAmS+iiEVOXYtz5Tb5H2AjZBPV4Vo= -github.com/git-pkgs/registries v0.11.1-0.20261024160437-36fa54623922/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1 h1:AkVQ15Rnrho4tbTnocYuHQuzNQ1m9jEjWKTA8P3xnPI= +github.com/git-pkgs/registries v0.11.1/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI=