diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index e3505f29..55f48367 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -558,6 +558,13 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images + m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + switch s := source.(type) { + case *mirror.PURLSource: + s.Versions = m.Versions + case *mirror.SBOMSource: + s.Versions = m.Versions + } if cfg.Scanning.Enabled { refuseUnscannedMirroring(m) } diff --git a/docs/configuration.md b/docs/configuration.md index 49156494..586d409e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1522,6 +1522,10 @@ Positional arguments are treated as PURLs: proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ``` +### Unversioned PURLs + +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. + ### Container images `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. diff --git a/go.mod b/go.mod index dfff6d7d..94fbe6c3 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 github.com/git-pkgs/purl v0.1.24 - github.com/git-pkgs/registries v0.11.0 + github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 22594491..28c811f5 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs= -github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/registries v0.11.1 h1:AkVQ15Rnrho4tbTnocYuHQuzNQ1m9jEjWKTA8P3xnPI= +github.com/git-pkgs/registries v0.11.1/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= diff --git a/internal/mirror/job.go b/internal/mirror/job.go index 202cf1e8..acf0f251 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -209,9 +209,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Platforms: req.Platforms}, nil + return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index e9468253..4b79ce9f 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -27,6 +27,10 @@ type Mirror struct { // other unsupported ecosystem. Images ImageMirrorer + // Versions lists the versions of unversioned PURLs in mirror jobs and + // schedules. Nil lists them from the public registries. + Versions VersionLister + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string diff --git a/internal/mirror/sbom_version_test.go b/internal/mirror/sbom_version_test.go index 190cc343..43dde047 100644 --- a/internal/mirror/sbom_version_test.go +++ b/internal/mirror/sbom_version_test.go @@ -40,7 +40,7 @@ func TestSBOMSourceVersionPrecedence(t *testing.T) { t.Run(test.name, func(t *testing.T) { data := marshalJSON(t, map[string]any{"components": []map[string]any{{"name": "app", "version": test.componentVersion, "purl": test.imagePURL}}}) var got []PackageVersion - source := &SBOMSource{Data: data, RegClient: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))} + source := &SBOMSource{Data: data, Versions: RegistryVersions{Client: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))}} if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil { t.Fatal(err) } diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 89bb5121..1df9a750 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -102,7 +102,7 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { } } started := time.Now() - progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms}) + progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) if ctx.Err() != nil { if s.Claim != nil && s.Release != nil { if err := s.Release(key); err != nil { diff --git a/internal/mirror/source.go b/internal/mirror/source.go index c4792892..a88a3a8f 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -5,8 +5,6 @@ import ( "fmt" "github.com/git-pkgs/purl" - "github.com/git-pkgs/registries" - _ "github.com/git-pkgs/registries/all" "github.com/git-pkgs/sbom" ) @@ -44,18 +42,21 @@ type Source interface { } // PURLSource yields packages from PURL strings. -// Versioned PURLs produce a single item. Unversioned PURLs look up all versions from the registry. +// Versioned PURLs produce a single item. Unversioned PURLs produce one item +// per version Versions lists. type PURLSource struct { - PURLs []string - RegClient *registries.Client + PURLs []string + // Versions lists the versions of unversioned PURLs. Nil lists them from + // the public registries. + Versions VersionLister // Platforms limit which platforms of container images are mirrored. Platforms []string } func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error { - client := s.RegClient - if client == nil { - client = registries.DefaultClient() + versions := s.Versions + if versions == nil { + versions = RegistryVersions{} } for _, purlStr := range s.PURLs { @@ -85,11 +86,11 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } // Unversioned: enumerate all versions - versions, err := s.fetchVersions(ctx, client, ecosystem, name) + listed, err := versions.ListVersions(ctx, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } - for _, v := range versions { + for _, v := range listed { if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil { return err } @@ -98,27 +99,13 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } -func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Client, ecosystem, name string) ([]string, error) { - reg, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", client) - if err != nil { - return nil, err - } - versions, err := reg.FetchVersions(ctx, name) - if err != nil { - return nil, err - } - result := make([]string, len(versions)) - for i, v := range versions { - result[i] = v.Number - } - return result, nil -} - // SBOMSource extracts package versions from CycloneDX or SPDX SBOM data. type SBOMSource struct { - Data []byte - Name string - RegClient *registries.Client + Data []byte + Name string + // Versions lists the versions of unversioned PURLs. Nil lists them from + // the public registries. + Versions VersionLister // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -132,7 +119,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, RegClient: s.RegClient, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go new file mode 100644 index 00000000..f62102b9 --- /dev/null +++ b/internal/mirror/versions.go @@ -0,0 +1,52 @@ +package mirror + +import ( + "context" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/all" +) + +// VersionLister lists the versions of a package, for PURLs without one. +type VersionLister interface { + ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) +} + +// RegistryVersions lists versions through the registries clients. +type RegistryVersions struct { + // Client makes the requests. Nil uses registries.DefaultClient. + Client *registries.Client + // URL returns the registry base URL to list a package from, by PURL + // type. Nil, or an empty URL, uses the public registry. + URL func(purlType, name string) (string, error) +} + +// ListVersions implements VersionLister. +func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + purlType := purl.EcosystemToPURLType(ecosystem) + var baseURL string + if r.URL != nil { + var err error + if baseURL, err = r.URL(purlType, name); err != nil { + return nil, err + } + } + client := r.Client + if client == nil { + client = registries.DefaultClient() + } + reg, err := registries.New(purlType, baseURL, client) + if err != nil { + return nil, err + } + versions, err := reg.FetchVersions(ctx, name) + if err != nil { + return nil, err + } + result := make([]string, len(versions)) + for i, v := range versions { + result[i] = v.Number + } + return result, nil +} diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go new file mode 100644 index 00000000..9c04a210 --- /dev/null +++ b/internal/mirror/versions_test.go @@ -0,0 +1,98 @@ +package mirror + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" +) + +// fakeVersions lists fixed versions and records what it was asked for. +type fakeVersions struct { + versions []string + err error + asked []string +} + +func (f *fakeVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) { + f.asked = append(f.asked, ecosystem+"/"+name) + return f.versions, f.err +} + +func enumerate(t *testing.T, source Source) []PackageVersion { + t.Helper() + var items []PackageVersion + if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { + items = append(items, pv) + return nil + }); err != nil { + t.Fatalf("Enumerate() error = %v", err) + } + return items +} + +func TestPURLSourceListsVersionsThroughLister(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0.0", "1.1.0"}} + source := &PURLSource{PURLs: []string{"pkg:npm/%40acme/widget"}, Versions: versions} + + items := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "npm", Name: "@acme/widget", Version: "1.0.0"}, + {Ecosystem: "npm", Name: "@acme/widget", Version: "1.1.0"}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } + if !reflect.DeepEqual(versions.asked, []string{"npm/@acme/widget"}) { + t.Errorf("lister asked for %v, want npm/@acme/widget", versions.asked) + } +} + +func TestPURLSourceReportsListerError(t *testing.T) { + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{err: errors.New("upstream down")}} + err := source.Enumerate(context.Background(), func(PackageVersion) error { return nil }) + if err == nil { + t.Fatal("Enumerate() succeeded although listing versions failed") + } +} + +func TestSBOMSourceListsVersionsThroughLister(t *testing.T) { + bom := map[string]any{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, + } + versions := &fakeVersions{versions: []string{"1.3.0"}} + source := &SBOMSource{Data: marshalJSON(t, bom), Versions: versions} + + items := enumerate(t, source) + + if len(items) != 1 || items[0].Version != "1.3.0" { + t.Errorf("items = %v, want left-pad@1.3.0 from the lister", items) + } +} + +func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { + m := setupTestMirror(t, 1) + versions := &fakeVersions{} + m.Versions = versions + js := NewJobStore(context.Background(), m) + + source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if got := source.(*PURLSource).Versions; got != versions { + t.Errorf("PURL source lists versions through %v, want the mirror's lister", got) + } + + source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if got := source.(*SBOMSource).Versions; got != versions { + t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) + } +} diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index 3b023375..172c2de2 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -17,8 +17,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) return } // One worker, so a schedule never fans out against the registry. - m := mirror.New(proxy, s.db, s.storage, s.logger, 1) - m.Images = s.newContainerHandler(proxy) + m := s.newMirror(proxy, 1) schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules)) for i, schedule := range s.cfg.Mirror.Schedules { schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go new file mode 100644 index 00000000..9b981fdc --- /dev/null +++ b/internal/server/mirror_versions.go @@ -0,0 +1,94 @@ +package server + +import ( + "cmp" + "fmt" + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +// NewMirrorVersions lists versions for proxy mirror from the configured +// upstreams, through client, so upstream.auth and the private address rules +// apply as they do when serving. +func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirror.VersionLister { //nolint:ireturn // the mirror takes any lister + return mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(client)), + URL: func(purlType, name string) (string, error) { + return mirrorVersionsURL(upstream, purlType, name) + }, + } +} + +// newMirror builds the mirror for mirror API jobs and schedules, which fetch +// and list versions from the configured upstreams. +func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { + m := mirror.New(proxy, s.db, s.storage, s.logger, workers) + m.Images = s.newContainerHandler(proxy) + m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) + return m +} + +// mirrorVersionsURL returns the registry base URL to list a package's +// versions from: the upstream the proxy serves it from, where that upstream +// has the API the registries client reads. An empty URL means the public +// registry, used only while the upstream is the public one too. +func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, error) { + d := config.Default().Upstream + switch purlType { + case "npm": + if routed, ok := handler.RoutedPackageURL(u.NPMRoutes, name); ok { + return routed, nil + } + return upstreamOr(u.NPM, d.NPM), nil + case "gem": + return upstreamOr(u.Gem, d.Gem), nil + case "golang": + return upstreamOr(u.Go, d.Go), nil + case "hex": + return upstreamOr(u.HexAPI, d.HexAPI), nil + case "pub": + return upstreamOr(u.Pub, d.Pub), nil + case "pypi": + return upstreamOr(u.PyPI, d.PyPI), nil + case "maven": + return upstreamOr(u.Maven, d.Maven), nil + case "nuget": + return upstreamOr(u.NuGet, d.NuGet) + "/v3", nil + case "composer": + if _, ok := handler.RoutedPackageURL(u.ComposerRoutes, name); ok { + return "", fmt.Errorf("listing versions of packages in upstream.composer_routes isn't supported; give a version") + } + return upstreamOr(u.Composer, d.Composer), nil + case "cran": + return upstreamOr(u.CRAN, d.CRAN), nil + case "brew": + return strings.TrimSuffix(upstreamOr(u.HomebrewAPI, d.HomebrewAPI), "/api"), nil + case "cargo": + return publicOnly("upstream.cargo", u.Cargo, d.Cargo) + case "conda": + return publicOnly("upstream.conda", u.Conda, d.Conda) + case "julia": + return publicOnly("upstream.julia", u.Julia, d.Julia) + default: + return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) + } +} + +func upstreamOr(configured, fallback string) string { + return strings.TrimSuffix(cmp.Or(configured, fallback), "/") +} + +// publicOnly is for upstreams that don't serve the API the registries +// client reads: versions come from the public registry while the upstream +// is the public one, and can't be listed otherwise. +func publicOnly(setting, configured, fallback string) (string, error) { + if upstreamOr(configured, fallback) != strings.TrimSuffix(fallback, "/") { + return "", fmt.Errorf("listing versions from a custom %s isn't supported; give a version", setting) + } + return "", nil +} diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go new file mode 100644 index 00000000..898c2128 --- /dev/null +++ b/internal/server/mirror_versions_test.go @@ -0,0 +1,174 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +// newPackumentUpstream serves an npm packument with the given versions for +// path and counts the requests it gets. +func newPackumentUpstream(t *testing.T, path string, versions ...string) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + entries := make([]string, len(versions)) + for i, v := range versions { + entries[i] = `"` + v + `":{"version":"` + v + `"}` + } + body := `{"name":"x","versions":{` + strings.Join(entries, ",") + `}}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.URL.EscapedPath(), path) { + http.NotFound(w, r) + return + } + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + return upstream, &hits +} + +func listMirrorVersions(t *testing.T, cfg *config.Config, ecosystem, name string) ([]string, error) { + t.Helper() + cfg.Upstream.AllowLoopback = true + proxy, _ := NewMirrorProxy(cfg, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + versions, err := NewMirrorVersions(cfg.Upstream, proxy.HTTPClient).ListVersions(context.Background(), ecosystem, name) + slices.Sort(versions) + return versions, err +} + +func TestMirrorVersionsComeFromConfiguredUpstream(t *testing.T) { + upstream, hits := newPackumentUpstream(t, "/left-pad", "1.0.0", "1.1.0") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + + versions, err := listMirrorVersions(t, cfg, "npm", "left-pad") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0", "1.1.0"}) { + t.Errorf("versions = %v, want the configured upstream's 1.0.0 and 1.1.0", versions) + } + if hits.Load() == 0 { + t.Error("the configured npm upstream was never asked") + } +} + +func TestMirrorVersionsFollowNPMRoutes(t *testing.T) { + public, publicHits := newPackumentUpstream(t, "/@acme%2fwidget", "9.9.9") + private, _ := newPackumentUpstream(t, "/@acme%2fwidget", "1.0.0") + cfg := config.Default() + cfg.Upstream.NPM = public.URL + cfg.Upstream.NPMRoutes = map[string]string{"@acme/*": private.URL} + + versions, err := listMirrorVersions(t, cfg, "npm", "@acme/widget") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0"}) { + t.Errorf("versions = %v, want only the routed registry's 1.0.0", versions) + } + if publicHits.Load() != 0 { + t.Error("a routed package was listed from upstream.npm") + } +} + +func TestMirrorVersionsURL(t *testing.T) { + custom := config.Default().Upstream + custom.NPM = "https://npm.example.com/" + custom.Gem = "https://gems.example.com" + custom.Go = "https://goproxy.example.com" + custom.HexAPI = "https://hex.example.com" + custom.Pub = "https://pub.example.com" + custom.PyPI = "https://pypi.example.com" + custom.Maven = "https://maven.example.com/maven2" + custom.NuGet = "https://nuget.example.com" + custom.Composer = "https://packagist.example.com" + custom.ComposerRoutes = map[string]string{"acme/*": "https://composer.example.com"} + custom.CRAN = "https://cran.example.com" + custom.HomebrewAPI = "https://brew.example.com/api" + custom.Cargo = "https://index.example.com" + custom.Conda = "https://conda.example.com" + custom.Julia = "https://julia.example.com" + + tests := []struct { + name string + upstream config.UpstreamConfig + ecosystem string + pkg string + want string + wantErr string + }{ + {"npm default", config.Default().Upstream, "npm", "left-pad", "https://registry.npmjs.org", ""}, + {"npm unset", config.UpstreamConfig{}, "npm", "left-pad", "https://registry.npmjs.org", ""}, + {"npm", custom, "npm", "left-pad", "https://npm.example.com", ""}, + {"gem", custom, "gem", "rails", "https://gems.example.com", ""}, + {"golang", custom, "golang", "golang.org/x/mod", "https://goproxy.example.com", ""}, + {"hex uses the API URL", custom, "hex", "jason", "https://hex.example.com", ""}, + {"pub", custom, "pub", "http", "https://pub.example.com", ""}, + {"pypi", custom, "pypi", "requests", "https://pypi.example.com", ""}, + {"maven", custom, "maven", "org.example:lib", "https://maven.example.com/maven2", ""}, + {"nuget lists through v3", custom, "nuget", "Newtonsoft.Json", "https://nuget.example.com/v3", ""}, + {"composer", custom, "composer", "monolog/monolog", "https://packagist.example.com", ""}, + {"composer route", custom, "composer", "acme/widget", "", "upstream.composer_routes"}, + {"cran", custom, "cran", "ggplot2", "https://cran.example.com", ""}, + {"brew", custom, "brew", "wget", "https://brew.example.com", ""}, + {"cargo default lists from crates.io", config.Default().Upstream, "cargo", "serde", "", ""}, + {"cargo index", custom, "cargo", "serde", "", "upstream.cargo"}, + {"conda default", config.Default().Upstream, "conda", "numpy", "", ""}, + {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, + {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, + {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, + {"helm", custom, "helm", "nginx", "", "give a version"}, + {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, + {"apk", custom, "apk", "curl", "", "give a version"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mirrorVersionsURL(tt.upstream, tt.ecosystem, tt.pkg) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %v, want one mentioning %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("error = %v", err) + } + if got != tt.want { + t.Errorf("URL = %q, want %q", got, tt.want) + } + }) + } +} + +func TestServerMirrorListsFromConfiguredUpstream(t *testing.T) { + upstream, hits := newPackumentUpstream(t, "/left-pad", "1.0.0") + cfg := config.Default() + cfg.Upstream.NPM = upstream.URL + cfg.Upstream.AllowLoopback = true + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, nil, nil, logger) + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + + m := s.newMirror(proxy, 1) + if m.Versions == nil { + t.Fatal("the server's mirror lists versions from the public registries") + } + versions, err := m.Versions.ListVersions(context.Background(), "npm", "left-pad") + if err != nil { + t.Fatalf("ListVersions() error = %v", err) + } + if !slices.Equal(versions, []string{"1.0.0"}) || hits.Load() == 0 { + t.Errorf("versions = %v, want 1.0.0 from the configured upstream", versions) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index bd2f287c..3dc2f9be 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -387,8 +387,7 @@ func (s *Server) serve(listener net.Listener) error { // Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env) if s.cfg.MirrorAPI { - mirrorSvc := mirror.New(proxy, s.db, s.storage, s.logger, 4) //nolint:mnd // default concurrency - mirrorSvc.Images = s.newContainerHandler(proxy) + mirrorSvc := s.newMirror(proxy, 4) //nolint:mnd // default concurrency jobStore := mirror.NewJobStore(bgCtx, mirrorSvc) mirrorAPI := NewMirrorAPIHandler(jobStore) r.Post("/api/mirror", mirrorAPI.HandleCreate)