diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 6832dd33..b19f6893 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -619,7 +619,7 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler. m := mirror.New(proxy, db, store, logger, workers) m.Images = images m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) - m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient) if cfg.Scanning.Enabled { refuseUnscannedMirroring(m) } diff --git a/docs/configuration.md b/docs/configuration.md index 84b869bd..f10e4b23 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1569,6 +1569,8 @@ A Composer release is mirrored with the dist archive `upstream.composer` lists f A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images. +A Julia package is mirrored with the source tarball `upstream.julia` serves at `/package/{uuid}/{tree-hash}`, cached under the package's name and tree hash as `/julia/package/...` serves it. The PURL needs the `uuid` qualifier, for example `pkg:julia/Example@0.5.5?uuid=7876af07-990d-54b4-ab0e-23690620f79a`. Only packages of the General registry can be mirrored: the uuid and tree hash are read from General on GitHub, also with another `upstream.julia`, since they are the same on every Pkg server serving General. The tarball isn't checked by the proxy, as when serving; Pkg checks the tree hash of what it unpacks, except on Windows. Dependencies, the artifacts in a package's `Artifacts.toml` and the registry aren't mirrored. An offline `Pkg.add` also needs the registry: one installed on the client, or the one the proxy cached when a client last fetched it, which it serves offline with `cache_metadata` on. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | @@ -1589,7 +1591,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ### Unversioned PURLs -A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. ### Container images diff --git a/internal/database/cached_names.go b/internal/database/cached_names.go new file mode 100644 index 00000000..0b9563b7 --- /dev/null +++ b/internal/database/cached_names.go @@ -0,0 +1,21 @@ +package database + +// CachedArtifactNames returns the names of the packages of ecosystem that +// have a cached artifact called filename, for protocols whose file names +// identify an artifact without its package name. +func (db *DB) CachedArtifactNames(ecosystem, filename string) ([]string, error) { + var names []string + query := db.Rebind(` + SELECT DISTINCT packages.name + FROM artifacts + JOIN versions ON versions.purl = artifacts.version_purl + JOIN packages ON packages.purl = versions.package_purl + WHERE packages.ecosystem = ? AND artifacts.filename = ? + AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL + ORDER BY packages.name + `) + if err := db.Select(&names, query, ecosystem, filename); err != nil { + return nil, err + } + return names, nil +} diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 213ae220..bd9ad289 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -394,3 +394,36 @@ func TestGetRetentionCandidatesTimezone(t *testing.T) { } }) } + +func TestArtifactFilenameIndex(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_artifacts_filename'` + if db.Dialect() == DialectPostgres { + indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_artifacts_filename'` + } + hasIndex := func() bool { + t.Helper() + var count int + if err := db.Get(&count, indexQuery); err != nil { + t.Fatalf("querying indexes: %v", err) + } + return count == 1 + } + + if !hasIndex() { + t.Error("fresh schema has no idx_artifacts_filename") + } + if _, err := db.Exec(`DROP INDEX idx_artifacts_filename`); err != nil { + t.Fatalf("dropping index: %v", err) + } + if err := migrateAddArtifactFilenameIndex(db); err != nil { + t.Fatalf("migration: %v", err) + } + if err := migrateAddArtifactFilenameIndex(db); err != nil { + t.Fatalf("migration is not idempotent: %v", err) + } + if !hasIndex() { + t.Error("migration did not create idx_artifacts_filename") + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 46a21eca..f0ec6516 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -222,6 +223,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -323,6 +325,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -347,6 +350,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -463,6 +467,7 @@ var migrations = []migration{ {"016_add_fetch_leases", migrateAddFetchLeases}, {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, + {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, } // isTableNotFound returns true if the error indicates a missing table. @@ -835,6 +840,16 @@ func migrateAddArtifactContentHashIndex(db *DB) error { return nil } +// migrateAddArtifactFilenameIndex indexes artifacts by filename, for +// lookups that don't know the version, such as CachedArtifactNames. +func migrateAddArtifactFilenameIndex(db *DB) error { + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename)`) + if err != nil { + return fmt.Errorf("creating artifacts filename index: %w", err) + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 67693b22..dbac3136 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -110,7 +110,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("julia registry request", "uuid", uuid, "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch registry") return @@ -137,8 +137,15 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash) + if name == uuid { + if result := h.cachedPackage(r.Context(), hash); result != nil { + ServeArtifactRequest(w, r, result) + return + } + } + upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -147,6 +154,54 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } +// cachedPackage returns the package tarball with tree hash hash from the +// cache, under whatever name it was cached as, or nil. Without the name map, +// which needs the registry from upstream, a request can't be matched to the +// name its package was cached under, but the tarball is identified by its +// tree hash alone. The registry and artifact tarballs cached under the same +// file name are skipped. +func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheResult { + if h.proxy.DB == nil { + return nil + } + filename := juliaTarball(hash) + names, err := h.proxy.DB.CachedArtifactNames("julia", filename) + if err != nil { + h.proxy.Logger.Warn("looking up cached julia package", "hash", hash, "error", err) + return nil + } + for _, name := range names { + if name == juliaRegistryName || name == juliaArtifactName { + continue + } + result, err := h.proxy.GetCachedArtifact(ctx, "julia", name, hash, filename) + if err == nil && result != nil { + return result + } + } + return nil +} + +// JuliaPackageDownload returns the URL of a package's tarball on the Pkg +// server at upstreamURL, or the default one when empty, and the file name the +// Julia handler caches it under. +func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) { + if !validJuliaUUID(uuid) { + return "", "", fmt.Errorf("invalid julia package uuid %q", uuid) + } + if !juliaHexPattern.MatchString(treeHash) { + return "", "", fmt.Errorf("invalid julia tree hash %q", treeHash) + } + base := configuredUpstreamURL(upstreamURL, juliaUpstream) + return base + "/package/" + uuid + "/" + treeHash, juliaTarball(treeHash), nil +} + +// juliaTarball is the file name a registry, package or artifact tarball is +// cached under. +func juliaTarball(hash string) string { + return hash + ".tar.gz" +} + // handleArtifact serves an immutable binary artifact tarball. Artifacts are // anonymous content-addressed blobs with no associated package name. func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { @@ -159,7 +214,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("julia artifact request", "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return diff --git a/internal/mirror/julia.go b/internal/mirror/julia.go new file mode 100644 index 00000000..613ba767 --- /dev/null +++ b/internal/mirror/julia.go @@ -0,0 +1,84 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// PackageFinder finds a package's registry entry, which for Julia gives the +// UUID the package is registered under. +type PackageFinder interface { + FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) +} + +// FindPackage implements PackageFinder. +func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) { + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, err + } + return reg.FetchPackage(ctx, name) +} + +// mirrorJulia fetches the tarball unchecked because Pkg checks the tree hash +// of what it extracts. +func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + uuid := strings.ToLower(pv.Qualifiers["uuid"]) + if uuid == "" { + fail(fmt.Errorf("julia packages need a uuid qualifier, such as pkg:julia/%s@%s?uuid=", pv.Name, pv.Version)) + return + } + + pkg, err := m.packageFinder().FindPackage(ctx, pv.Ecosystem, pv.Name) + if err != nil { + fail(err) + return + } + registered, _ := pkg.Metadata["uuid"].(string) + if !strings.EqualFold(registered, uuid) { + fail(fmt.Errorf("%s in the General registry has uuid %s, not %s; only General packages can be mirrored", pv.Name, registered, uuid)) + return + } + // The handler names packages as the registry does. + name := pkg.Name + if name == "" { + name = pv.Name + } + + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) + if err != nil { + fail(err) + return + } + treeHash, _ := v.Metadata["git-tree-sha1"].(string) + if treeHash == "" { + fail(errors.New("the registry lists no tree hash for this version")) + return + } + url, filename, err := handler.JuliaPackageDownload(m.JuliaUpstream, uuid, treeHash) + if err != nil { + fail(err) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, treeHash, filename, url) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// packageFinder finds packages where Versions lists versions, so both come +// from the same registry. +func (m *Mirror) packageFinder() PackageFinder { //nolint:ireturn // any finder will do + if finder, ok := m.Versions.(PackageFinder); ok { + return finder + } + return RegistryVersions{} +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 0a1142d9..dd879b09 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -36,6 +36,10 @@ type Mirror struct { // schedules. Nil lists them from the public registries. Versions VersionLister + // JuliaUpstream is the Pkg server Julia packages are fetched from. Empty + // uses https://pkg.julialang.org. + JuliaUpstream string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string @@ -243,6 +247,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorHelm(ctx, pv, tracker) return } + if pv.Ecosystem == "julia" { + m.mirrorJulia(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index d8778b5c..b742e002 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,7 +16,8 @@ type PackageVersion struct { Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select - // which files of the version are mirrored. + // which files of the version are mirrored. Julia uses uuid to identify + // the package. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -75,6 +76,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro name = p.Namespace + "/" + p.Name } qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) + if ecosystem == "julia" { + qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + } // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 64fb36d2..f56fba62 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -58,6 +58,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, err + } + if purl.EcosystemToPURLType(ecosystem) == "helm" { + // The namespace names the repository, which URL resolves to the + // index the chart is listed in. + _, name, _ = strings.Cut(name, "/") + } + return reg.FetchVersions(ctx, name) +} + +// registry returns the client for the registry name is listed in. +func (r RegistryVersions) registry(ecosystem, name string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -70,14 +84,5 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str if client == nil { client = registries.DefaultClient() } - reg, err := registries.New(purlType, baseURL, client) - if err != nil { - return nil, err - } - if purlType == "helm" { - // The namespace names the repository, which URL resolves to the - // index the chart is listed in. - _, name, _ = strings.Cut(name, "/") - } - return reg.FetchVersions(ctx, name) + return registries.New(purlType, baseURL, client) } diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go index 9c04a210..29f9f41a 100644 --- a/internal/mirror/versions_test.go +++ b/internal/mirror/versions_test.go @@ -96,3 +96,23 @@ func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) } } + +func TestPURLSourceKeepsJuliaUUID(t *testing.T) { + const uuid = "7876af07-990d-54b4-ab0e-23690620f79a" + versions := &fakeVersions{versions: []string{"0.5.5"}} + source := &PURLSource{ + PURLs: []string{"pkg:julia/Example@0.5.4?uuid=" + uuid, "pkg:julia/Example?uuid=" + uuid}, + Versions: versions, + } + + items := enumerate(t, source) + + qualifiers := map[string]string{"uuid": uuid} + want := []PackageVersion{ + {Ecosystem: "julia", Name: "Example", Version: "0.5.4", Qualifiers: qualifiers}, + {Ecosystem: "julia", Name: "Example", Version: "0.5.5", Qualifiers: qualifiers}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 86a5d8e0..dba27d30 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -63,6 +63,7 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m := mirror.New(proxy, db, store, logger, 1) m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) m.Versions = versions + m.JuliaUpstream = cfg.Upstream.Julia if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_julia_test.go b/internal/server/mirror_julia_test.go new file mode 100644 index 00000000..607a2dca --- /dev/null +++ b/internal/server/mirror_julia_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +const ( + juliaExampleUUID = "7876af07-990d-54b4-ab0e-23690620f79a" + juliaExampleTree = "e1f0e1a832ccd8e97d6d0348dec33ee139a5aeaf" + juliaGeneralUUID = "23338594-aafe-5451-b93e-139f81909106" + juliaRegistryTree = "465d1e55da75e226473a37c40bf2c20aee19234d" +) + +// juliaUpstream serves the General registry's files for Example, which has +// version 0.5.5, and as a Pkg server /registries, the General registry +// tarball and Example's 0.5.5 source tarball. It counts package downloads. +type juliaUpstream struct { + *httptest.Server + tarball string + packageHits atomic.Int32 + registryHits atomic.Int32 +} + +func newJuliaUpstream(t *testing.T) *juliaUpstream { + t.Helper() + u := &juliaUpstream{tarball: "Example 0.5.5 source"} + registry := juliaRegistryTarball(t, "[packages]\n"+juliaExampleUUID+` = { name = "Example", path = "E/Example" }`+"\n") + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/E/Example/Package.toml": + _, _ = io.WriteString(w, "name = \"Example\"\nuuid = \""+juliaExampleUUID+"\"\nrepo = \"https://github.com/JuliaLang/Example.jl.git\"\n") + case "/E/Example/Versions.toml": + _, _ = io.WriteString(w, "[\"0.5.4\"]\ngit-tree-sha1 = \"11820aa9c229fd3833d4bd69e5e75ef4e7273bf1\"\n\n[\"0.5.5\"]\ngit-tree-sha1 = \""+juliaExampleTree+"\"\n") + case "/registries": + _, _ = io.WriteString(w, "/registry/"+juliaGeneralUUID+"/"+juliaRegistryTree+"\n") + case "/registry/" + juliaGeneralUUID + "/" + juliaRegistryTree: + u.registryHits.Add(1) + _, _ = w.Write(registry) + case "/package/" + juliaExampleUUID + "/" + juliaExampleTree: + u.packageHits.Add(1) + _, _ = io.WriteString(w, u.tarball) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +// juliaRegistryTarball returns a gzipped registry tarball holding registryToml. +func juliaRegistryTarball(t *testing.T, registryToml string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: int64(len(registryToml))}); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, registryToml); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// mirrorJulia mirrors purls from upstream as the Pkg server, with the +// registry's files read from it too. +func mirrorJulia(t *testing.T, upstream *juliaUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + cfg := config.Default() + cfg.Upstream.Julia = upstream.URL + versions := mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(upstream.Client())), + URL: func(string, string) (string, error) { return upstream.URL, nil }, + } + return mirrorWithVersions(t, cfg, versions, purls...) +} + +func getJuliaPackage(julia *handler.JuliaHandler) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + julia.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/package/"+juliaExampleUUID+"/"+juliaExampleTree, nil)) + return rec +} + +func TestMirrorJuliaPackage(t *testing.T) { + upstream := newJuliaUpstream(t) + + progress, proxy := mirrorJulia(t, upstream, "pkg:julia/Example@0.5.5?uuid="+juliaExampleUUID) + + if progress.Completed != 1 || progress.Failed != 0 || upstream.packageHits.Load() != 1 { + t.Fatalf("completed = %d, failed = %d, package hits = %d, errors = %v", progress.Completed, progress.Failed, upstream.packageHits.Load(), progress.Errors) + } + if cached, _ := proxy.GetCachedArtifact(t.Context(), "julia", "Example", juliaExampleTree, juliaExampleTree+".tar.gz"); cached == nil { + t.Fatal("the tarball isn't cached under Example and its tree hash") + } else { + _ = cached.Reader.Close() + } + + // With the registry's names loaded, the handler finds the tarball under + // the package's name without fetching it again. + julia := handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL) + if rec := getJuliaPackage(julia); rec.Code != http.StatusOK || rec.Body.String() != upstream.tarball { + t.Errorf("status = %d, body = %q, want 200 and the tarball", rec.Code, rec.Body) + } + if upstream.registryHits.Load() == 0 { + t.Error("the handler didn't load the registry's names") + } + + // Without the upstream, a new handler can't load the names, and finds + // the tarball by its tree hash. + upstream.Close() + if rec := getJuliaPackage(handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL)); rec.Code != http.StatusOK || rec.Body.String() != upstream.tarball { + t.Errorf("from cache: status = %d, body = %q, want 200 and the tarball", rec.Code, rec.Body) + } + if got := upstream.packageHits.Load(); got != 1 { + t.Errorf("package fetched %d times, want 1", got) + } +} + +func TestMirrorJuliaRefusesOtherUUID(t *testing.T) { + upstream := newJuliaUpstream(t) + + progress, _ := mirrorJulia(t, upstream, "pkg:julia/Example@0.5.5?uuid=00000000-0000-0000-0000-000000000000") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, juliaExampleUUID) { + t.Errorf("failed = %d, errors = %v; want a failure naming General's uuid", progress.Failed, progress.Errors) + } + if upstream.packageHits.Load() != 0 { + t.Error("a tarball was fetched for a package General doesn't have") + } +} + +func TestServerMirrorFetchesJuliaFromUpstream(t *testing.T) { + cfg := config.Default() + cfg.Upstream.Julia = "https://julia.example.com" + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, nil, nil, logger) + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + + if got := s.newMirror(proxy, 1).JuliaUpstream; got != cfg.Upstream.Julia { + t.Errorf("JuliaUpstream = %q, want upstream.julia", got) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index e093a985..4fca4b99 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -31,10 +31,18 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m := mirror.New(proxy, s.db, s.storage, s.logger, workers) m.Images = s.newContainerHandler(proxy) m.Charts = handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) - m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) + UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m } +// UseMirrorUpstreams points m at the configured upstreams: it lists +// versions from them, through client, and fetches Julia packages from +// upstream.julia. +func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { + m.Versions = NewMirrorVersions(upstream, client) + m.JuliaUpstream = upstream.Julia +} + // mirrorVersionsURL returns the registry base URL to list a package's // versions from: the upstream the proxy serves it from, where that upstream // has the API the registries client reads. An empty URL means the public @@ -80,7 +88,10 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, } return "", nil case "julia": - return publicOnly("upstream.julia", u.Julia, d.Julia) + // Packages are identified by uuid and tree hash, the same on every + // Pkg server serving General, so General's are used whatever the + // upstream. + return "", nil case "helm": // The namespace names the repository; see RegistryVersions. repository, _, ok := strings.Cut(name, "/") diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go index 89fbea32..f707bb7a 100644 --- a/internal/server/mirror_versions_test.go +++ b/internal/server/mirror_versions_test.go @@ -128,7 +128,7 @@ func TestMirrorVersionsURL(t *testing.T) { {"conda default", config.Default().Upstream, "conda", "numpy", "", ""}, {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, - {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, + {"julia server lists from General", custom, "julia", "JSON", "", ""}, {"helm", custom, "helm", "stable/nginx", "https://charts.example.com/stable", ""}, {"helm without repository", custom, "helm", "nginx", "", "namespace"}, {"helm unknown repository", custom, "helm", "other/nginx", "", "upstream.helm"},