From 3f83d9c0cecfc733ad05479c8adb6bdec8ccf41a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:37:46 +0100 Subject: [PATCH 1/2] Mirror Julia packages proxy mirror and mirror jobs now take pkg:julia PURLs with the uuid qualifier. The tree hash of the version comes from the General registry, after checking the uuid is the one General lists for the name, and the tarball is fetched from upstream.julia at /package/{uuid}/{tree-hash} and cached under the name and tree hash the Julia handler uses. The handler names a package from the registry's Registry.toml, which it loads from the upstream. Without the upstream it fell back to caching and looking up by uuid, so a package cached under its name, mirrored or fetched before a restart, was not served offline. It now looks the tarball up by its tree hash in that case. Julia versions are listed from General whatever upstream.julia is, as packages are identified by uuid and tree hash on every Pkg server serving General. --- cmd/proxy/main.go | 2 +- docs/configuration.md | 4 +- internal/database/cached_names.go | 21 +++ internal/handler/julia.go | 63 ++++++++- internal/mirror/julia.go | 88 +++++++++++++ internal/mirror/mirror.go | 8 ++ internal/mirror/source.go | 6 +- internal/mirror/versions.go | 15 ++- internal/mirror/versions_test.go | 20 +++ internal/server/mirror_fetch_test.go | 1 + internal/server/mirror_julia_test.go | 162 ++++++++++++++++++++++++ internal/server/mirror_versions.go | 15 ++- internal/server/mirror_versions_test.go | 2 +- 13 files changed, 393 insertions(+), 14 deletions(-) create mode 100644 internal/database/cached_names.go create mode 100644 internal/mirror/julia.go create mode 100644 internal/server/mirror_julia_test.go diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index a61534a4..7f212ea3 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -560,7 +560,7 @@ func runMirror() { m := mirror.New(proxy, db, store, logger, *concurrency) m.Images = images - m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient) switch s := source.(type) { case *mirror.PURLSource: s.Versions = m.Versions diff --git a/docs/configuration.md b/docs/configuration.md index 1049f2b4..58dde960 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1530,6 +1530,8 @@ A conda version is mirrored with every build anaconda.org lists for it, across a A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. +A Julia package is mirrored with the source tarball `upstream.julia` serves at `/package/{uuid}/{tree-hash}`, cached under the package's name and tree hash as `/julia/package/...` serves it. The PURL needs the `uuid` qualifier, for example `pkg:julia/Example@0.5.5?uuid=7876af07-990d-54b4-ab0e-23690620f79a`. Only packages of the General registry can be mirrored: the uuid and tree hash are read from General on GitHub, also with another `upstream.julia`, since they are the same on every Pkg server serving General. The tarball isn't checked by the proxy, as when serving; Pkg checks the tree hash of what it unpacks, except on Windows. Dependencies, the artifacts in a package's `Artifacts.toml` and the registry aren't mirrored. An offline `Pkg.add` also needs the registry: one installed on the client, or the one the proxy cached when a client last fetched it, which it serves offline with `cache_metadata` on. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | @@ -1550,7 +1552,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ### Unversioned PURLs -A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. ### Container images diff --git a/internal/database/cached_names.go b/internal/database/cached_names.go new file mode 100644 index 00000000..0b9563b7 --- /dev/null +++ b/internal/database/cached_names.go @@ -0,0 +1,21 @@ +package database + +// CachedArtifactNames returns the names of the packages of ecosystem that +// have a cached artifact called filename, for protocols whose file names +// identify an artifact without its package name. +func (db *DB) CachedArtifactNames(ecosystem, filename string) ([]string, error) { + var names []string + query := db.Rebind(` + SELECT DISTINCT packages.name + FROM artifacts + JOIN versions ON versions.purl = artifacts.version_purl + JOIN packages ON packages.purl = versions.package_purl + WHERE packages.ecosystem = ? AND artifacts.filename = ? + AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL + ORDER BY packages.name + `) + if err := db.Select(&names, query, ecosystem, filename); err != nil { + return nil, err + } + return names, nil +} diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 67693b22..6ec02095 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -110,7 +110,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("julia registry request", "uuid", uuid, "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch registry") return @@ -137,8 +137,15 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash) + if name == uuid { + if result := h.cachedPackage(r.Context(), hash); result != nil { + ServeArtifactRequest(w, r, result) + return + } + } + upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return @@ -147,6 +154,56 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } +// cachedPackage returns the package tarball with tree hash hash from the +// cache, under whatever name it was cached as, or nil. Without the name map, +// which needs the registry from upstream, a request can't be matched to the +// name its package was cached under, but the tarball is identified by its +// tree hash alone. The registry and artifact tarballs cached under the same +// file name are skipped. +func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheResult { + if h.proxy.DB == nil { + return nil + } + filename := juliaTarball(hash) + names, err := h.proxy.DB.CachedArtifactNames("julia", filename) + if err != nil { + h.proxy.Logger.Warn("looking up cached julia package", "hash", hash, "error", err) + return nil + } + for _, name := range names { + if name == juliaRegistryName || name == juliaArtifactName { + continue + } + result, err := h.proxy.GetCachedArtifact(ctx, "julia", name, hash, filename) + if err == nil && result != nil { + return result + } + } + return nil +} + +// JuliaPackageDownload returns the URL of a package's tarball on the Pkg +// server at upstreamURL, or the default one when empty, and the file name the +// Julia handler caches it under. The handler caches a package tarball as +// version treeHash, the git tree hash of the package's source, under the +// package's name in the General registry. +func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) { + if !validJuliaUUID(uuid) { + return "", "", fmt.Errorf("invalid julia package uuid %q", uuid) + } + if !juliaHexPattern.MatchString(treeHash) { + return "", "", fmt.Errorf("invalid julia tree hash %q", treeHash) + } + base := configuredUpstreamURL(upstreamURL, juliaUpstream) + return base + "/package/" + uuid + "/" + treeHash, juliaTarball(treeHash), nil +} + +// juliaTarball is the file name a registry, package or artifact tarball is +// cached under. +func juliaTarball(hash string) string { + return hash + ".tar.gz" +} + // handleArtifact serves an immutable binary artifact tarball. Artifacts are // anonymous content-addressed blobs with no associated package name. func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { @@ -159,7 +216,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("julia artifact request", "hash", hash) upstreamURL := h.upstreamURL + r.URL.Path - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, juliaTarball(hash), upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return diff --git a/internal/mirror/julia.go b/internal/mirror/julia.go new file mode 100644 index 00000000..30efb683 --- /dev/null +++ b/internal/mirror/julia.go @@ -0,0 +1,88 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// PackageFinder finds a package's registry entry, which for Julia gives the +// UUID the package is registered under. +type PackageFinder interface { + FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) +} + +// FindPackage implements PackageFinder. +func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) { + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, err + } + return reg.FetchPackage(ctx, name) +} + +// mirrorJulia mirrors the source tarball of a version of a package in the +// General registry from the Pkg server, under the name and tree hash the +// Julia handler caches it as. The registry gives the tree hash, and the +// tarball is fetched unchecked, as the handler fetches it: Pkg checks the +// tree hash of what it extracts. The registry and the package's artifacts +// are not mirrored. +func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + uuid := strings.ToLower(pv.Qualifiers["uuid"]) + if uuid == "" { + fail(fmt.Errorf("julia packages need a uuid qualifier, such as pkg:julia/%s@%s?uuid=", pv.Name, pv.Version)) + return + } + + pkg, err := m.packageFinder().FindPackage(ctx, pv.Ecosystem, pv.Name) + if err != nil { + fail(err) + return + } + registered, _ := pkg.Metadata["uuid"].(string) + if !strings.EqualFold(registered, uuid) { + fail(fmt.Errorf("%s in the General registry has uuid %s, not %s; only General packages can be mirrored", pv.Name, registered, uuid)) + return + } + // The handler names packages as the registry does. + name := pkg.Name + if name == "" { + name = pv.Name + } + + v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version) + if err != nil { + fail(err) + return + } + treeHash, _ := v.Metadata["git-tree-sha1"].(string) + if treeHash == "" { + fail(errors.New("the registry lists no tree hash for this version")) + return + } + url, filename, err := handler.JuliaPackageDownload(m.JuliaUpstream, uuid, treeHash) + if err != nil { + fail(err) + return + } + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, treeHash, filename, url) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// packageFinder finds packages where Versions lists versions, so both come +// from the same registry. +func (m *Mirror) packageFinder() PackageFinder { //nolint:ireturn // any finder will do + if finder, ok := m.Versions.(PackageFinder); ok { + return finder + } + return RegistryVersions{} +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 6bd8d183..2fa626a7 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -33,6 +33,10 @@ type Mirror struct { // schedules. Nil lists them from the public registries. Versions VersionLister + // JuliaUpstream is the Pkg server Julia packages are fetched from. Empty + // uses https://pkg.julialang.org. + JuliaUpstream string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string @@ -236,6 +240,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorComposer(ctx, pv, tracker) return } + if pv.Ecosystem == "julia" { + m.mirrorJulia(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index d8778b5c..b742e002 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,7 +16,8 @@ type PackageVersion struct { Version string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select - // which files of the version are mirrored. + // which files of the version are mirrored. Julia uses uuid to identify + // the package. Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string @@ -75,6 +76,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro name = p.Namespace + "/" + p.Name } qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map()) + if ecosystem == "julia" { + qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + } // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 1b68a6dd..2c33e7e9 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -57,6 +57,15 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, err + } + return reg.FetchVersions(ctx, name) +} + +// registry returns the client for the registry name is listed in. +func (r RegistryVersions) registry(ecosystem, name string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -69,9 +78,5 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str if client == nil { client = registries.DefaultClient() } - reg, err := registries.New(purlType, baseURL, client) - if err != nil { - return nil, err - } - return reg.FetchVersions(ctx, name) + return registries.New(purlType, baseURL, client) } diff --git a/internal/mirror/versions_test.go b/internal/mirror/versions_test.go index 9c04a210..29f9f41a 100644 --- a/internal/mirror/versions_test.go +++ b/internal/mirror/versions_test.go @@ -96,3 +96,23 @@ func TestSourceFromRequestUsesMirrorVersions(t *testing.T) { t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got) } } + +func TestPURLSourceKeepsJuliaUUID(t *testing.T) { + const uuid = "7876af07-990d-54b4-ab0e-23690620f79a" + versions := &fakeVersions{versions: []string{"0.5.5"}} + source := &PURLSource{ + PURLs: []string{"pkg:julia/Example@0.5.4?uuid=" + uuid, "pkg:julia/Example?uuid=" + uuid}, + Versions: versions, + } + + items := enumerate(t, source) + + qualifiers := map[string]string{"uuid": uuid} + want := []PackageVersion{ + {Ecosystem: "julia", Name: "Example", Version: "0.5.4", Qualifiers: qualifiers}, + {Ecosystem: "julia", Name: "Example", Version: "0.5.5", Qualifiers: qualifiers}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index 142594c9..5ab3ca39 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -61,6 +61,7 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio proxy, _ := NewMirrorProxy(cfg, db, store, logger) m := mirror.New(proxy, db, store, logger, 1) m.Versions = versions + m.JuliaUpstream = cfg.Upstream.Julia if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_julia_test.go b/internal/server/mirror_julia_test.go new file mode 100644 index 00000000..607a2dca --- /dev/null +++ b/internal/server/mirror_julia_test.go @@ -0,0 +1,162 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/registries" +) + +const ( + juliaExampleUUID = "7876af07-990d-54b4-ab0e-23690620f79a" + juliaExampleTree = "e1f0e1a832ccd8e97d6d0348dec33ee139a5aeaf" + juliaGeneralUUID = "23338594-aafe-5451-b93e-139f81909106" + juliaRegistryTree = "465d1e55da75e226473a37c40bf2c20aee19234d" +) + +// juliaUpstream serves the General registry's files for Example, which has +// version 0.5.5, and as a Pkg server /registries, the General registry +// tarball and Example's 0.5.5 source tarball. It counts package downloads. +type juliaUpstream struct { + *httptest.Server + tarball string + packageHits atomic.Int32 + registryHits atomic.Int32 +} + +func newJuliaUpstream(t *testing.T) *juliaUpstream { + t.Helper() + u := &juliaUpstream{tarball: "Example 0.5.5 source"} + registry := juliaRegistryTarball(t, "[packages]\n"+juliaExampleUUID+` = { name = "Example", path = "E/Example" }`+"\n") + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/E/Example/Package.toml": + _, _ = io.WriteString(w, "name = \"Example\"\nuuid = \""+juliaExampleUUID+"\"\nrepo = \"https://github.com/JuliaLang/Example.jl.git\"\n") + case "/E/Example/Versions.toml": + _, _ = io.WriteString(w, "[\"0.5.4\"]\ngit-tree-sha1 = \"11820aa9c229fd3833d4bd69e5e75ef4e7273bf1\"\n\n[\"0.5.5\"]\ngit-tree-sha1 = \""+juliaExampleTree+"\"\n") + case "/registries": + _, _ = io.WriteString(w, "/registry/"+juliaGeneralUUID+"/"+juliaRegistryTree+"\n") + case "/registry/" + juliaGeneralUUID + "/" + juliaRegistryTree: + u.registryHits.Add(1) + _, _ = w.Write(registry) + case "/package/" + juliaExampleUUID + "/" + juliaExampleTree: + u.packageHits.Add(1) + _, _ = io.WriteString(w, u.tarball) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +// juliaRegistryTarball returns a gzipped registry tarball holding registryToml. +func juliaRegistryTarball(t *testing.T, registryToml string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: "Registry.toml", Mode: 0o644, Size: int64(len(registryToml))}); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, registryToml); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// mirrorJulia mirrors purls from upstream as the Pkg server, with the +// registry's files read from it too. +func mirrorJulia(t *testing.T, upstream *juliaUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + cfg := config.Default() + cfg.Upstream.Julia = upstream.URL + versions := mirror.RegistryVersions{ + Client: registries.NewClient(registries.WithHTTPClient(upstream.Client())), + URL: func(string, string) (string, error) { return upstream.URL, nil }, + } + return mirrorWithVersions(t, cfg, versions, purls...) +} + +func getJuliaPackage(julia *handler.JuliaHandler) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + julia.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/package/"+juliaExampleUUID+"/"+juliaExampleTree, nil)) + return rec +} + +func TestMirrorJuliaPackage(t *testing.T) { + upstream := newJuliaUpstream(t) + + progress, proxy := mirrorJulia(t, upstream, "pkg:julia/Example@0.5.5?uuid="+juliaExampleUUID) + + if progress.Completed != 1 || progress.Failed != 0 || upstream.packageHits.Load() != 1 { + t.Fatalf("completed = %d, failed = %d, package hits = %d, errors = %v", progress.Completed, progress.Failed, upstream.packageHits.Load(), progress.Errors) + } + if cached, _ := proxy.GetCachedArtifact(t.Context(), "julia", "Example", juliaExampleTree, juliaExampleTree+".tar.gz"); cached == nil { + t.Fatal("the tarball isn't cached under Example and its tree hash") + } else { + _ = cached.Reader.Close() + } + + // With the registry's names loaded, the handler finds the tarball under + // the package's name without fetching it again. + julia := handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL) + if rec := getJuliaPackage(julia); rec.Code != http.StatusOK || rec.Body.String() != upstream.tarball { + t.Errorf("status = %d, body = %q, want 200 and the tarball", rec.Code, rec.Body) + } + if upstream.registryHits.Load() == 0 { + t.Error("the handler didn't load the registry's names") + } + + // Without the upstream, a new handler can't load the names, and finds + // the tarball by its tree hash. + upstream.Close() + if rec := getJuliaPackage(handler.NewJuliaHandlerWithUpstream(proxy, upstream.URL)); rec.Code != http.StatusOK || rec.Body.String() != upstream.tarball { + t.Errorf("from cache: status = %d, body = %q, want 200 and the tarball", rec.Code, rec.Body) + } + if got := upstream.packageHits.Load(); got != 1 { + t.Errorf("package fetched %d times, want 1", got) + } +} + +func TestMirrorJuliaRefusesOtherUUID(t *testing.T) { + upstream := newJuliaUpstream(t) + + progress, _ := mirrorJulia(t, upstream, "pkg:julia/Example@0.5.5?uuid=00000000-0000-0000-0000-000000000000") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, juliaExampleUUID) { + t.Errorf("failed = %d, errors = %v; want a failure naming General's uuid", progress.Failed, progress.Errors) + } + if upstream.packageHits.Load() != 0 { + t.Error("a tarball was fetched for a package General doesn't have") + } +} + +func TestServerMirrorFetchesJuliaFromUpstream(t *testing.T) { + cfg := config.Default() + cfg.Upstream.Julia = "https://julia.example.com" + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, nil, nil, logger) + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + + if got := s.newMirror(proxy, 1).JuliaUpstream; got != cfg.Upstream.Julia { + t.Errorf("JuliaUpstream = %q, want upstream.julia", got) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 404388d7..25bee85b 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -30,10 +30,18 @@ func NewMirrorVersions(upstream config.UpstreamConfig, client *http.Client) mirr func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m := mirror.New(proxy, s.db, s.storage, s.logger, workers) m.Images = s.newContainerHandler(proxy) - m.Versions = NewMirrorVersions(s.cfg.Upstream, proxy.HTTPClient) + UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m } +// UseMirrorUpstreams points m at the configured upstreams: it lists +// versions from them, through client, and fetches Julia packages from +// upstream.julia. +func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { + m.Versions = NewMirrorVersions(upstream, client) + m.JuliaUpstream = upstream.Julia +} + // mirrorVersionsURL returns the registry base URL to list a package's // versions from: the upstream the proxy serves it from, where that upstream // has the API the registries client reads. An empty URL means the public @@ -79,7 +87,10 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, } return "", nil case "julia": - return publicOnly("upstream.julia", u.Julia, d.Julia) + // Packages are identified by uuid and tree hash, the same on every + // Pkg server serving General, so General's are used whatever the + // upstream. + return "", nil default: return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) } diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go index 898c2128..7e59dd43 100644 --- a/internal/server/mirror_versions_test.go +++ b/internal/server/mirror_versions_test.go @@ -127,7 +127,7 @@ func TestMirrorVersionsURL(t *testing.T) { {"conda default", config.Default().Upstream, "conda", "numpy", "", ""}, {"conda channel", custom, "conda", "numpy", "", "upstream.conda"}, {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, - {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, + {"julia server lists from General", custom, "julia", "JSON", "", ""}, {"helm", custom, "helm", "nginx", "", "give a version"}, {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, {"apk", custom, "apk", "curl", "", "give a version"}, From 391c5b709fc6c3a43b72c67af58364313903f34c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:40:15 +0100 Subject: [PATCH 2/2] Index artifacts by filename and shorten Julia comments CachedArtifactNames filters on filename alone, which the unique (version_purl, filename) index can't serve. --- internal/database/retention_test.go | 33 +++++++++++++++++++++++++++++ internal/database/schema.go | 15 +++++++++++++ internal/handler/julia.go | 4 +--- internal/mirror/julia.go | 8 ++----- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 213ae220..bd9ad289 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -394,3 +394,36 @@ func TestGetRetentionCandidatesTimezone(t *testing.T) { } }) } + +func TestArtifactFilenameIndex(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_artifacts_filename'` + if db.Dialect() == DialectPostgres { + indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_artifacts_filename'` + } + hasIndex := func() bool { + t.Helper() + var count int + if err := db.Get(&count, indexQuery); err != nil { + t.Fatalf("querying indexes: %v", err) + } + return count == 1 + } + + if !hasIndex() { + t.Error("fresh schema has no idx_artifacts_filename") + } + if _, err := db.Exec(`DROP INDEX idx_artifacts_filename`); err != nil { + t.Fatalf("dropping index: %v", err) + } + if err := migrateAddArtifactFilenameIndex(db); err != nil { + t.Fatalf("migration: %v", err) + } + if err := migrateAddArtifactFilenameIndex(db); err != nil { + t.Fatalf("migration is not idempotent: %v", err) + } + if !hasIndex() { + t.Error("migration did not create idx_artifacts_filename") + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index 46a21eca..f0ec6516 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -222,6 +223,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); CREATE TABLE IF NOT EXISTS pending_deletes ( @@ -323,6 +325,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -347,6 +350,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash); +CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); ` @@ -463,6 +467,7 @@ var migrations = []migration{ {"016_add_fetch_leases", migrateAddFetchLeases}, {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, + {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, } // isTableNotFound returns true if the error indicates a missing table. @@ -835,6 +840,16 @@ func migrateAddArtifactContentHashIndex(db *DB) error { return nil } +// migrateAddArtifactFilenameIndex indexes artifacts by filename, for +// lookups that don't know the version, such as CachedArtifactNames. +func migrateAddArtifactFilenameIndex(db *DB) error { + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename)`) + if err != nil { + return fmt.Errorf("creating artifacts filename index: %w", err) + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 6ec02095..dbac3136 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -184,9 +184,7 @@ func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheRes // JuliaPackageDownload returns the URL of a package's tarball on the Pkg // server at upstreamURL, or the default one when empty, and the file name the -// Julia handler caches it under. The handler caches a package tarball as -// version treeHash, the git tree hash of the package's source, under the -// package's name in the General registry. +// Julia handler caches it under. func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) { if !validJuliaUUID(uuid) { return "", "", fmt.Errorf("invalid julia package uuid %q", uuid) diff --git a/internal/mirror/julia.go b/internal/mirror/julia.go index 30efb683..613ba767 100644 --- a/internal/mirror/julia.go +++ b/internal/mirror/julia.go @@ -25,12 +25,8 @@ func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name strin return reg.FetchPackage(ctx, name) } -// mirrorJulia mirrors the source tarball of a version of a package in the -// General registry from the Pkg server, under the name and tree hash the -// Julia handler caches it as. The registry gives the tree hash, and the -// tarball is fetched unchecked, as the handler fetches it: Pkg checks the -// tree hash of what it extracts. The registry and the package's artifacts -// are not mirrored. +// mirrorJulia fetches the tarball unchecked because Pkg checks the tree hash +// of what it extracts. func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } uuid := strings.ToLower(pv.Qualifiers["uuid"])