Mirror Julia packages

cmd/proxy/main.go

@@ -619,7 +619,7 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.
 	m := mirror.New(proxy, db, store, logger, workers)
 	m.Images = images
 	m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
-	m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
+	server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient)
 	if cfg.Scanning.Enabled {
 		refuseUnscannedMirroring(m)
 	}

docs/configuration.md

@@ -1569,6 +1569,8 @@ A Composer release is mirrored with the dist archive `upstream.composer` lists f
 
 A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images.
 
+A Julia package is mirrored with the source tarball `upstream.julia` serves at `/package/{uuid}/{tree-hash}`, cached under the package's name and tree hash as `/julia/package/...` serves it. The PURL needs the `uuid` qualifier, for example `pkg:julia/Example@0.5.5?uuid=7876af07-990d-54b4-ab0e-23690620f79a`. Only packages of the General registry can be mirrored: the uuid and tree hash are read from General on GitHub, also with another `upstream.julia`, since they are the same on every Pkg server serving General. The tarball isn't checked by the proxy, as when serving; Pkg checks the tree hash of what it unpacks, except on Windows. Dependencies, the artifacts in a package's `Artifacts.toml` and the registry aren't mirrored. An offline `Pkg.add` also needs the registry: one installed on the client, or the one the proxy cached when a client last fetched it, which it serves offline with `cache_metadata` on.
+
 | Flag | Default | Description |
 |------|---------|-------------|
 | `--sbom` | | Path to CycloneDX or SPDX SBOM file |
@@ -1589,7 +1591,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
 
 ### Unversioned PURLs
 
-A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version.
+A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry.
 
 ### Container images
 

internal/database/cached_names.go

@@ -0,0 +1,21 @@
+package database
+
+// CachedArtifactNames returns the names of the packages of ecosystem that
+// have a cached artifact called filename, for protocols whose file names
+// identify an artifact without its package name.
+func (db *DB) CachedArtifactNames(ecosystem, filename string) ([]string, error) {
+	var names []string
+	query := db.Rebind(`
+		SELECT DISTINCT packages.name
+		FROM artifacts
+		JOIN versions ON versions.purl = artifacts.version_purl
+		JOIN packages ON packages.purl = versions.package_purl
+		WHERE packages.ecosystem = ? AND artifacts.filename = ?
+		  AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL
+		ORDER BY packages.name
+	`)
+	if err := db.Select(&names, query, ecosystem, filename); err != nil {
+		return nil, err
+	}
+	return names, nil
+}

internal/database/retention_test.go

@@ -394,3 +394,36 @@ func TestGetRetentionCandidatesTimezone(t *testing.T) {
 		}
 	})
 }
+
+func TestArtifactFilenameIndex(t *testing.T) {
+	runWithBothDatabases(t, func(t *testing.T, db *DB) {
+		indexQuery := `SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_artifacts_filename'`
+		if db.Dialect() == DialectPostgres {
+			indexQuery = `SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'idx_artifacts_filename'`
+		}
+		hasIndex := func() bool {
+			t.Helper()
+			var count int
+			if err := db.Get(&count, indexQuery); err != nil {
+				t.Fatalf("querying indexes: %v", err)
+			}
+			return count == 1
+		}
+
+		if !hasIndex() {
+			t.Error("fresh schema has no idx_artifacts_filename")
+		}
+		if _, err := db.Exec(`DROP INDEX idx_artifacts_filename`); err != nil {
+			t.Fatalf("dropping index: %v", err)
+		}
+		if err := migrateAddArtifactFilenameIndex(db); err != nil {
+			t.Fatalf("migration: %v", err)
+		}
+		if err := migrateAddArtifactFilenameIndex(db); err != nil {
+			t.Fatalf("migration is not idempotent: %v", err)
+		}
+		if !hasIndex() {
+			t.Error("migration did not create idx_artifacts_filename")
+		}
+	})
+}

internal/database/schema.go

@@ -80,6 +80,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
 CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
+CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 
 CREATE TABLE IF NOT EXISTS pending_deletes (
@@ -222,6 +223,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
 CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
+CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 
 CREATE TABLE IF NOT EXISTS pending_deletes (
@@ -323,6 +325,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
 CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
+CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 `
 
@@ -347,6 +350,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
 CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
+CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 `
 
@@ -463,6 +467,7 @@ var migrations = []migration{
 	{"016_add_fetch_leases", migrateAddFetchLeases},
 	{"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex},
 	{"018_add_upstream_holds", migrateAddUpstreamHolds},
+	{"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex},
 }
 
 // isTableNotFound returns true if the error indicates a missing table.
@@ -835,6 +840,16 @@ func migrateAddArtifactContentHashIndex(db *DB) error {
 	return nil
 }
 
+// migrateAddArtifactFilenameIndex indexes artifacts by filename, for
+// lookups that don't know the version, such as CachedArtifactNames.
+func migrateAddArtifactFilenameIndex(db *DB) error {
+	_, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_filename ON artifacts(filename)`)
+	if err != nil {
+		return fmt.Errorf("creating artifacts filename index: %w", err)
+	}
+	return nil
+}
+
 // migrateAddMetadataDates creates the table of Last-Modified dates for
 // rewritten metadata, which replicas sharing the database agree on.
 func migrateAddMetadataDates(db *DB) error {

internal/handler/julia.go

@@ -110,7 +110,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) {
 	h.proxy.Logger.Debug("julia registry request", "uuid", uuid, "hash", hash)
 
 	upstreamURL := h.upstreamURL + r.URL.Path
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, juliaTarball(hash), upstreamURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch registry")
 		return
@@ -137,8 +137,15 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) {
 
 	h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash)
 
+	if name == uuid {
+		if result := h.cachedPackage(r.Context(), hash); result != nil {
+			ServeArtifactRequest(w, r, result)
+			return
+		}
+	}
+
 	upstreamURL := h.upstreamURL + r.URL.Path
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, juliaTarball(hash), upstreamURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch package")
 		return
@@ -147,6 +154,54 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) {
 	ServeArtifactRequest(w, r, result)
 }
 
+// cachedPackage returns the package tarball with tree hash hash from the
+// cache, under whatever name it was cached as, or nil. Without the name map,
+// which needs the registry from upstream, a request can't be matched to the
+// name its package was cached under, but the tarball is identified by its
+// tree hash alone. The registry and artifact tarballs cached under the same
+// file name are skipped.
+func (h *JuliaHandler) cachedPackage(ctx context.Context, hash string) *CacheResult {
+	if h.proxy.DB == nil {
+		return nil
+	}
+	filename := juliaTarball(hash)
+	names, err := h.proxy.DB.CachedArtifactNames("julia", filename)
+	if err != nil {
+		h.proxy.Logger.Warn("looking up cached julia package", "hash", hash, "error", err)
+		return nil
+	}
+	for _, name := range names {
+		if name == juliaRegistryName || name == juliaArtifactName {
+			continue
+		}
+		result, err := h.proxy.GetCachedArtifact(ctx, "julia", name, hash, filename)
+		if err == nil && result != nil {
+			return result
+		}
+	}
+	return nil
+}
+
+// JuliaPackageDownload returns the URL of a package's tarball on the Pkg
+// server at upstreamURL, or the default one when empty, and the file name the
+// Julia handler caches it under.
+func JuliaPackageDownload(upstreamURL, uuid, treeHash string) (url, filename string, err error) {
+	if !validJuliaUUID(uuid) {
+		return "", "", fmt.Errorf("invalid julia package uuid %q", uuid)
+	}
+	if !juliaHexPattern.MatchString(treeHash) {
+		return "", "", fmt.Errorf("invalid julia tree hash %q", treeHash)
+	}
+	base := configuredUpstreamURL(upstreamURL, juliaUpstream)
+	return base + "/package/" + uuid + "/" + treeHash, juliaTarball(treeHash), nil
+}
+
+// juliaTarball is the file name a registry, package or artifact tarball is
+// cached under.
+func juliaTarball(hash string) string {
+	return hash + ".tar.gz"
+}
+
 // handleArtifact serves an immutable binary artifact tarball. Artifacts are
 // anonymous content-addressed blobs with no associated package name.
 func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) {
@@ -159,7 +214,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) {
 	h.proxy.Logger.Debug("julia artifact request", "hash", hash)
 
 	upstreamURL := h.upstreamURL + r.URL.Path
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, juliaTarball(hash), upstreamURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch artifact")
 		return

internal/mirror/julia.go

@@ -0,0 +1,84 @@
+package mirror
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/registries"
+)
+
+// PackageFinder finds a package's registry entry, which for Julia gives the
+// UUID the package is registered under.
+type PackageFinder interface {
+	FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error)
+}
+
+// FindPackage implements PackageFinder.
+func (r RegistryVersions) FindPackage(ctx context.Context, ecosystem, name string) (*registries.Package, error) {
+	reg, err := r.registry(ecosystem, name)
+	if err != nil {
+		return nil, err
+	}
+	return reg.FetchPackage(ctx, name)
+}
+
+// mirrorJulia fetches the tarball unchecked because Pkg checks the tree hash
+// of what it extracts.
+func (m *Mirror) mirrorJulia(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
+	fail := func(err error) { m.recordFailure(tracker, pv, err) }
+	uuid := strings.ToLower(pv.Qualifiers["uuid"])
+	if uuid == "" {
+		fail(fmt.Errorf("julia packages need a uuid qualifier, such as pkg:julia/%s@%s?uuid=<uuid>", pv.Name, pv.Version))
+		return
+	}
+
+	pkg, err := m.packageFinder().FindPackage(ctx, pv.Ecosystem, pv.Name)
+	if err != nil {
+		fail(err)
+		return
+	}
+	registered, _ := pkg.Metadata["uuid"].(string)
+	if !strings.EqualFold(registered, uuid) {
+		fail(fmt.Errorf("%s in the General registry has uuid %s, not %s; only General packages can be mirrored", pv.Name, registered, uuid))
+		return
+	}
+	// The handler names packages as the registry does.
+	name := pkg.Name
+	if name == "" {
+		name = pv.Name
+	}
+
+	v, err := m.versionFinder().FindVersion(ctx, pv.Ecosystem, name, pv.Version)
+	if err != nil {
+		fail(err)
+		return
+	}
+	treeHash, _ := v.Metadata["git-tree-sha1"].(string)
+	if treeHash == "" {
+		fail(errors.New("the registry lists no tree hash for this version"))
+		return
+	}
+	url, filename, err := handler.JuliaPackageDownload(m.JuliaUpstream, uuid, treeHash)
+	if err != nil {
+		fail(err)
+		return
+	}
+	result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, treeHash, filename, url)
+	if err != nil {
+		fail(err)
+		return
+	}
+	m.recordResult(tracker, pv, result)
+}
+
+// packageFinder finds packages where Versions lists versions, so both come
+// from the same registry.
+func (m *Mirror) packageFinder() PackageFinder { //nolint:ireturn // any finder will do
+	if finder, ok := m.Versions.(PackageFinder); ok {
+		return finder
+	}
+	return RegistryVersions{}
+}

internal/mirror/mirror.go

@@ -36,6 +36,10 @@ type Mirror struct {
 	// schedules. Nil lists them from the public registries.
 	Versions VersionLister
 
+	// JuliaUpstream is the Pkg server Julia packages are fetched from. Empty
+	// uses https://pkg.julialang.org.
+	JuliaUpstream string
+
 	// RefusePackages, when set, fails every item Images does not take with
 	// this reason instead of fetching it, so nothing is cached.
 	RefusePackages string
@@ -243,6 +247,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
 		m.mirrorHelm(ctx, pv, tracker)
 		return
 	}
+	if pv.Ecosystem == "julia" {
+		m.mirrorJulia(ctx, pv, tracker)
+		return
+	}
 	result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "")
 	if err != nil {
 		m.recordFailure(tracker, pv, err)

internal/mirror/source.go

@@ -16,7 +16,8 @@ type PackageVersion struct {
 	Version   string
 	// Qualifiers are the PURL qualifiers. Images use repository_url, tag and
 	// arch; PyPI uses file_name, and conda subdir, build and type, to select
-	// which files of the version are mirrored.
+	// which files of the version are mirrored. Julia uses uuid to identify
+	// the package.
 	Qualifiers map[string]string
 	// Platforms limit which platforms of an image index are mirrored.
 	Platforms []string
@@ -75,6 +76,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 			name = p.Namespace + "/" + p.Name
 		}
 		qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map())
+		if ecosystem == "julia" {
+			qualifiers = map[string]string{"uuid": p.Qualifier("uuid")}
+		}
 
 		// Images are never expanded to all tags, and conda packages without
 		// a channel aren't listed; both fail when they are mirrored.

internal/mirror/versions.go

@@ -58,6 +58,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers
 }
 
 func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) {
+	reg, err := r.registry(ecosystem, name)
+	if err != nil {
+		return nil, err
+	}
+	if purl.EcosystemToPURLType(ecosystem) == "helm" {
+		// The namespace names the repository, which URL resolves to the
+		// index the chart is listed in.
+		_, name, _ = strings.Cut(name, "/")
+	}
+	return reg.FetchVersions(ctx, name)
+}
+
+// registry returns the client for the registry name is listed in.
+func (r RegistryVersions) registry(ecosystem, name string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface
 	purlType := purl.EcosystemToPURLType(ecosystem)
 	var baseURL string
 	if r.URL != nil {
@@ -70,14 +84,5 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str
 	if client == nil {
 		client = registries.DefaultClient()
 	}
-	reg, err := registries.New(purlType, baseURL, client)
-	if err != nil {
-		return nil, err
-	}
-	if purlType == "helm" {
-		// The namespace names the repository, which URL resolves to the
-		// index the chart is listed in.
-		_, name, _ = strings.Cut(name, "/")
-	}
-	return reg.FetchVersions(ctx, name)
+	return registries.New(purlType, baseURL, client)
 }