Mirror Helm charts

cmd/proxy/main.go

@@ -125,6 +125,7 @@ import (
 	"github.com/git-pkgs/proxy/internal/config"
 	"github.com/git-pkgs/proxy/internal/database"
 	"github.com/git-pkgs/proxy/internal/denylist"
+	"github.com/git-pkgs/proxy/internal/handler"
 	"github.com/git-pkgs/proxy/internal/memlimit"
 	"github.com/git-pkgs/proxy/internal/mirror"
 	"github.com/git-pkgs/proxy/internal/server"
@@ -558,18 +559,13 @@ func runMirror() {
 	proxy.MetadataTTL = cfg.ParseMetadataTTL()
 	proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
 
-	m := mirror.New(proxy, db, store, logger, *concurrency)
-	m.Images = images
-	m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
+	m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency)
 	switch s := source.(type) {
 	case *mirror.PURLSource:
 		s.Versions = m.Versions
 	case *mirror.SBOMSource:
 		s.Versions = m.Versions
 	}
-	if cfg.Scanning.Enabled {
-		refuseUnscannedMirroring(m)
-	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
@@ -616,6 +612,20 @@ func runMirror() {
 	}
 }
 
+// newCommandMirror builds the mirror proxy mirror runs, which fetches and
+// lists versions from the configured upstreams.
+func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.ContainerHandler,
+	db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror {
+	m := mirror.New(proxy, db, store, logger, workers)
+	m.Images = images
+	m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
+	m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
+	if cfg.Scanning.Enabled {
+		refuseUnscannedMirroring(m)
+	}
+	return m
+}
+
 // mirrorExitCode returns the exit status for a finished mirror run: 1 when any
 // package version failed to mirror, 0 otherwise.
 func mirrorExitCode(progress *mirror.Progress) int {

docs/configuration.md

@@ -1565,6 +1565,8 @@ A conda version is mirrored with every build anaconda.org lists for it, across a
 
 A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`.
 
+A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images.
+
 | Flag | Default | Description |
 |------|---------|-------------|
 | `--sbom` | | Path to CycloneDX or SPDX SBOM file |

internal/handler/helm.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
@@ -82,7 +83,7 @@ func (h *HelmHandler) handleIndex(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	index := h.fetchIndex(r, repository, upstreamURL)
+	index := h.fetchIndex(r.Context(), repository, upstreamURL)
 	if index.err != nil {
 		h.serveIndexError(w, index.err)
 		return
@@ -130,7 +131,7 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	index := h.fetchIndex(r, repository, upstreamURL)
+	index := h.fetchIndex(r.Context(), repository, upstreamURL)
 	if index.err != nil {
 		h.serveIndexError(w, index.err)
 		return
@@ -182,9 +183,9 @@ func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL s
 	return name, upstreamURL, ok
 }
 
-func (h *HelmHandler) fetchIndex(r *http.Request, repository, upstreamURL string) metadataResult {
+func (h *HelmHandler) fetchIndex(ctx context.Context, repository, upstreamURL string) metadataResult {
 	return h.proxy.fetchMetadataDocument(
-		r.Context(),
+		ctx,
 		helmMetadataEcosystem,
 		h.indexCacheKey(repository, upstreamURL),
 		upstreamURL+"/"+helmIndexFilename,
@@ -278,10 +279,11 @@ func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index me
 }
 
 // helmChartLookup holds the HTTP chart downloads an index lists, by chart
-// digest, in index order.
+// digest, in index order, and each release by chart name and version.
 type helmChartLookup struct {
 	indexDigest string
 	byDigest    map[string][]helmChartEntry
+	byRelease   map[helmRelease]helmReleaseDownload
 }
 
 type helmChartEntry struct {
@@ -290,6 +292,16 @@ type helmChartEntry struct {
 	downloads []helmChartDownload
 }
 
+type helmRelease struct{ chart, version string }
+
+// helmReleaseDownload is what helm downloads for a release: the first URL
+// the index lists for it, checked against its digest.
+type helmReleaseDownload struct {
+	digest  string
+	created time.Time
+	first   helmChartDownload
+}
+
 // chartLookup returns the charts the index lists, parsing it only when the
 // last lookup built for the repository came from other bytes. An index
 // without a digest, read with the metadata cache off, is parsed every time.
@@ -326,7 +338,7 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh
 		return nil, err
 	}
 
-	lookup := &helmChartLookup{byDigest: make(map[string][]helmChartEntry)}
+	lookup := &helmChartLookup{byDigest: make(map[string][]helmChartEntry), byRelease: make(map[helmRelease]helmReleaseDownload)}
 	for i := 0; i < len(entries.Content); i += 2 {
 		chartName := entries.Content[i].Value
 		releases := entries.Content[i+1]
@@ -339,6 +351,18 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh
 				continue
 			}
 			entry := helmChartEntry{name: chartName, created: chart.created}
+			if version := helmMappingValue(release, "version"); version != nil && version.Value != "" {
+				// helm takes the first release listed for a version.
+				key := helmRelease{chartName, version.Value}
+				if _, listed := lookup.byRelease[key]; !listed {
+					first := chart.downloads[0]
+					lookup.byRelease[key] = helmReleaseDownload{
+						digest:  chart.digest,
+						created: chart.created,
+						first:   helmChartDownload{url: first.url, filename: first.filename, oci: first.oci},
+					}
+				}
+			}
 			for _, download := range chart.downloads {
 				if !download.oci {
 					// Drop the YAML node, so the lookup doesn't keep the

internal/handler/helm_mirror.go

@@ -0,0 +1,67 @@
+package handler
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"slices"
+	"strings"
+)
+
+// MirrorChart caches a chart release of a configured repository under the
+// key handleChart serves it from, checked against the digest the index
+// lists. The index is read through the metadata cache, as handleIndex reads
+// it, so with metadata caching on helm finds the chart there as well while
+// the repository is down.
+func (h *HelmHandler) MirrorChart(ctx context.Context, repository, chart, version string) (*CacheResult, error) {
+	upstreamURL, ok := h.repositories[repository]
+	if !ok {
+		return nil, h.unknownRepositoryError(repository)
+	}
+	index := h.fetchIndex(ctx, repository, upstreamURL)
+	if index.err != nil {
+		return nil, fmt.Errorf("fetching the %s index: %w", repository, index.err)
+	}
+	charts, err := h.chartLookup(repository, upstreamURL, index)
+	if err != nil {
+		return nil, err
+	}
+
+	release, ok := charts.byRelease[helmRelease{chart, version}]
+	switch {
+	case !ok:
+		return nil, fmt.Errorf("%w: %s %s is not in the %s index", ErrUpstreamNotFound, chart, version, repository)
+	case release.first.oci:
+		return nil, fmt.Errorf("%s %s is published as %s, which helm pulls through the container registry routes; mirror it as an image: %s",
+			chart, version, release.first.url, ociChartPURL(release.first.url, version))
+	case h.chartOnCooldown(chart, release.created):
+		return nil, fmt.Errorf("%s %s is in cooldown, so the index doesn't list it", chart, version)
+	}
+	return h.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, helmMetadataEcosystem, repository, release.digest,
+		release.first.filename, release.first.url, "sha256:"+release.digest)
+}
+
+func (h *HelmHandler) unknownRepositoryError(repository string) error {
+	names := make([]string, 0, len(h.repositories))
+	for name := range h.repositories {
+		names = append(names, name)
+	}
+	if len(names) == 0 {
+		return fmt.Errorf("unknown Helm repository %q: upstream.helm configures none", repository)
+	}
+	slices.Sort(names)
+	return fmt.Errorf("unknown Helm repository %q: upstream.helm configures %s", repository, strings.Join(names, ", "))
+}
+
+// ociChartPURL returns the image PURL that mirrors an OCI chart reference,
+// tagged with the chart version when the reference has no tag or digest, as
+// helm pulls it.
+func ociChartPURL(reference, version string) string {
+	repository := strings.TrimPrefix(reference, "oci://")
+	if at := strings.LastIndex(repository, "@"); at >= 0 {
+		repository, version = repository[:at], repository[at+1:]
+	} else if colon := strings.LastIndex(repository, ":"); colon > strings.LastIndex(repository, "/") {
+		repository, version = repository[:colon], repository[colon+1:]
+	}
+	return "pkg:oci/" + path.Base(repository) + "@" + version + "?repository_url=" + repository
+}

internal/handler/helm_mirror_test.go

@@ -0,0 +1,17 @@
+package handler
+
+import "testing"
+
+func TestOCIChartPURL(t *testing.T) {
+	tests := []struct{ reference, want string }{
+		{"oci://ghcr.io/owner/demo:2.0.0", "pkg:oci/demo@2.0.0?repository_url=ghcr.io/owner/demo"},
+		{"oci://ghcr.io/owner/demo", "pkg:oci/demo@1.0.0?repository_url=ghcr.io/owner/demo"},
+		{"oci://localhost:5000/demo", "pkg:oci/demo@1.0.0?repository_url=localhost:5000/demo"},
+		{"oci://ghcr.io/owner/demo@sha256:abc", "pkg:oci/demo@sha256:abc?repository_url=ghcr.io/owner/demo"},
+	}
+	for _, tt := range tests {
+		if got := ociChartPURL(tt.reference, "1.0.0"); got != tt.want {
+			t.Errorf("ociChartPURL(%q) = %q, want %q", tt.reference, got, tt.want)
+		}
+	}
+}

internal/handler/helm_pypi_rewrite_test.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -138,7 +139,7 @@ func TestHelmChartLookupFollowsIndex(t *testing.T) {
 	proxy.MetadataTTL = time.Hour
 	h := NewHelmHandler(proxy, "http://proxy.example", map[string]string{"stable": upstream.URL})
 	index := func() metadataResult {
-		return h.fetchIndex(httptest.NewRequest(http.MethodGet, "/", nil), "stable", upstream.URL)
+		return h.fetchIndex(context.Background(), "stable", upstream.URL)
 	}
 
 	first, err := h.findChartDownload("stable", upstream.URL, index(), digestA, "a-1.0.0.tgz")

internal/mirror/helm.go

@@ -0,0 +1,40 @@
+package mirror
+
+import (
+	"context"
+	"errors"
+	"strings"
+
+	"github.com/git-pkgs/proxy/internal/handler"
+)
+
+// ChartMirrorer caches Helm charts from the configured repositories the way
+// the Helm handler serves them.
+type ChartMirrorer interface {
+	MirrorChart(ctx context.Context, repository, chart, version string) (*handler.CacheResult, error)
+}
+
+var (
+	errHelmNoRepository = errors.New("helm charts need the upstream.helm repository as the namespace, such as pkg:helm/{repository}/{chart}@{version}")
+	errHelmUnavailable  = errors.New("mirroring Helm charts isn't available here")
+)
+
+// mirrorHelm mirrors a chart release. The PURL namespace names the
+// upstream.helm repository, since the handler caches charts per repository.
+func (m *Mirror) mirrorHelm(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
+	repository, chart, ok := strings.Cut(pv.Name, "/")
+	if !ok || repository == "" || chart == "" {
+		m.recordFailure(tracker, pv, errHelmNoRepository)
+		return
+	}
+	if m.Charts == nil {
+		m.recordFailure(tracker, pv, errHelmUnavailable)
+		return
+	}
+	result, err := m.Charts.MirrorChart(ctx, repository, chart, pv.Version)
+	if err != nil {
+		m.recordFailure(tracker, pv, err)
+		return
+	}
+	m.recordResult(tracker, pv, result)
+}

internal/mirror/mirror.go

@@ -29,6 +29,9 @@ type Mirror struct {
 	// other unsupported ecosystem.
 	Images ImageMirrorer
 
+	// Charts mirrors helm charts. Without it they fail.
+	Charts ChartMirrorer
+
 	// Versions lists the versions of unversioned PURLs in mirror jobs and
 	// schedules. Nil lists them from the public registries.
 	Versions VersionLister
@@ -236,6 +239,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
 		m.mirrorComposer(ctx, pv, tracker)
 		return
 	}
+	if pv.Ecosystem == "helm" {
+		m.mirrorHelm(ctx, pv, tracker)
+		return
+	}
 	result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "")
 	if err != nil {
 		m.recordFailure(tracker, pv, err)

internal/mirror/versions.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/git-pkgs/purl"
 	"github.com/git-pkgs/registries"
@@ -73,5 +74,10 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str
 	if err != nil {
 		return nil, err
 	}
+	if purlType == "helm" {
+		// The namespace names the repository, which URL resolves to the
+		// index the chart is listed in.
+		_, name, _ = strings.Cut(name, "/")
+	}
 	return reg.FetchVersions(ctx, name)
 }

internal/server/mirror_fetch_test.go

@@ -59,12 +59,14 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio
 	}
 	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
 	proxy, _ := NewMirrorProxy(cfg, db, store, logger)
+	proxy.CacheMetadata = true // as proxy mirror does
 	m := mirror.New(proxy, db, store, logger, 1)
+	m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
 	m.Versions = versions
 	if versions == nil {
 		m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
 	}
-	progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls})
+	progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls, Versions: m.Versions})
 	if err != nil {
 		t.Fatalf("Run() error = %v", err)
 	}