Mirror conda packages

docs/configuration.md

@@ -1510,6 +1510,8 @@ Packages are downloaded from the configured upstreams: npm, cargo, gem, golang,
 
 A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`.
 
+A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else.
+
 | Flag | Default | Description |
 |------|---------|-------------|
 | `--sbom` | | Path to CycloneDX or SPDX SBOM file |

internal/handler/conda.go

@@ -142,9 +142,15 @@ func (h *CondaHandler) isPackageFile(filename string) bool {
 }
 
 // parseFilename extracts name and version from a Conda package filename.
+func (h *CondaHandler) parseFilename(filename string) (name, version string) {
+	return CondaFilenameIdentity(filename)
+}
+
+// CondaFilenameIdentity returns the name and version a download of filename
+// is cached under, or empty strings when it is not a package file name.
 // Conda filenames are: {name}-{version}-{build}.{ext}
 // e.g., "numpy-1.24.0-py311h64a7726_0.conda"
-func (h *CondaHandler) parseFilename(filename string) (name, version string) {
+func CondaFilenameIdentity(filename string) (name, version string) {
 	// Remove extension
 	base := filename
 	for _, ext := range []string{".conda", ".tar.bz2"} {

internal/mirror/files.go

@@ -0,0 +1,132 @@
+package mirror
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/registries"
+)
+
+// fileEcosystem describes an ecosystem whose versions publish several files,
+// each mirrored on its own.
+type fileEcosystem struct {
+	// qualifiers are the PURL qualifiers that select files. A file is
+	// mirrored when each one given matches the value attrs reports for it.
+	qualifiers []string
+	attrs      func(file registries.Artifact) map[string]string
+	// cacheKey returns the name, version and filename the download handler
+	// caches file under, or false when the handler would not cache it.
+	cacheKey func(pv PackageVersion, file registries.Artifact) (name, version, filename string, ok bool)
+	// check refuses a package the ecosystem can't mirror, before its files
+	// are listed.
+	check func(pv PackageVersion) error
+}
+
+var fileEcosystems = map[string]fileEcosystem{
+	"pypi": {
+		qualifiers: []string{"file_name"},
+		attrs: func(file registries.Artifact) map[string]string {
+			return map[string]string{"file_name": file.Filename}
+		},
+		cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) {
+			name, version := handler.PyPIFilenameIdentity(file.Filename)
+			if name == "" {
+				name, version = pv.Name, pv.Version
+			}
+			return name, version, file.Filename, true
+		},
+	},
+	"conda": {
+		qualifiers: []string{"subdir", "build", "type"},
+		attrs: func(file registries.Artifact) map[string]string {
+			subdir, _ := file.Metadata["subdir"].(string)
+			build, _ := file.Metadata["build"].(string)
+			return map[string]string{"subdir": subdir, "build": build, "type": condaFileType(file.Filename)}
+		},
+		cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) {
+			channel, _, _ := strings.Cut(pv.Name, "/")
+			subdir, _ := file.Metadata["subdir"].(string)
+			name, version := handler.CondaFilenameIdentity(file.Filename)
+			if name == "" || subdir == "" || condaFileType(file.Filename) == "" {
+				return "", "", "", false
+			}
+			return channel + "/" + name, version, subdir + "/" + file.Filename, true
+		},
+		check: func(pv PackageVersion) error {
+			if !strings.Contains(pv.Name, "/") {
+				return fmt.Errorf("conda packages need a channel, such as pkg:conda/conda-forge/%s", pv.Name)
+			}
+			return nil
+		},
+	},
+}
+
+// condaFileType returns the PURL type qualifier of a conda package file.
+func condaFileType(filename string) string {
+	switch {
+	case strings.HasSuffix(filename, ".conda"):
+		return "conda"
+	case strings.HasSuffix(filename, ".tar.bz2"):
+		return "tar.bz2"
+	default:
+		return ""
+	}
+}
+
+// selection returns the qualifiers of pv that select files, as
+// "key=value" pairs, for error messages.
+func (e fileEcosystem) selection(pv PackageVersion) string {
+	var pairs []string
+	for _, key := range e.qualifiers {
+		if value := pv.Qualifiers[key]; value != "" {
+			pairs = append(pairs, key+"="+value)
+		}
+	}
+	return strings.Join(pairs, " ")
+}
+
+// selects reports whether file matches every selecting qualifier of pv.
+func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool {
+	attrs := e.attrs(file)
+	for _, key := range e.qualifiers {
+		if want := pv.Qualifiers[key]; want != "" && attrs[key] != want {
+			return false
+		}
+	}
+	return true
+}
+
+// fileQualifiers keeps the qualifiers of a PURL that select files of its
+// ecosystem, or returns nil.
+func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string {
+	e, ok := fileEcosystems[ecosystem]
+	if !ok {
+		return nil
+	}
+	var kept map[string]string
+	for key, value := range qualifiers {
+		if value != "" && slices.Contains(e.qualifiers, key) {
+			if kept == nil {
+				kept = map[string]string{}
+			}
+			kept[key] = value
+		}
+	}
+	return kept
+}
+
+// condaName returns a conda package name with its channel, taken from the
+// namespace or the channel qualifier. Without either it stays bare, and
+// mirroring refuses it.
+func condaName(namespace, name, channel string) string {
+	switch {
+	case namespace != "":
+		return namespace + "/" + name
+	case channel != "":
+		return channel + "/" + name
+	default:
+		return name
+	}
+}

internal/mirror/files_test.go

@@ -0,0 +1,38 @@
+package mirror
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestPURLSourceCondaChannelAndQualifiers(t *testing.T) {
+	versions := &fakeVersions{versions: []string{"1.0", "1.1"}}
+	source := &PURLSource{
+		PURLs: []string{
+			"pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&build=py312h_0&type=conda",
+			"pkg:conda/numpy@2.1.0?channel=conda-forge&repository_url=https://example.com",
+			"pkg:conda/numpy@2.1.0",
+			"pkg:conda/numpy",
+			"pkg:conda/numpy?channel=bioconda&subdir=noarch",
+		},
+		Versions: versions,
+	}
+
+	got := enumerate(t, source)
+
+	want := []PackageVersion{
+		{Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0",
+			Qualifiers: map[string]string{"subdir": "linux-64", "build": "py312h_0", "type": "conda"}},
+		{Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0"},
+		{Ecosystem: "conda", Name: "numpy", Version: "2.1.0"},
+		{Ecosystem: "conda", Name: "numpy"},
+		{Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.0", Qualifiers: map[string]string{"subdir": "noarch"}},
+		{Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.1", Qualifiers: map[string]string{"subdir": "noarch"}},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("items = %+v\nwant %+v", got, want)
+	}
+	if !reflect.DeepEqual(versions.asked, []string{"conda/bioconda/numpy"}) {
+		t.Errorf("listed versions of %v, want only the package with a channel", versions.asked)
+	}
+}

internal/mirror/mirror.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -227,8 +228,8 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
 			"ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages)
 		return
 	}
-	if pv.Ecosystem == "pypi" {
-		m.mirrorFiles(ctx, pv, tracker)
+	if eco, ok := fileEcosystems[pv.Ecosystem]; ok {
+		m.mirrorFiles(ctx, pv, eco, tracker)
 		return
 	}
 	result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "")
@@ -257,34 +258,39 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
 	}
 }
 
-// mirrorFiles mirrors every file of a PyPI version, or only the one named by
-// the file_name qualifier. Each file is cached under the name and version
-// the download handler reads from its filename, so clients are served it.
-func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
+// mirrorFiles mirrors every file of a version, or those its qualifiers
+// select. Each file is cached under the name and version the download
+// handler reads from its filename, so clients are served it.
+func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, eco fileEcosystem, tracker *progressTracker) {
 	fail := func(err error) {
 		tracker.failed.Add(1)
 		tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error())
 		m.logger.Warn("mirror failed",
 			"ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "error", err)
 	}
+	if eco.check != nil {
+		if err := eco.check(pv); err != nil {
+			fail(err)
+			return
+		}
+	}
 	files, err := m.fileLister().ListFiles(ctx, pv.Ecosystem, pv.Name, pv.Version)
 	if err != nil {
 		fail(fmt.Errorf("listing files: %w", err))
 		return
 	}
-	wantFile := pv.Qualifiers["file_name"]
 	var matched, fetched int
 	var size int64
 	for _, file := range files {
-		if wantFile != "" && file.Filename != wantFile {
+		if !eco.selects(pv, file) {
 			continue
 		}
-		matched++
-		name, version := handler.PyPIFilenameIdentity(file.Filename)
-		if name == "" {
-			name, version = pv.Name, pv.Version
+		name, version, filename, ok := eco.cacheKey(pv, file)
+		if !ok {
+			continue
 		}
-		result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, name, version, file.Filename, file.URL, file.Integrity)
+		matched++
+		result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, name, version, filename, file.URL, fileDigest(file.Integrity))
 		if err != nil {
 			fail(fmt.Errorf("%s: %w", file.Filename, err))
 			return
@@ -297,9 +303,9 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr
 			size += result.Artifact.Size
 		}
 	}
-	switch {
-	case matched == 0 && wantFile != "":
-		fail(fmt.Errorf("no file named %q", wantFile))
+	switch selection := eco.selection(pv); {
+	case matched == 0 && selection != "":
+		fail(fmt.Errorf("no file matches %s", selection))
 	case matched == 0:
 		fail(errors.New("version has no files"))
 	case fetched == 0:
@@ -315,6 +321,18 @@ func (m *Mirror) mirrorFiles(ctx context.Context, pv PackageVersion, tracker *pr
 	}
 }
 
+// fileDigest returns the SHA-256 of one file as a digest, which is checked
+// against that file alone. The integrity GetOrFetchArtifactFromURLWithIntegrity
+// takes is kept with the version instead, and the files of a version that
+// share a name and version would overwrite each other's. Other checksums are
+// not checked, as the download handlers don't check them either.
+func fileDigest(integrity string) string {
+	if sum, ok := strings.CutPrefix(integrity, "sha256-"); ok {
+		return "sha256:" + sum
+	}
+	return ""
+}
+
 // fileLister lists files from where Versions lists versions, so both come
 // from the configured upstream.
 func (m *Mirror) fileLister() FileLister { //nolint:ireturn // any lister will do

internal/mirror/source.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/git-pkgs/purl"
 	"github.com/git-pkgs/sbom"
@@ -14,7 +15,8 @@ type PackageVersion struct {
 	Name      string
 	Version   string
 	// Qualifiers are the PURL qualifiers. Images use repository_url, tag and
-	// arch; PyPI uses file_name to mirror one file of the version.
+	// arch; PyPI uses file_name, and conda subdir, build and type, to select
+	// which files of the version are mirrored.
 	Qualifiers map[string]string
 	// Platforms limit which platforms of an image index are mirrored.
 	Platforms []string
@@ -67,19 +69,20 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 
 		ecosystem := handlerEcosystem(p.Type)
 		name := p.Name
-		if p.Namespace != "" {
+		if ecosystem == "conda" {
+			name = condaName(p.Namespace, p.Name, p.Qualifier("channel"))
+		} else if p.Namespace != "" {
 			name = p.Namespace + "/" + p.Name
 		}
+		qualifiers := fileQualifiers(ecosystem, p.Qualifiers.Map())
 
-		if p.Version != "" || isImageEcosystem(ecosystem) {
-			// Images are never expanded to all tags; an unversioned image
-			// fails when it is mirrored.
-			pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version}
+		// Images are never expanded to all tags, and conda packages without
+		// a channel aren't listed; both fail when they are mirrored.
+		if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) {
+			pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers}
 			if isImageEcosystem(ecosystem) {
 				pv.Qualifiers = p.Qualifiers.Map()
 				pv.Platforms = s.Platforms
-			} else if fileName := p.Qualifier("file_name"); fileName != "" && ecosystem == "pypi" {
-				pv.Qualifiers = map[string]string{"file_name": fileName}
 			}
 			if err := fn(pv); err != nil {
 				return err
@@ -93,7 +96,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 			return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err)
 		}
 		for _, v := range listed {
-			if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil {
+			if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers}); err != nil {
 				return err
 			}
 		}