Mirror Composer releases

docs/configuration.md

@@ -1510,6 +1510,10 @@ Packages are downloaded from the configured upstreams: npm, cargo, gem, golang,
 
 A PyPI version is mirrored with every file `upstream.pypi` lists for it: the sdist and each wheel, checked against the listed sha256 and cached under the name and version the download handler reads from the filename. Add the `file_name` qualifier to mirror one file, for example `pkg:pypi/zope.interface@7.2?file_name=zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl`.
 
+A conda version is mirrored with every build anaconda.org lists for it, across all subdirs, and each build is cached the way `/conda/{channel}/{subdir}/{file}` serves it. Give the channel as the namespace or the `channel` qualifier, for example `pkg:conda/conda-forge/numpy@2.1.0`; a package without one is refused. The `subdir`, `build` and `type` qualifiers select builds, for example `pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&type=conda`. Builds with a listed sha256 are checked against it. Files are listed from anaconda.org, so mirroring conda fails while `upstream.conda` points somewhere else.
+
+A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`.
+
 | Flag | Default | Description |
 |------|---------|-------------|
 | `--sbom` | | Path to CycloneDX or SPDX SBOM file |

internal/handler/composer.go

@@ -1011,6 +1011,36 @@ func composerIntegrity(shasum, version string, listed bool) string {
 	return sha1Checksum(shasum)
 }
 
+// ComposerReleaseArchive returns the file name the Composer handler caches a
+// tagged release's archive under, and the integrity it checks it against,
+// given the dist the release's metadata lists. It refuses versions the
+// handler only serves after reading the metadata again: branches, whose
+// archive changes with every push, and archives whose URL has a query, which
+// clients ask for with one.
+func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename, declared string, err error) {
+	switch {
+	case isDevVersion(version):
+		return "", "", fmt.Errorf("%s is a branch; Composer branches are fetched when requested, not mirrored", version)
+	case distURL == "":
+		return "", "", fmt.Errorf("%s has no dist archive", version)
+	case strings.ContainsAny(distURL, "?#"):
+		return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version)
+	}
+	return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil
+}
+
+// RecordComposerPublishTime stores the publish time of a tagged release from
+// the default repository, as the Composer handler does when it resolves the
+// release's dist, so the archive is served from the cache without the
+// metadata, with or without cooldown.
+func (p *Proxy) RecordComposerPublishTime(packageName, version string, publishedAt time.Time) error {
+	if p.DB == nil || isDevVersion(version) || publishedAt.IsZero() {
+		return nil
+	}
+	return p.DB.SetVersionPublishedAt(canonicalVersionPURL("composer", packageName, version),
+		canonicalPackagePURL("composer", packageName), publishedAt)
+}
+
 // proxyUpstream forwards a request to packagist.org without caching.
 func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
 	upstreamURL := h.upstreamURL + r.URL.Path

internal/handler/conda.go

@@ -142,9 +142,15 @@ func (h *CondaHandler) isPackageFile(filename string) bool {
 }
 
 // parseFilename extracts name and version from a Conda package filename.
+func (h *CondaHandler) parseFilename(filename string) (name, version string) {
+	return CondaFilenameIdentity(filename)
+}
+
+// CondaFilenameIdentity returns the name and version a download of filename
+// is cached under, or empty strings when it is not a package file name.
 // Conda filenames are: {name}-{version}-{build}.{ext}
 // e.g., "numpy-1.24.0-py311h64a7726_0.conda"
-func (h *CondaHandler) parseFilename(filename string) (name, version string) {
+func CondaFilenameIdentity(filename string) (name, version string) {
 	// Remove extension
 	base := filename
 	for _, ext := range []string{".conda", ".tar.bz2"} {

internal/mirror/files.go

@@ -0,0 +1,132 @@
+package mirror
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/registries"
+)
+
+// fileEcosystem describes an ecosystem whose versions publish several files,
+// each mirrored on its own.
+type fileEcosystem struct {
+	// qualifiers are the PURL qualifiers that select files. A file is
+	// mirrored when each one given matches the value attrs reports for it.
+	qualifiers []string
+	attrs      func(file registries.Artifact) map[string]string
+	// cacheKey returns the name, version and filename the download handler
+	// caches file under, or false when the handler would not cache it.
+	cacheKey func(pv PackageVersion, file registries.Artifact) (name, version, filename string, ok bool)
+	// check refuses a package the ecosystem can't mirror, before its files
+	// are listed.
+	check func(pv PackageVersion) error
+}
+
+var fileEcosystems = map[string]fileEcosystem{
+	"pypi": {
+		qualifiers: []string{"file_name"},
+		attrs: func(file registries.Artifact) map[string]string {
+			return map[string]string{"file_name": file.Filename}
+		},
+		cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) {
+			name, version := handler.PyPIFilenameIdentity(file.Filename)
+			if name == "" {
+				name, version = pv.Name, pv.Version
+			}
+			return name, version, file.Filename, true
+		},
+	},
+	"conda": {
+		qualifiers: []string{"subdir", "build", "type"},
+		attrs: func(file registries.Artifact) map[string]string {
+			subdir, _ := file.Metadata["subdir"].(string)
+			build, _ := file.Metadata["build"].(string)
+			return map[string]string{"subdir": subdir, "build": build, "type": condaFileType(file.Filename)}
+		},
+		cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) {
+			channel, _, _ := strings.Cut(pv.Name, "/")
+			subdir, _ := file.Metadata["subdir"].(string)
+			name, version := handler.CondaFilenameIdentity(file.Filename)
+			if name == "" || subdir == "" || condaFileType(file.Filename) == "" {
+				return "", "", "", false
+			}
+			return channel + "/" + name, version, subdir + "/" + file.Filename, true
+		},
+		check: func(pv PackageVersion) error {
+			if !strings.Contains(pv.Name, "/") {
+				return fmt.Errorf("conda packages need a channel, such as pkg:conda/conda-forge/%s", pv.Name)
+			}
+			return nil
+		},
+	},
+}
+
+// condaFileType returns the PURL type qualifier of a conda package file.
+func condaFileType(filename string) string {
+	switch {
+	case strings.HasSuffix(filename, ".conda"):
+		return "conda"
+	case strings.HasSuffix(filename, ".tar.bz2"):
+		return "tar.bz2"
+	default:
+		return ""
+	}
+}
+
+// selection returns the qualifiers of pv that select files, as
+// "key=value" pairs, for error messages.
+func (e fileEcosystem) selection(pv PackageVersion) string {
+	var pairs []string
+	for _, key := range e.qualifiers {
+		if value := pv.Qualifiers[key]; value != "" {
+			pairs = append(pairs, key+"="+value)
+		}
+	}
+	return strings.Join(pairs, " ")
+}
+
+// selects reports whether file matches every selecting qualifier of pv.
+func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool {
+	attrs := e.attrs(file)
+	for _, key := range e.qualifiers {
+		if want := pv.Qualifiers[key]; want != "" && attrs[key] != want {
+			return false
+		}
+	}
+	return true
+}
+
+// fileQualifiers keeps the qualifiers of a PURL that select files of its
+// ecosystem, or returns nil.
+func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]string {
+	e, ok := fileEcosystems[ecosystem]
+	if !ok {
+		return nil
+	}
+	var kept map[string]string
+	for key, value := range qualifiers {
+		if value != "" && slices.Contains(e.qualifiers, key) {
+			if kept == nil {
+				kept = map[string]string{}
+			}
+			kept[key] = value
+		}
+	}
+	return kept
+}
+
+// condaName returns a conda package name with its channel, taken from the
+// namespace or the channel qualifier. Without either it stays bare, and
+// mirroring refuses it.
+func condaName(namespace, name, channel string) string {
+	switch {
+	case namespace != "":
+		return namespace + "/" + name
+	case channel != "":
+		return channel + "/" + name
+	default:
+		return name
+	}
+}

internal/mirror/files_test.go

@@ -0,0 +1,38 @@
+package mirror
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestPURLSourceCondaChannelAndQualifiers(t *testing.T) {
+	versions := &fakeVersions{versions: []string{"1.0", "1.1"}}
+	source := &PURLSource{
+		PURLs: []string{
+			"pkg:conda/conda-forge/numpy@2.1.0?subdir=linux-64&build=py312h_0&type=conda",
+			"pkg:conda/numpy@2.1.0?channel=conda-forge&repository_url=https://example.com",
+			"pkg:conda/numpy@2.1.0",
+			"pkg:conda/numpy",
+			"pkg:conda/numpy?channel=bioconda&subdir=noarch",
+		},
+		Versions: versions,
+	}
+
+	got := enumerate(t, source)
+
+	want := []PackageVersion{
+		{Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0",
+			Qualifiers: map[string]string{"subdir": "linux-64", "build": "py312h_0", "type": "conda"}},
+		{Ecosystem: "conda", Name: "conda-forge/numpy", Version: "2.1.0"},
+		{Ecosystem: "conda", Name: "numpy", Version: "2.1.0"},
+		{Ecosystem: "conda", Name: "numpy"},
+		{Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.0", Qualifiers: map[string]string{"subdir": "noarch"}},
+		{Ecosystem: "conda", Name: "bioconda/numpy", Version: "1.1", Qualifiers: map[string]string{"subdir": "noarch"}},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("items = %+v\nwant %+v", got, want)
+	}
+	if !reflect.DeepEqual(versions.asked, []string{"conda/bioconda/numpy"}) {
+		t.Errorf("listed versions of %v, want only the package with a channel", versions.asked)
+	}
+}