From f5251d38344e3beb97e16cb5154b9165decec786 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:40:13 +0100 Subject: [PATCH 1/4] Mirror CRAN source packages proxy mirror pkg:cran/@ now fetches the source tarball from upstream.cran and caches it under the key the CRAN handler reads, ("cran", name, version, name_version.tar.gz), which serves both /src/contrib/{file} and /src/contrib/Archive/{name}/{file}. A current version comes from src/contrib and is checked against the MD5sum in src/contrib/PACKAGES.gz, read once per five minutes. Other versions come from src/contrib/Archive/{name}/, checked against the MD5sum in that directory's PACKAGES.rds, an R-serialized character matrix read by a small decoder. Without a PACKAGES.rds the archived tarball is fetched unchecked, as the handler fetches it. Listing versions dropped duplicates: the CRAN client lists each archived version twice. --- docs/configuration.md | 2 + internal/mirror/cran.go | 260 ++++++++++++++ internal/mirror/cran_rds.go | 332 ++++++++++++++++++ internal/mirror/cran_test.go | 61 ++++ internal/mirror/mirror.go | 6 + .../mirror/testdata/cran-archive-PACKAGES.rds | Bin 0 -> 370 bytes internal/mirror/versions.go | 12 +- internal/server/mirror_cran_test.go | 179 ++++++++++ 8 files changed, 849 insertions(+), 3 deletions(-) create mode 100644 internal/mirror/cran.go create mode 100644 internal/mirror/cran_rds.go create mode 100644 internal/mirror/cran_test.go create mode 100644 internal/mirror/testdata/cran-archive-PACKAGES.rds create mode 100644 internal/server/mirror_cran_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 832860c2..7cdbe64a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1625,6 +1625,8 @@ A Julia package is mirrored with the source tarball `upstream.julia` serves at ` A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. +A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions from `src/contrib/Archive/{name}/`, checked against the MD5 sum in that directory's `PACKAGES.rds`, or unchecked when the repository has none there. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/mirror/cran.go b/internal/mirror/cran.go new file mode 100644 index 00000000..b462ce60 --- /dev/null +++ b/internal/mirror/cran.go @@ -0,0 +1,260 @@ +package mirror + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "github.com/git-pkgs/registries" +) + +const ( + cranDefaultURL = "https://cloud.r-project.org" + // cranIndexTTL is how long a fetched src/contrib/PACKAGES is reused, so + // mirroring many packages, or every version of one, reads it once. + cranIndexTTL = 5 * time.Minute + // cranMaxIndexSize limits the decompressed PACKAGES and PACKAGES.rds. + // CRAN's PACKAGES is about 7 MB. + cranMaxIndexSize = 64 << 20 + // cranMaxLine limits a line of PACKAGES. + cranMaxLine = 1 << 20 +) + +var ( + cranNamePattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.]*$`) + cranVersionPattern = regexp.MustCompile(`^[0-9]+[0-9.-]*$`) +) + +// cranIndex caches the MD5 sums src/contrib/PACKAGES lists for the current +// source packages of a CRAN repository, by file name. +type cranIndex struct { + mu sync.Mutex + baseURL string + fetched time.Time + md5s map[string]string +} + +// mirrorCRAN mirrors the source package of a CRAN version under the key the +// CRAN handler caches it under, ("cran", name, version, name_version.tar.gz), +// which both /src/contrib/{file} and /src/contrib/Archive/{name}/{file} are +// served from. A current version is fetched from src/contrib and checked +// against the MD5 sum src/contrib/PACKAGES lists; any other version from +// src/contrib/Archive, checked against the MD5 sum in that directory's +// PACKAGES.rds, or unchecked where the repository has none, as the handler +// fetches it. +func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + if !cranNamePattern.MatchString(pv.Name) { + fail(fmt.Errorf("invalid CRAN package name %q", pv.Name)) + return + } + if !cranVersionPattern.MatchString(pv.Version) { + fail(fmt.Errorf("invalid CRAN version %q", pv.Version)) + return + } + baseURL, client, err := m.cranRepository(pv.Name) + if err != nil { + fail(err) + return + } + filename := pv.Name + "_" + pv.Version + ".tar.gz" + + current, err := m.cranCurrentMD5s(ctx, baseURL, client) + if err != nil { + fail(fmt.Errorf("reading src/contrib/PACKAGES: %w", err)) + return + } + downloadURL := baseURL + "/src/contrib/" + filename + sum, ok := current[filename] + if !ok { + archive := baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" + downloadURL = archive + filename + if sum, err = cranArchivedMD5(ctx, client, archive+"PACKAGES.rds", pv.Version); err != nil { + fail(fmt.Errorf("reading the archive's PACKAGES.rds: %w", err)) + return + } + } + var declared string + if sum != "" { + declared = "md5-" + sum + } + result, err := m.proxy.GetOrFetchArtifactFromURLWithIntegrity(ctx, pv.Ecosystem, pv.Name, pv.Version, filename, downloadURL, declared) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) +} + +// cranRepository returns the repository to mirror a CRAN package from and +// the client to read its indexes with: those Versions lists versions with, +// which come from upstream.cran, or the public CRAN. +func (m *Mirror) cranRepository(name string) (string, *registries.Client, error) { + r, _ := m.Versions.(RegistryVersions) + baseURL := cranDefaultURL + if r.URL != nil { + configured, err := r.URL("cran", name) + if err != nil { + return "", nil, err + } + if configured != "" { + baseURL = strings.TrimSuffix(configured, "/") + } + } + client := r.Client + if client == nil { + client = registries.DefaultClient() + } + return baseURL, client, nil +} + +// cranCurrentMD5s returns the MD5 sums of the current source packages of the +// repository at baseURL, reading src/contrib/PACKAGES.gz unless it was read +// within cranIndexTTL. +func (m *Mirror) cranCurrentMD5s(ctx context.Context, baseURL string, client *registries.Client) (map[string]string, error) { + m.cran.mu.Lock() + defer m.cran.mu.Unlock() + if m.cran.md5s != nil && m.cran.baseURL == baseURL && time.Since(m.cran.fetched) < cranIndexTTL { + return m.cran.md5s, nil + } + body, err := client.GetBody(ctx, baseURL+"/src/contrib/PACKAGES.gz") + if err != nil { + return nil, err + } + data, err := gunzipLimited(body) + if err != nil { + return nil, err + } + sums, err := parseCRANPackages(data) + if err != nil { + return nil, err + } + m.cran.md5s = sums + m.cran.baseURL = baseURL + m.cran.fetched = time.Now() + return m.cran.md5s, nil +} + +// cranArchivedMD5 returns the MD5 sum the PACKAGES.rds at rdsURL lists for +// version, or "" when the repository has no PACKAGES.rds there or it doesn't +// list version. +func cranArchivedMD5(ctx context.Context, client *registries.Client, rdsURL, version string) (string, error) { + body, err := client.GetBody(ctx, rdsURL) + var httpErr *registries.HTTPError + if errors.As(err, &httpErr) && httpErr.IsNotFound() { + return "", nil + } + if err != nil { + return "", err + } + data, err := gunzipLimited(body) + if err != nil { + return "", err + } + sums, err := parseCRANArchiveRDS(data) + if err != nil { + return "", err + } + return sums[version], nil +} + +// gunzipLimited decompresses data, or returns it as is when it isn't +// gzipped, as R reads both. +func gunzipLimited(data []byte) ([]byte, error) { + if !bytes.HasPrefix(data, []byte{0x1f, 0x8b}) { + return data, nil + } + zr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer func() { _ = zr.Close() }() + out, err := io.ReadAll(io.LimitReader(zr, cranMaxIndexSize+1)) + if err != nil { + return nil, err + } + if len(out) > cranMaxIndexSize { + return nil, fmt.Errorf("index is larger than %d bytes", cranMaxIndexSize) + } + return out, nil +} + +// parseCRANPackages returns the MD5sum of each source package a PACKAGES +// file lists, by file name. Entries without one are listed with "". +func parseCRANPackages(data []byte) (map[string]string, error) { + sums := map[string]string{} + var name, version, sum string + flush := func() { + if name != "" && version != "" { + sums[name+"_"+version+".tar.gz"] = sum + } + name, version, sum = "", "", "" + } + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(nil, cranMaxLine) + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + flush() + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") { + continue + } + switch value = strings.TrimSpace(value); key { + case "Package": + name = value + case "Version": + version = value + case "MD5sum": + sum = value + } + } + flush() + return sums, scanner.Err() +} + +// parseCRANArchiveRDS returns the MD5sum of each version in a CRAN archive +// directory's PACKAGES.rds, a character matrix with a column per field, +// saved by R. +func parseCRANArchiveRDS(data []byte) (map[string]string, error) { + values, nrow, columns, err := readRDSStringMatrix(data) + if err != nil { + return nil, err + } + versionCol, sumCol := -1, -1 + for i, column := range columns { + switch column { + case "Version": + versionCol = i + case "MD5sum": + sumCol = i + } + } + if versionCol < 0 || sumCol < 0 { + return nil, errors.New("PACKAGES.rds has no Version or MD5sum column") + } + sums := make(map[string]string, nrow) + for row := range nrow { + version := values[versionCol*nrow+row] + if version != nil { + sum := values[sumCol*nrow+row] + if sum != nil { + sums[*version] = *sum + } else { + sums[*version] = "" + } + } + } + return sums, nil +} diff --git a/internal/mirror/cran_rds.go b/internal/mirror/cran_rds.go new file mode 100644 index 00000000..016c8b26 --- /dev/null +++ b/internal/mirror/cran_rds.go @@ -0,0 +1,332 @@ +package mirror + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" +) + +// R serialization types and flags, from R's serialize.c. Only those a +// character matrix and its dimnames are written with are read. +const ( + rdsSymSXP = 1 + rdsListSXP = 2 + rdsCharSXP = 9 + rdsLglSXP = 10 + rdsIntSXP = 13 + rdsStrSXP = 16 + rdsVecSXP = 19 + rdsNilValue = 254 + rdsRefSXP = 255 + + rdsTypeMask = 0xff + rdsRefShift = 8 + rdsHasAttr = 1 << 9 + rdsHasTag = 1 << 10 + + rdsMagic = "X\n" + rdsMatrix = 2 // dimensions of a matrix + rdsFormatV2 = 2 + rdsFormatV3 = 3 + + // rdsMaxDepth limits nesting; a matrix needs four levels. + rdsMaxDepth = 16 +) + +// rdsSymbol is a symbol, the tag of an attribute. +type rdsSymbol string + +// rdsAttr is one attribute of a vector. +type rdsAttr struct { + tag rdsSymbol + value any +} + +// rdsVector is an R vector: []*string (nil for NA) for a character vector, +// []int32 for an integer or logical one, []any for a list. +type rdsVector struct { + values any + attrs []rdsAttr +} + +// rdsReader reads R's XDR serialization format, as saveRDS writes it +// uncompressed. +type rdsReader struct { + r *bytes.Reader + refs []any +} + +// readRDSStringMatrix reads an R character matrix saved by saveRDS, +// uncompressed, returning its values in column-major order, nil for NA, its +// number of rows and its column names. +func readRDSStringMatrix(data []byte) (values []*string, nrow int, columns []string, err error) { + rd := &rdsReader{r: bytes.NewReader(data)} + if err := rd.header(); err != nil { + return nil, 0, nil, err + } + item, err := rd.item(0) + if err != nil { + return nil, 0, nil, fmt.Errorf("reading rds: %w", err) + } + vec, ok := item.(*rdsVector) + if !ok { + return nil, 0, nil, errors.New("rds is not a vector") + } + values, ok = vec.values.([]*string) + if !ok { + return nil, 0, nil, errors.New("rds is not a character vector") + } + var dim []int32 + var dimnames []any + for _, attr := range vec.attrs { + v, _ := attr.value.(*rdsVector) + if v == nil { + continue + } + switch attr.tag { + case "dim": + dim, _ = v.values.([]int32) + case "dimnames": + dimnames, _ = v.values.([]any) + } + } + if len(dim) != rdsMatrix || dim[0] < 0 || dim[1] < 0 || int(dim[0])*int(dim[1]) != len(values) { + return nil, 0, nil, errors.New("rds is not a matrix") + } + if len(dimnames) != rdsMatrix { + return nil, 0, nil, errors.New("rds matrix has no column names") + } + names, _ := dimnames[1].(*rdsVector) + if names == nil { + return nil, 0, nil, errors.New("rds matrix has no column names") + } + colnames, _ := names.values.([]*string) + if len(colnames) != int(dim[1]) { + return nil, 0, nil, errors.New("rds matrix has no column names") + } + columns = make([]string, len(colnames)) + for i, name := range colnames { + if name != nil { + columns[i] = *name + } + } + return values, int(dim[0]), columns, nil +} + +func (rd *rdsReader) header() error { + magic := make([]byte, len(rdsMagic)) + if _, err := io.ReadFull(rd.r, magic); err != nil || string(magic) != rdsMagic { + return errors.New("rds is not in R's XDR format") + } + version, err := rd.int() + if err != nil { + return err + } + // The R version that wrote it and the oldest that reads it. + for range 2 { + if _, err := rd.int(); err != nil { + return err + } + } + switch version { + case rdsFormatV2: + case rdsFormatV3: + // The native encoding, unused: CHARSXPs are UTF-8 or ASCII. + n, err := rd.int() + if err != nil { + return err + } + if n < 0 || int64(n) > int64(rd.r.Len()) { + return errors.New("rds header is truncated") + } + if _, err := rd.r.Seek(int64(n), io.SeekCurrent); err != nil { + return err + } + default: + return fmt.Errorf("unsupported rds format version %d", version) + } + return nil +} + +func (rd *rdsReader) int() (int32, error) { + var v int32 + if err := binary.Read(rd.r, binary.BigEndian, &v); err != nil { + return 0, fmt.Errorf("rds is truncated: %w", err) + } + return v, nil +} + +// length reads a vector length, which needs no more items than are left. +func (rd *rdsReader) length() (int, error) { + n, err := rd.int() + if err != nil { + return 0, err + } + if n < 0 { + return 0, errors.New("rds has a long vector") + } + // Each element takes at least four bytes. + if int64(n)*4 > int64(rd.r.Len()) { + return 0, errors.New("rds is truncated") + } + return int(n), nil +} + +func (rd *rdsReader) item(depth int) (any, error) { + if depth > rdsMaxDepth { + return nil, errors.New("rds is nested too deeply") + } + flags, err := rd.int() + if err != nil { + return nil, err + } + kind := flags & rdsTypeMask + hasAttr := flags&rdsHasAttr != 0 + switch kind { + case rdsNilValue: + return nil, nil + case rdsRefSXP: + index := int(flags >> rdsRefShift) + if index == 0 { + i, err := rd.int() + if err != nil { + return nil, err + } + index = int(i) + } + if index < 1 || index > len(rd.refs) { + return nil, fmt.Errorf("rds refers to unknown item %d", index) + } + return rd.refs[index-1], nil + case rdsSymSXP: + name, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + s, ok := name.(*string) + if !ok || s == nil { + return nil, errors.New("rds symbol has no name") + } + sym := rdsSymbol(*s) + rd.refs = append(rd.refs, sym) + return sym, nil + case rdsCharSXP: + return rd.charsxp() + case rdsListSXP: + return rd.pairlist(flags, depth) + case rdsStrSXP, rdsIntSXP, rdsLglSXP, rdsVecSXP: + vec := &rdsVector{} + if vec.values, err = rd.vector(kind, depth); err != nil { + return nil, err + } + if hasAttr { + attrs, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + vec.attrs, _ = attrs.([]rdsAttr) + } + return vec, nil + default: + return nil, fmt.Errorf("unsupported rds item type %d", kind) + } +} + +func (rd *rdsReader) charsxp() (*string, error) { + n, err := rd.int() + if err != nil { + return nil, err + } + if n == -1 { + return nil, nil // NA_character_ + } + if n < 0 || int64(n) > int64(rd.r.Len()) { + return nil, errors.New("rds is truncated") + } + b := make([]byte, n) + if _, err := io.ReadFull(rd.r, b); err != nil { + return nil, err + } + s := string(b) + return &s, nil +} + +func (rd *rdsReader) vector(kind int32, depth int) (any, error) { + n, err := rd.length() + if err != nil { + return nil, err + } + switch kind { + case rdsStrSXP: + values := make([]*string, n) + for i := range values { + flags, err := rd.int() + if err != nil { + return nil, err + } + if flags&rdsTypeMask != rdsCharSXP { + return nil, errors.New("rds character vector holds a non-string") + } + if values[i], err = rd.charsxp(); err != nil { + return nil, err + } + } + return values, nil + case rdsIntSXP, rdsLglSXP: + values := make([]int32, n) + for i := range values { + if values[i], err = rd.int(); err != nil { + return nil, err + } + } + return values, nil + default: + values := make([]any, n) + for i := range values { + if values[i], err = rd.item(depth + 1); err != nil { + return nil, err + } + } + return values, nil + } +} + +// pairlist reads an attribute pairlist, its first cell's flags already +// read, as its tags and values. +func (rd *rdsReader) pairlist(flags int32, depth int) ([]rdsAttr, error) { + var attrs []rdsAttr + for { + if flags&rdsHasAttr != 0 { + if _, err := rd.item(depth + 1); err != nil { + return nil, err + } + } + var attr rdsAttr + if flags&rdsHasTag != 0 { + tag, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + attr.tag, _ = tag.(rdsSymbol) + } + value, err := rd.item(depth + 1) + if err != nil { + return nil, err + } + attr.value = value + attrs = append(attrs, attr) + // The rest of the list follows as the next cell, or ends. + if flags, err = rd.int(); err != nil { + return nil, err + } + switch flags & rdsTypeMask { + case rdsNilValue: + return attrs, nil + case rdsListSXP: + default: + return nil, errors.New("rds pairlist is malformed") + } + } +} diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go new file mode 100644 index 00000000..451af00a --- /dev/null +++ b/internal/mirror/cran_test.go @@ -0,0 +1,61 @@ +package mirror + +import ( + "maps" + "os" + "testing" +) + +// testdata/cran-archive-PACKAGES.rds was written by R 4.4.1 with saveRDS, +// shaped like the PACKAGES.rds in CRAN's src/contrib/Archive/{name}: a +// character matrix of fields, NA where a version has none. +func TestParseCRANArchiveRDS(t *testing.T) { + data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") + if err != nil { + t.Fatal(err) + } + data, err = gunzipLimited(data) + if err != nil { + t.Fatal(err) + } + + sums, err := parseCRANArchiveRDS(data) + if err != nil { + t.Fatal(err) + } + + want := map[string]string{ + "0.9": "b7f01da919c10e36d6777fc337f48dfa", + "1.0": "e51e3cbb209131a84f3ff5938dd93602", + } + if !maps.Equal(sums, want) { + t.Errorf("sums = %v, want %v", sums, want) + } + + for i := range data { + if _, err := parseCRANArchiveRDS(data[:i]); err == nil { + t.Fatalf("truncated to %d bytes: no error", i) + } + } +} + +func TestParseCRANPackages(t *testing.T) { + data := []byte(`Package: A3 +Version: 1.0.0 +Depends: R (>= 2.15.0), xtable, + pbapply +MD5sum: 027ebdd8affce8f0effaecfcd5f5ade2 +NeedsCompilation: no + +Package: abc +Version: 2.2.1 +NeedsCompilation: no +`) + want := map[string]string{ + "A3_1.0.0.tar.gz": "027ebdd8affce8f0effaecfcd5f5ade2", + "abc_2.2.1.tar.gz": "", + } + if got, err := parseCRANPackages(data); err != nil || !maps.Equal(got, want) { + t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 9b55a8c8..6f74e22d 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -43,6 +43,8 @@ type Mirror struct { // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string + + cran cranIndex } // New creates a new Mirror with the given dependencies. @@ -255,6 +257,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorSwift(ctx, pv, tracker) return } + if pv.Ecosystem == "cran" { + m.mirrorCRAN(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/testdata/cran-archive-PACKAGES.rds b/internal/mirror/testdata/cran-archive-PACKAGES.rds new file mode 100644 index 0000000000000000000000000000000000000000..9cc5a61abb09e00cde420421b11a10752b422fba GIT binary patch literal 370 zcmV-&0ge72iwFP!000001Fcb8OT#b}&bH27KsFR#rH_g@+GShY2L;&$2D@RW_*Sws zsbOncnhxBbS2NcXQSm_x9KP?%`OZ1HJkO|BddsrtQsaeJe~R`>Foy`^(&pS4bdm#{&40F zf64sMOnqfh!}O73I@mM_?D;**vM7eoq8;0(Ncl+DBoMDwZNo94iR=!Aly)6x`@REv zMmzHrdGv_^tteSRJ*v+_3CH_n7L6~KsrkMjVs8e!?)}LJe= 4.0)\nMD5sum: "+hex.EncodeToString(sum[:])+"\nNeedsCompilation: no\n") + _ = zw.Close() + + u := &cranUpstream{ + files: map[string]string{ + cranIndexPath: index.String(), + cranArchiveRDS: string(rds), + cranCurrentPath: current, + cranArchivedPath: "example 1.0 source", + "/src/contrib/Archive/example/example_0.9.tar.gz": "example 0.9 source", + "/web/packages/example/DESCRIPTION": "Package: example\nVersion: 1.1\n", + "/src/contrib/Archive/example/": `example_0.9.tar.gz +example_1.0.tar.gz`, + }, + hits: map[string]int{}, + } + for path, body := range tampered { + u.files[path] = body + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *cranUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func cranMirrorConfig(upstream *cranUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.CRAN = upstream.URL + return cfg +} + +// getCRAN requests path from a CRAN handler for upstream. +func getCRAN(proxy *handler.Proxy, upstream *cranUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + cran := handler.NewCRANHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL) + cran.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +func TestMirrorCRANSourcePackagesAreServedFromCache(t *testing.T) { + upstream := newCRANUpstream(t, nil) + + progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1", "pkg:cran/example@1.0") + + if progress.Completed != 2 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{cranCurrentPath, cranArchivedPath} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // R gets both from the cache once the upstream is down, the current + // version also from Archive/, where CRAN moves it once a newer one is + // published. + upstream.Close() + for path, want := range map[string]string{ + cranCurrentPath: "example 1.1 source", + "/src/contrib/Archive/example/example_1.1.tar.gz": "example 1.1 source", + cranArchivedPath: "example 1.0 source", + } { + if rec := getCRAN(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorCRANRefusesMD5Mismatch(t *testing.T) { + upstream := newCRANUpstream(t, map[string]string{ + cranCurrentPath: "tampered 1.1", + cranArchivedPath: "tampered 1.0", + }) + + progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1", "pkg:cran/example@1.0") + + if progress.Failed != 2 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want both refused", progress.Completed, progress.Failed, progress.Errors) + } + upstream.Close() + for _, path := range []string{cranCurrentPath, cranArchivedPath} { + if rec := getCRAN(proxy, upstream, path); rec.Code == http.StatusOK { + t.Errorf("%s was cached although its MD5 doesn't match", path) + } + } +} + +func TestMirrorCRANArchiveWithoutPackagesRDS(t *testing.T) { + // A repository without the archive's PACKAGES.rds publishes no checksum + // for archived versions; they are fetched unchecked, as R gets them + // through the handler. + upstream := newCRANUpstream(t, nil) + delete(upstream.files, cranArchiveRDS) + + progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.0") + + if progress.Completed != 1 || upstream.hitsFor(cranArchivedPath) != 1 { + t.Errorf("completed = %d, hits = %d, errors = %v", progress.Completed, upstream.hitsFor(cranArchivedPath), progress.Errors) + } +} + +func TestMirrorCRANAllVersions(t *testing.T) { + upstream := newCRANUpstream(t, nil) + + progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example") + + // The archive listing names each file twice, in the link and its text. + if progress.Total != 3 || progress.Completed != 3 || progress.Failed != 0 { + t.Fatalf("total = %d, completed = %d, failed = %d, errors = %v; want 0.9, 1.0 and 1.1 once each", progress.Total, progress.Completed, progress.Failed, progress.Errors) + } + if got := upstream.hitsFor(cranIndexPath); got != 1 { + t.Errorf("PACKAGES.gz fetched %d times, want once", got) + } +} From 3fd79ccb3666864d6cbf4639eaed82e0b7b8cf97 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 19:44:07 +0100 Subject: [PATCH 2/4] Fuzz the PACKAGES.rds decoder It reads files from the network, so malformed input must fail with an error. Seeded with the real PACKAGES.rds; 3.7M executions found nothing. --- internal/mirror/cran_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go index 451af00a..2dc130f2 100644 --- a/internal/mirror/cran_test.go +++ b/internal/mirror/cran_test.go @@ -59,3 +59,25 @@ NeedsCompilation: no t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) } } + +// FuzzReadRDSStringMatrix feeds the decoder arbitrary bytes, seeded with a +// real PACKAGES.rds: it reads files from the network, so it must reject +// anything malformed with an error rather than panic or allocate without +// bound. +func FuzzReadRDSStringMatrix(f *testing.F) { + data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") + if err != nil { + f.Fatal(err) + } + if data, err = gunzipLimited(data); err != nil { + f.Fatal(err) + } + f.Add(data) + f.Add(data[:len(data)/2]) + f.Fuzz(func(t *testing.T, b []byte) { + values, nrow, columns, err := readRDSStringMatrix(b) + if err == nil && nrow > 0 && len(values) != nrow*len(columns) { + t.Errorf("%d values for %d rows and %d columns", len(values), nrow, len(columns)) + } + }) +} From 842fa21c0a406436e2d33adb6e244d8b12df46fa Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:42:08 +0100 Subject: [PATCH 3/4] Fetch archived CRAN versions unchecked and drop the version dedup PACKAGES.rds comes from the same host as the tarball, so checking against it doesn't justify a decoder for R serialization. The duplicate archived versions are tracked in git-pkgs/registries#119. --- internal/mirror/cran.go | 81 +---- internal/mirror/cran_rds.go | 332 ------------------ internal/mirror/cran_test.go | 56 --- .../mirror/testdata/cran-archive-PACKAGES.rds | Bin 370 -> 0 bytes internal/mirror/versions.go | 12 +- internal/server/mirror_cran_test.go | 49 +-- 6 files changed, 19 insertions(+), 511 deletions(-) delete mode 100644 internal/mirror/cran_rds.go delete mode 100644 internal/mirror/testdata/cran-archive-PACKAGES.rds diff --git a/internal/mirror/cran.go b/internal/mirror/cran.go index b462ce60..02d26bdb 100644 --- a/internal/mirror/cran.go +++ b/internal/mirror/cran.go @@ -5,7 +5,6 @@ import ( "bytes" "compress/gzip" "context" - "errors" "fmt" "io" "net/url" @@ -22,7 +21,7 @@ const ( // cranIndexTTL is how long a fetched src/contrib/PACKAGES is reused, so // mirroring many packages, or every version of one, reads it once. cranIndexTTL = 5 * time.Minute - // cranMaxIndexSize limits the decompressed PACKAGES and PACKAGES.rds. + // cranMaxIndexSize limits the decompressed PACKAGES. // CRAN's PACKAGES is about 7 MB. cranMaxIndexSize = 64 << 20 // cranMaxLine limits a line of PACKAGES. @@ -43,14 +42,9 @@ type cranIndex struct { md5s map[string]string } -// mirrorCRAN mirrors the source package of a CRAN version under the key the -// CRAN handler caches it under, ("cran", name, version, name_version.tar.gz), -// which both /src/contrib/{file} and /src/contrib/Archive/{name}/{file} are -// served from. A current version is fetched from src/contrib and checked -// against the MD5 sum src/contrib/PACKAGES lists; any other version from -// src/contrib/Archive, checked against the MD5 sum in that directory's -// PACKAGES.rds, or unchecked where the repository has none, as the handler -// fetches it. +// mirrorCRAN mirrors a CRAN source package under ("cran", name, version, +// name_version.tar.gz), the one entry the handler serves for both +// src/contrib and src/contrib/Archive. func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } if !cranNamePattern.MatchString(pv.Name) { @@ -76,12 +70,7 @@ func (m *Mirror) mirrorCRAN(ctx context.Context, pv PackageVersion, tracker *pro downloadURL := baseURL + "/src/contrib/" + filename sum, ok := current[filename] if !ok { - archive := baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" - downloadURL = archive + filename - if sum, err = cranArchivedMD5(ctx, client, archive+"PACKAGES.rds", pv.Version); err != nil { - fail(fmt.Errorf("reading the archive's PACKAGES.rds: %w", err)) - return - } + downloadURL = baseURL + "/src/contrib/Archive/" + url.PathEscape(pv.Name) + "/" + filename } var declared string if sum != "" { @@ -144,31 +133,8 @@ func (m *Mirror) cranCurrentMD5s(ctx context.Context, baseURL string, client *re return m.cran.md5s, nil } -// cranArchivedMD5 returns the MD5 sum the PACKAGES.rds at rdsURL lists for -// version, or "" when the repository has no PACKAGES.rds there or it doesn't -// list version. -func cranArchivedMD5(ctx context.Context, client *registries.Client, rdsURL, version string) (string, error) { - body, err := client.GetBody(ctx, rdsURL) - var httpErr *registries.HTTPError - if errors.As(err, &httpErr) && httpErr.IsNotFound() { - return "", nil - } - if err != nil { - return "", err - } - data, err := gunzipLimited(body) - if err != nil { - return "", err - } - sums, err := parseCRANArchiveRDS(data) - if err != nil { - return "", err - } - return sums[version], nil -} - // gunzipLimited decompresses data, or returns it as is when it isn't -// gzipped, as R reads both. +// gzipped. func gunzipLimited(data []byte) ([]byte, error) { if !bytes.HasPrefix(data, []byte{0x1f, 0x8b}) { return data, nil @@ -223,38 +189,3 @@ func parseCRANPackages(data []byte) (map[string]string, error) { flush() return sums, scanner.Err() } - -// parseCRANArchiveRDS returns the MD5sum of each version in a CRAN archive -// directory's PACKAGES.rds, a character matrix with a column per field, -// saved by R. -func parseCRANArchiveRDS(data []byte) (map[string]string, error) { - values, nrow, columns, err := readRDSStringMatrix(data) - if err != nil { - return nil, err - } - versionCol, sumCol := -1, -1 - for i, column := range columns { - switch column { - case "Version": - versionCol = i - case "MD5sum": - sumCol = i - } - } - if versionCol < 0 || sumCol < 0 { - return nil, errors.New("PACKAGES.rds has no Version or MD5sum column") - } - sums := make(map[string]string, nrow) - for row := range nrow { - version := values[versionCol*nrow+row] - if version != nil { - sum := values[sumCol*nrow+row] - if sum != nil { - sums[*version] = *sum - } else { - sums[*version] = "" - } - } - } - return sums, nil -} diff --git a/internal/mirror/cran_rds.go b/internal/mirror/cran_rds.go deleted file mode 100644 index 016c8b26..00000000 --- a/internal/mirror/cran_rds.go +++ /dev/null @@ -1,332 +0,0 @@ -package mirror - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" -) - -// R serialization types and flags, from R's serialize.c. Only those a -// character matrix and its dimnames are written with are read. -const ( - rdsSymSXP = 1 - rdsListSXP = 2 - rdsCharSXP = 9 - rdsLglSXP = 10 - rdsIntSXP = 13 - rdsStrSXP = 16 - rdsVecSXP = 19 - rdsNilValue = 254 - rdsRefSXP = 255 - - rdsTypeMask = 0xff - rdsRefShift = 8 - rdsHasAttr = 1 << 9 - rdsHasTag = 1 << 10 - - rdsMagic = "X\n" - rdsMatrix = 2 // dimensions of a matrix - rdsFormatV2 = 2 - rdsFormatV3 = 3 - - // rdsMaxDepth limits nesting; a matrix needs four levels. - rdsMaxDepth = 16 -) - -// rdsSymbol is a symbol, the tag of an attribute. -type rdsSymbol string - -// rdsAttr is one attribute of a vector. -type rdsAttr struct { - tag rdsSymbol - value any -} - -// rdsVector is an R vector: []*string (nil for NA) for a character vector, -// []int32 for an integer or logical one, []any for a list. -type rdsVector struct { - values any - attrs []rdsAttr -} - -// rdsReader reads R's XDR serialization format, as saveRDS writes it -// uncompressed. -type rdsReader struct { - r *bytes.Reader - refs []any -} - -// readRDSStringMatrix reads an R character matrix saved by saveRDS, -// uncompressed, returning its values in column-major order, nil for NA, its -// number of rows and its column names. -func readRDSStringMatrix(data []byte) (values []*string, nrow int, columns []string, err error) { - rd := &rdsReader{r: bytes.NewReader(data)} - if err := rd.header(); err != nil { - return nil, 0, nil, err - } - item, err := rd.item(0) - if err != nil { - return nil, 0, nil, fmt.Errorf("reading rds: %w", err) - } - vec, ok := item.(*rdsVector) - if !ok { - return nil, 0, nil, errors.New("rds is not a vector") - } - values, ok = vec.values.([]*string) - if !ok { - return nil, 0, nil, errors.New("rds is not a character vector") - } - var dim []int32 - var dimnames []any - for _, attr := range vec.attrs { - v, _ := attr.value.(*rdsVector) - if v == nil { - continue - } - switch attr.tag { - case "dim": - dim, _ = v.values.([]int32) - case "dimnames": - dimnames, _ = v.values.([]any) - } - } - if len(dim) != rdsMatrix || dim[0] < 0 || dim[1] < 0 || int(dim[0])*int(dim[1]) != len(values) { - return nil, 0, nil, errors.New("rds is not a matrix") - } - if len(dimnames) != rdsMatrix { - return nil, 0, nil, errors.New("rds matrix has no column names") - } - names, _ := dimnames[1].(*rdsVector) - if names == nil { - return nil, 0, nil, errors.New("rds matrix has no column names") - } - colnames, _ := names.values.([]*string) - if len(colnames) != int(dim[1]) { - return nil, 0, nil, errors.New("rds matrix has no column names") - } - columns = make([]string, len(colnames)) - for i, name := range colnames { - if name != nil { - columns[i] = *name - } - } - return values, int(dim[0]), columns, nil -} - -func (rd *rdsReader) header() error { - magic := make([]byte, len(rdsMagic)) - if _, err := io.ReadFull(rd.r, magic); err != nil || string(magic) != rdsMagic { - return errors.New("rds is not in R's XDR format") - } - version, err := rd.int() - if err != nil { - return err - } - // The R version that wrote it and the oldest that reads it. - for range 2 { - if _, err := rd.int(); err != nil { - return err - } - } - switch version { - case rdsFormatV2: - case rdsFormatV3: - // The native encoding, unused: CHARSXPs are UTF-8 or ASCII. - n, err := rd.int() - if err != nil { - return err - } - if n < 0 || int64(n) > int64(rd.r.Len()) { - return errors.New("rds header is truncated") - } - if _, err := rd.r.Seek(int64(n), io.SeekCurrent); err != nil { - return err - } - default: - return fmt.Errorf("unsupported rds format version %d", version) - } - return nil -} - -func (rd *rdsReader) int() (int32, error) { - var v int32 - if err := binary.Read(rd.r, binary.BigEndian, &v); err != nil { - return 0, fmt.Errorf("rds is truncated: %w", err) - } - return v, nil -} - -// length reads a vector length, which needs no more items than are left. -func (rd *rdsReader) length() (int, error) { - n, err := rd.int() - if err != nil { - return 0, err - } - if n < 0 { - return 0, errors.New("rds has a long vector") - } - // Each element takes at least four bytes. - if int64(n)*4 > int64(rd.r.Len()) { - return 0, errors.New("rds is truncated") - } - return int(n), nil -} - -func (rd *rdsReader) item(depth int) (any, error) { - if depth > rdsMaxDepth { - return nil, errors.New("rds is nested too deeply") - } - flags, err := rd.int() - if err != nil { - return nil, err - } - kind := flags & rdsTypeMask - hasAttr := flags&rdsHasAttr != 0 - switch kind { - case rdsNilValue: - return nil, nil - case rdsRefSXP: - index := int(flags >> rdsRefShift) - if index == 0 { - i, err := rd.int() - if err != nil { - return nil, err - } - index = int(i) - } - if index < 1 || index > len(rd.refs) { - return nil, fmt.Errorf("rds refers to unknown item %d", index) - } - return rd.refs[index-1], nil - case rdsSymSXP: - name, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - s, ok := name.(*string) - if !ok || s == nil { - return nil, errors.New("rds symbol has no name") - } - sym := rdsSymbol(*s) - rd.refs = append(rd.refs, sym) - return sym, nil - case rdsCharSXP: - return rd.charsxp() - case rdsListSXP: - return rd.pairlist(flags, depth) - case rdsStrSXP, rdsIntSXP, rdsLglSXP, rdsVecSXP: - vec := &rdsVector{} - if vec.values, err = rd.vector(kind, depth); err != nil { - return nil, err - } - if hasAttr { - attrs, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - vec.attrs, _ = attrs.([]rdsAttr) - } - return vec, nil - default: - return nil, fmt.Errorf("unsupported rds item type %d", kind) - } -} - -func (rd *rdsReader) charsxp() (*string, error) { - n, err := rd.int() - if err != nil { - return nil, err - } - if n == -1 { - return nil, nil // NA_character_ - } - if n < 0 || int64(n) > int64(rd.r.Len()) { - return nil, errors.New("rds is truncated") - } - b := make([]byte, n) - if _, err := io.ReadFull(rd.r, b); err != nil { - return nil, err - } - s := string(b) - return &s, nil -} - -func (rd *rdsReader) vector(kind int32, depth int) (any, error) { - n, err := rd.length() - if err != nil { - return nil, err - } - switch kind { - case rdsStrSXP: - values := make([]*string, n) - for i := range values { - flags, err := rd.int() - if err != nil { - return nil, err - } - if flags&rdsTypeMask != rdsCharSXP { - return nil, errors.New("rds character vector holds a non-string") - } - if values[i], err = rd.charsxp(); err != nil { - return nil, err - } - } - return values, nil - case rdsIntSXP, rdsLglSXP: - values := make([]int32, n) - for i := range values { - if values[i], err = rd.int(); err != nil { - return nil, err - } - } - return values, nil - default: - values := make([]any, n) - for i := range values { - if values[i], err = rd.item(depth + 1); err != nil { - return nil, err - } - } - return values, nil - } -} - -// pairlist reads an attribute pairlist, its first cell's flags already -// read, as its tags and values. -func (rd *rdsReader) pairlist(flags int32, depth int) ([]rdsAttr, error) { - var attrs []rdsAttr - for { - if flags&rdsHasAttr != 0 { - if _, err := rd.item(depth + 1); err != nil { - return nil, err - } - } - var attr rdsAttr - if flags&rdsHasTag != 0 { - tag, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - attr.tag, _ = tag.(rdsSymbol) - } - value, err := rd.item(depth + 1) - if err != nil { - return nil, err - } - attr.value = value - attrs = append(attrs, attr) - // The rest of the list follows as the next cell, or ends. - if flags, err = rd.int(); err != nil { - return nil, err - } - switch flags & rdsTypeMask { - case rdsNilValue: - return attrs, nil - case rdsListSXP: - default: - return nil, errors.New("rds pairlist is malformed") - } - } -} diff --git a/internal/mirror/cran_test.go b/internal/mirror/cran_test.go index 2dc130f2..3eae797b 100644 --- a/internal/mirror/cran_test.go +++ b/internal/mirror/cran_test.go @@ -2,43 +2,9 @@ package mirror import ( "maps" - "os" "testing" ) -// testdata/cran-archive-PACKAGES.rds was written by R 4.4.1 with saveRDS, -// shaped like the PACKAGES.rds in CRAN's src/contrib/Archive/{name}: a -// character matrix of fields, NA where a version has none. -func TestParseCRANArchiveRDS(t *testing.T) { - data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") - if err != nil { - t.Fatal(err) - } - data, err = gunzipLimited(data) - if err != nil { - t.Fatal(err) - } - - sums, err := parseCRANArchiveRDS(data) - if err != nil { - t.Fatal(err) - } - - want := map[string]string{ - "0.9": "b7f01da919c10e36d6777fc337f48dfa", - "1.0": "e51e3cbb209131a84f3ff5938dd93602", - } - if !maps.Equal(sums, want) { - t.Errorf("sums = %v, want %v", sums, want) - } - - for i := range data { - if _, err := parseCRANArchiveRDS(data[:i]); err == nil { - t.Fatalf("truncated to %d bytes: no error", i) - } - } -} - func TestParseCRANPackages(t *testing.T) { data := []byte(`Package: A3 Version: 1.0.0 @@ -59,25 +25,3 @@ NeedsCompilation: no t.Errorf("parseCRANPackages = %v, %v, want %v", got, err, want) } } - -// FuzzReadRDSStringMatrix feeds the decoder arbitrary bytes, seeded with a -// real PACKAGES.rds: it reads files from the network, so it must reject -// anything malformed with an error rather than panic or allocate without -// bound. -func FuzzReadRDSStringMatrix(f *testing.F) { - data, err := os.ReadFile("testdata/cran-archive-PACKAGES.rds") - if err != nil { - f.Fatal(err) - } - if data, err = gunzipLimited(data); err != nil { - f.Fatal(err) - } - f.Add(data) - f.Add(data[:len(data)/2]) - f.Fuzz(func(t *testing.T, b []byte) { - values, nrow, columns, err := readRDSStringMatrix(b) - if err == nil && nrow > 0 && len(values) != nrow*len(columns) { - t.Errorf("%d values for %d rows and %d columns", len(values), nrow, len(columns)) - } - }) -} diff --git a/internal/mirror/testdata/cran-archive-PACKAGES.rds b/internal/mirror/testdata/cran-archive-PACKAGES.rds deleted file mode 100644 index 9cc5a61abb09e00cde420421b11a10752b422fba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 370 zcmV-&0ge72iwFP!000001Fcb8OT#b}&bH27KsFR#rH_g@+GShY2L;&$2D@RW_*Sws zsbOncnhxBbS2NcXQSm_x9KP?%`OZ1HJkO|BddsrtQsaeJe~R`>Foy`^(&pS4bdm#{&40F zf64sMOnqfh!}O73I@mM_?D;**vM7eoq8;0(Ncl+DBoMDwZNo94iR=!Aly)6x`@REv zMmzHrdGv_^tteSRJ*v+_3CH_n7L6~KsrkMjVs8e!?)}LJeexample_0.9.tar.gz -example_1.0.tar.gz`, + "/src/contrib/Archive/example/": `0.9 +1.0`, }, hits: map[string]int{}, } @@ -132,35 +123,16 @@ func TestMirrorCRANSourcePackagesAreServedFromCache(t *testing.T) { } func TestMirrorCRANRefusesMD5Mismatch(t *testing.T) { - upstream := newCRANUpstream(t, map[string]string{ - cranCurrentPath: "tampered 1.1", - cranArchivedPath: "tampered 1.0", - }) + upstream := newCRANUpstream(t, map[string]string{cranCurrentPath: "tampered 1.1"}) - progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1", "pkg:cran/example@1.0") + progress, proxy := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.1") - if progress.Failed != 2 || progress.Completed != 0 { - t.Fatalf("completed = %d, failed = %d, errors = %v; want both refused", progress.Completed, progress.Failed, progress.Errors) + if progress.Failed != 1 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want it refused", progress.Completed, progress.Failed, progress.Errors) } upstream.Close() - for _, path := range []string{cranCurrentPath, cranArchivedPath} { - if rec := getCRAN(proxy, upstream, path); rec.Code == http.StatusOK { - t.Errorf("%s was cached although its MD5 doesn't match", path) - } - } -} - -func TestMirrorCRANArchiveWithoutPackagesRDS(t *testing.T) { - // A repository without the archive's PACKAGES.rds publishes no checksum - // for archived versions; they are fetched unchecked, as R gets them - // through the handler. - upstream := newCRANUpstream(t, nil) - delete(upstream.files, cranArchiveRDS) - - progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example@1.0") - - if progress.Completed != 1 || upstream.hitsFor(cranArchivedPath) != 1 { - t.Errorf("completed = %d, hits = %d, errors = %v", progress.Completed, upstream.hitsFor(cranArchivedPath), progress.Errors) + if rec := getCRAN(proxy, upstream, cranCurrentPath); rec.Code == http.StatusOK { + t.Errorf("%s was cached although its MD5 doesn't match", cranCurrentPath) } } @@ -169,7 +141,6 @@ func TestMirrorCRANAllVersions(t *testing.T) { progress, _ := mirrorWithConfig(t, cranMirrorConfig(upstream), "pkg:cran/example") - // The archive listing names each file twice, in the link and its text. if progress.Total != 3 || progress.Completed != 3 || progress.Failed != 0 { t.Fatalf("total = %d, completed = %d, failed = %d, errors = %v; want 0.9, 1.0 and 1.1 once each", progress.Total, progress.Completed, progress.Failed, progress.Errors) } From 9babab7380b132e1ee6ccaa4a5900ceb64a317ac Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 08:51:35 +0000 Subject: [PATCH 4/4] Describe archived CRAN versions as fetched unchecked --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7cdbe64a..75878cc6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1625,7 +1625,7 @@ A Julia package is mirrored with the source tarball `upstream.julia` serves at ` A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. -A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions from `src/contrib/Archive/{name}/`, checked against the MD5 sum in that directory's `PACKAGES.rds`, or unchecked when the repository has none there. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. +A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. | Flag | Default | Description | |------|---------|-------------|