diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a63..f8b34656 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1164,3 +1164,46 @@ func TestVersionPublishedAtPreserved(t *testing.T) { } }) } + +// Publish times are written in UTC whatever zone the caller's time is in, so +// a TIMESTAMP column without a zone reads back the same instant. +func TestVersionPublishedAtIsUTC(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + written := time.Date(2026, 10, 10, 21, 0, 1, 0, tokyo) + want := written.UTC() + check := func(purl string) { + t.Helper() + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil || !got.PublishedAt.Valid { + t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) + } + // Postgres reads TIMESTAMP back as a fixed +0000 zone rather + // than time.UTC, so compare the offset. + if _, offset := got.PublishedAt.Time.Zone(); !got.PublishedAt.Time.Equal(want) || offset != 0 { + t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) + } + } + + if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@1.0.0") + + if err := db.UpsertVersion(&Version{ + PURL: "pkg:cargo/demo@2.0.0", + PackagePURL: "pkg:cargo/demo", + PublishedAt: sql.NullTime{Time: written, Valid: true}, + }); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@2.0.0") + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 7af5d070..9fc9f945 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -183,7 +183,7 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { _, err := exec.Exec(query, v.PURL, v.PackagePURL, v.License, v.Integrity, - v.PublishedAt, v.Yanked, v.EnrichedAt, now, now, + inUTC(v.PublishedAt), v.Yanked, v.EnrichedAt, now, now, ) if err != nil { return fmt.Errorf("upserting version: %w", err) @@ -216,7 +216,7 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt ` } - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now) if err != nil { return fmt.Errorf("setting version publish time: %w", err) } diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 578c9ecd..0f5b0145 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -2,6 +2,7 @@ package handler import ( "bufio" + "bytes" "encoding/json" "errors" "fmt" @@ -186,6 +187,26 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr } } +// publishTime reads a crate version's pubtime from the sparse index, sharing +// the index cache with handleIndex. Entries published before crates.io +// recorded pubtime have none. +func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, h.buildIndexPath(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "cargo", name, upstreamURL, "text/plain") + if err != nil { + return time.Time{}, false, err + } + for line := range bytes.SplitSeq(body, []byte("\n")) { + if vers, _, _ := lookupJSONString(line, "vers"); vers != version { + continue + } + pubtime, _, _ := lookupJSONString(line, "pubtime") + published, err := time.Parse(time.RFC3339, pubtime) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // buildIndexPath builds the sparse index path for a crate name. func (h *CargoHandler) buildIndexPath(name string) string { name = strings.ToLower(name) @@ -217,6 +238,13 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "cargo", policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + }) { + return + } + downloadURL := fmt.Sprintf( "%s/%s/%s", h.downloadURL, diff --git a/internal/handler/conda.go b/internal/handler/conda.go index d1f6fb2a..9d694f1d 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -2,8 +2,11 @@ package handler import ( "bufio" + "bytes" "compress/gzip" "encoding/json" + "errors" + "fmt" "io" "net/http" "slices" @@ -127,6 +130,15 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) + // Builds and subdirs of one version each have their own timestamp, so + // the check is per file and keeps no time in the shared versions row. + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "conda", policyName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, + }) { + return + } + upstreamURL := h.upstreamURL + r.URL.Path // The same filename can hold different bytes in each subdir, so the // subdir is part of the cache key. @@ -361,6 +373,101 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { return hashedCacheKey(h.upstreamURL, urlPath) } +// publishTime reads a package file's timestamp from the subdir's +// repodata.json, fetched with the cache key and encoding proxyCached uses. +// The repodata is indexed once per document (see publishIndex), since a +// subdir's index runs to hundreds of megabytes and an install downloads many +// files from it. A compressed copy over metadata_max_size can't be cached, +// so it is streamed from upstream for each check, as handleRepodata does. +func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { + repodataPath := "/" + channel + "/" + arch + "/repodata.json" + cacheKey := h.metadataCacheKey(repodataPath) + res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) + var index map[string]time.Time + var err error + switch { + case errors.Is(res.err, ErrMetadataTooLarge): + index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + case res.err != nil: + err = res.err + default: + index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) + } + if err != nil { + return time.Time{}, false, err + } + published, ok := index[filename] + return published, ok, nil +} + +// indexRepodata indexes a cached repodata.json body. +func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { + var reader io.Reader = bytes.NewReader(body) + if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { + decoded, err := gzip.NewReader(reader) + if err != nil { + return nil, err + } + reader = decoded + } + return readRepodataIndex(reader) +} + +// streamRepodataIndex indexes repodata.json as it streams from upstream. +func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { + return nil, err + } + req.Header.Set(headerAcceptEncoding, "gzip") + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("repodata returned %d", resp.StatusCode) + } + var reader io.Reader = resp.Body + if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { + decoded, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err + } + defer func() { _ = decoded.Close() }() + reader = decoded + } + return readRepodataIndex(reader) +} + +// readRepodataIndex maps each package file in a repodata.json to its +// timestamp, leaving out files without one. It reads the document as it +// streams, under the same bound as filterRepodata, so the decompressed size +// is not limited by metadata_max_size. +func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { + index := make(map[string]time.Time) + err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error { + if m.Key != "packages" && m.Key != "packages.conda" { + return m.Copy() + } + return m.Object(func(entry *jsonstream.Member) error { + raw, err := entry.Raw() + if err != nil { + return err + } + var e repodataEntry + if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { + index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + } + return nil + }) + }) + if err != nil { + return nil, err + } + return index, nil +} + // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { cacheKey := h.metadataCacheKey(r.URL.Path) diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go new file mode 100644 index 00000000..5aec9599 --- /dev/null +++ b/internal/handler/download_cooldown.go @@ -0,0 +1,144 @@ +package handler + +import ( + "net/http" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// publishTimeLookup reads a version's publish time from upstream metadata. ok +// is false when the metadata has no usable time for the version. +type publishTimeLookup func() (published time.Time, ok bool, err error) + +// downloadVersion names the version an artifact download is for, so its +// cooldown can be checked before the artifact is served. +type downloadVersion struct { + ecosystem string + // policyName is the package name the cooldown policy is keyed by, + // canonicalPackagePURL(ecosystem, policyName). + policyName string + // storedName and version name the versions row the publish time is kept + // in, the same names the artifact is cached under. An empty storedName + // keeps no time, and every check asks lookup. + storedName string + version string + lookup publishTimeLookup +} + +// withholdInCooldown reports whether a download is inside the cooldown +// window, and if so answers it with a 404. Filtering metadata is not enough on +// its own: lockfiles record artifact URLs, so clients reach the download path +// without reading the metadata first, and a cached artifact would be served +// even after the policy was raised. +// +// A publish time never changes, so the stored versions row is read first and +// lookup is only asked for a version the proxy has no time for yet; the time +// it finds is stored. A version with no usable publish time is allowed +// through, matching how the metadata filters treat it. +func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) bool { + if !p.versionInCooldown(d) { + return false + } + p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", + "name", d.policyName, "version", d.version) + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + return true +} + +func (p *Proxy) versionInCooldown(d downloadVersion) bool { + if p.Cooldown == nil || !p.Cooldown.Enabled() { + return false + } + policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) + if p.Cooldown.For(d.ecosystem, policyKey) <= 0 { + return false + } + + var versionPURL string + if d.storedName != "" { + versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) + if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time) + } + } + + published, ok, err := d.lookup() + if err != nil { + p.Logger.Warn("cooldown: could not read publish time for download check", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + return false + } + if !ok { + return false + } + + if versionPURL != "" { + if err := p.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL(d.ecosystem, d.storedName), published); err != nil { + p.Logger.Warn("cooldown: could not store publish time", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + } + } + + return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) +} + +// maxPublishIndexes bounds how many documents publishIndexCache keeps an +// index for. +const maxPublishIndexes = 64 + +// publishIndexCache keeps, for each large metadata document, the publish +// times a download check reads from it, so a document is decompressed and +// parsed once per version from upstream rather than on every download. +// Concurrent checks for a document not indexed yet share one parse. +type publishIndexCache struct { + mu sync.Mutex + entries map[string]publishIndexEntry + group singleflight.Group +} + +type publishIndexEntry struct { + digest string + index any +} + +// publishIndex returns build's index of res, reusing the one built for the +// same document under key. res.digest identifies the document; without one, +// as when the metadata cache is off, the index is built for this call alone. +func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([]byte, string) (map[string]V, error)) (map[string]V, error) { + c := &p.publishIndexes + if res.digest == "" { + return build(res.body, res.contentEncoding) + } + c.mu.Lock() + if e, ok := c.entries[key]; ok && e.digest == res.digest { + c.mu.Unlock() + return e.index.(map[string]V), nil + } + c.mu.Unlock() + + v, err, _ := c.group.Do(key+"\x00"+res.digest, func() (any, error) { + index, err := build(res.body, res.contentEncoding) + if err != nil { + return nil, err + } + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = make(map[string]publishIndexEntry) + } + if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { + for evict := range c.entries { + delete(c.entries, evict) + break + } + } + c.entries[key] = publishIndexEntry{digest: res.digest, index: index} + return index, nil + }) + if err != nil { + return nil, err + } + return v.(map[string]V), nil +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go new file mode 100644 index 00000000..1dd44e0a --- /dev/null +++ b/internal/handler/download_cooldown_test.go @@ -0,0 +1,467 @@ +package handler + +import ( + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// Versions every download cooldown fixture lists: one published before the +// window, one inside it, and one without a publish time. +const ( + cooldownOldVersion = "1.0.0" + cooldownRecentVersion = "2.0.0" + cooldownUntimed = "3.0.0" +) + +// downloadCooldownCase drives one ecosystem's artifact endpoint. The package +// is always named demo, so the policy key is pkg:{ecosystem}/demo. +type downloadCooldownCase struct { + ecosystem string + // cachedName is the name the artifact is cached under, and keepsTime + // whether the publish time is kept in its versions row. + cachedName string + keepsTime bool + // artifactVersion maps a release to the version the artifact is cached + // under, and filename and path to its file and download path. + artifactVersion func(version string) string + filename func(version string) string + path func(version string) string + // metadata serves the upstream document holding the publish times; a + // zero time is listed without one. + metadata func(w http.ResponseWriter, r *http.Request, times map[string]time.Time) + handler func(proxy *Proxy, upstreamURL string) http.Handler +} + +func sameVersion(version string) string { return version } + +func chartContent(version string) string { return "chart " + version } + +func downloadCooldownCases() []downloadCooldownCase { + return []downloadCooldownCase{ + { + ecosystem: "cargo", + cachedName: "demo", + keepsTime: true, + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".crate" }, + path: func(v string) string { return "/crates/demo/" + v + "/download" }, + metadata: cargoCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewCargoHandler(p, "http://proxy", u, u).Routes() }, + }, + { + ecosystem: "gem", + cachedName: "demo", + keepsTime: true, + // Platform gems carry the platform in the filename and the + // cached version, but not in the versions API number. + artifactVersion: func(v string) string { return v + "-x86_64-linux" }, + filename: func(v string) string { return "demo-" + v + "-x86_64-linux.gem" }, + path: func(v string) string { return "/gems/demo-" + v + "-x86_64-linux.gem" }, + metadata: gemCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewGemHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + ecosystem: "hex", + cachedName: "demo", + keepsTime: true, + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar" }, + path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, + metadata: hexCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHexHandlerWithUpstreams(p, "http://proxy", u, u).Routes() + }, + }, + { + ecosystem: "pub", + cachedName: "demo", + keepsTime: true, + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar.gz" }, + path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, + metadata: pubCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewPubHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + // Conda caches under {channel}/{name}, but the policy is keyed by + // the name alone, as the repodata filter keys it. Each build has + // its own timestamp, so none is kept in the version's row. + ecosystem: "conda", + cachedName: "conda-forge/demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, + path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, + metadata: condaCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewCondaHandlerWithUpstream(p, "http://proxy", u).Routes() + }, + }, + { + // Helm caches charts under the repository and digest, so the + // chart name and time come from the index on every check. + cachedName: "stable", + ecosystem: "helm", + artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, + filename: func(v string) string { return "demo-" + v + ".tgz" }, + path: func(v string) string { return "/stable/charts/" + sha256Hex(chartContent(v)) + "/demo-" + v + ".tgz" }, + metadata: helmCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHelmHandler(p, "http://proxy", map[string]string{"stable": u + "/charts"}).Routes() + }, + }, + } +} + +func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/de/mo/demo" { + http.NotFound(w, r) + return + } + for version, published := range times { + line := map[string]string{"name": "demo", "vers": version} + if !published.IsZero() { + line["pubtime"] = published.Format(time.RFC3339) + } + encoded, _ := json.Marshal(line) + _, _ = fmt.Fprintf(w, "%s\n", encoded) + } +} + +func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/v1/versions/demo.json" { + http.NotFound(w, r) + return + } + var versions []gemVersion + for version, published := range times { + v := gemVersion{Number: version, Platform: "x86_64-linux"} + if !published.IsZero() { + v.CreatedAt = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(versions) +} + +func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var pkg hexPackageAPI + for version, published := range times { + release := hexRelease{Version: version} + if !published.IsZero() { + release.InsertedAt = published.Format(time.RFC3339Nano) + } + pkg.Releases = append(pkg.Releases, release) + } + _ = json.NewEncoder(w).Encode(pkg) +} + +func pubCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var versions []map[string]string + for version, published := range times { + v := map[string]string{"version": version} + if !published.IsZero() { + v["published"] = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions}) +} + +func condaCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/conda-forge/noarch/repodata.json" { + http.NotFound(w, r) + return + } + packages := map[string]map[string]any{} + for version, published := range times { + entry := map[string]any{"name": "demo", "version": version} + if !published.IsZero() { + entry["timestamp"] = published.UnixMilli() + } + packages["demo-"+version+"-py_0.conda"] = entry + } + // The download check asks for gzip, as proxyCached does. + w.Header().Set(headerContentEncoding, "gzip") + gz := gzip.NewWriter(w) + _ = json.NewEncoder(gz).Encode(map[string]any{"packages.conda": packages}) + _ = gz.Close() +} + +func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/charts/index.yaml" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintln(w, "apiVersion: v1\nentries:\n demo:") + for version, published := range times { + _, _ = fmt.Fprintf(w, " - digest: %s\n urls: [demo-%s.tgz]\n version: %s\n", + sha256Hex(chartContent(version)), version, version) + if !published.IsZero() { + _, _ = fmt.Fprintf(w, " created: %s\n", published.Format(time.RFC3339)) + } + } +} + +// seedCachedArtifact caches the release's artifact the way an earlier, +// cooldown-free download would have. +func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { + t.Helper() + db, store := proxy.DB, proxy.Storage.(*mockStorage) + content := "artifact " + version + if c.ecosystem == "helm" { + content = chartContent(version) + } + seedPackage(t, db, store, c.ecosystem, c.cachedName, c.artifactVersion(version), c.filename(version), content) +} + +// TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's +// artifact endpoint for a version that was cached before the cooldown was +// raised. A version inside the window must be withheld, from the cache too; +// one past the window, or without a known publish time, is served. +func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { + now := time.Now().Truncate(time.Second) + times := map[string]time.Time{ + cooldownOldVersion: now.Add(-30 * 24 * time.Hour), + cooldownRecentVersion: now.Add(-time.Hour), + cooldownUntimed: {}, + } + statuses := map[string]int{ + cooldownOldVersion: http.StatusOK, + cooldownRecentVersion: http.StatusNotFound, + cooldownUntimed: http.StatusOK, + } + + for _, c := range downloadCooldownCases() { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.metadata(w, r, times) + })) + t.Cleanup(upstream.Close) + + for _, version := range []string{cooldownOldVersion, cooldownRecentVersion, cooldownUntimed} { + t.Run(c.ecosystem+"/"+version, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, version) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "demo"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + + if w.Code != statuses[version] { + t.Fatalf("status = %d, want %d (body %q)", w.Code, statuses[version], w.Body.String()) + } + if w.Code == http.StatusOK && w.Body.String() == "" { + t.Error("served an empty body") + } + if !c.keepsTime || times[version].IsZero() { + return + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(version))) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("publish time not stored: %+v, %v", ver, err) + } + if got, want := ver.PublishedAt.Time, times[version]; !got.Equal(want) { + t.Errorf("stored publish time = %v, want %v", got, want) + } + }) + } + } +} + +// TestDownloadCooldownUsesStoredPublishTime checks that a stored publish +// time decides the download without asking the upstream, which here fails. +func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + + for _, c := range downloadCooldownCases() { + if !c.keepsTime { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + versionPURL := canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(cooldownRecentVersion)) + packagePURL := canonicalPackagePURL(c.ecosystem, c.cachedName) + if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { + t.Fatalf("seeding publish time: %v", err) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %q)", w.Code, w.Body.String()) + } + }) + } +} + +// TestDownloadCooldownOtherPackagesSkipLookup checks that a download outside +// every cooldown rule is served without reading upstream metadata. Helm +// needs the index to learn the chart name, so it is left out. +func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem == "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected upstream request %s", r.URL.Path) + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "other"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %q)", w.Code, strings.TrimSpace(w.Body.String())) + } + }) + } +} + +// TestCondaDownloadCooldownPerBuild checks that builds of one version are +// judged by their own timestamps, whichever is requested first: a rebuild +// inside the window is withheld and an old build stays available. +func TestCondaDownloadCooldownPerBuild(t *testing.T) { + now := time.Now() + repodata := map[string]any{"packages.conda": map[string]any{ + "demo-1.0.0-py_0.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-400 * 24 * time.Hour).UnixMilli()}, + "demo-1.0.0-py_1.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-time.Hour).UnixMilli()}, + }} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(repodata) + })) + t.Cleanup(upstream.Close) + + builds := map[string]int{"py_0": http.StatusOK, "py_1": http.StatusNotFound} + for _, order := range [][]string{{"py_0", "py_1", "py_0"}, {"py_1", "py_0", "py_1"}} { + t.Run(strings.Join(order, ","), func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for build := range builds { + filename := "noarch/demo-1.0.0-" + build + ".conda" + seedPackage(t, db, store, "conda", "conda-forge/demo", "1.0.0", filename, "artifact "+build) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + for _, build := range order { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/demo-1.0.0-"+build+".conda", nil)) + if w.Code != builds[build] { + t.Fatalf("%s status = %d, want %d", build, w.Code, builds[build]) + } + } + }) + } +} + +// TestPublishIndexBuildsOncePerDocument checks that an index is reused for +// the same document, rebuilt for a new one, and not kept without a digest. +func TestPublishIndexBuildsOncePerDocument(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + builds := 0 + build := func(body []byte, _ string) (map[string]bool, error) { + builds++ + return map[string]bool{string(body): true}, nil + } + steps := []struct { + res metadataResult + want string + wantBuilds int + }{ + {metadataResult{body: []byte("a"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, + {metadataResult{body: []byte("c")}, "c", 3}, + {metadataResult{body: []byte("c")}, "c", 4}, + } + for i, step := range steps { + got, err := publishIndex(proxy, "key", step.res, build) + if err != nil || !got[step.want] || builds != step.wantBuilds { + t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) + } + } +} + +// TestCondaDownloadCooldownPastMetadataLimit checks that a cached package is +// withheld when its repodata is over metadata_max_size, as the repodata +// endpoint hides it: once only decompressed, and once already compressed. +func TestCondaDownloadCooldownPastMetadataLimit(t *testing.T) { + plain := largeRepodata(500) + compressed := gzipPayload(t, plain) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + })) + t.Cleanup(upstream.Close) + + for _, tt := range []struct { + name string + limit int64 + }{ + {"decompressed over the limit", int64(len(compressed)) + 1}, + {"compressed over the limit", int64(len(compressed)) - 1}, + } { + t.Run(tt.name, func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.MetadataMaxSize = tt.limit + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for _, file := range []string{"pkg-0-old.conda", "pkg-1-new.conda"} { + seedPackage(t, db, store, "conda", "conda-forge/pkg", strings.Split(file, "-")[1], "linux-64/"+file, "artifact "+file) + } + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "pkg-1-new.conda") { + t.Fatalf("repodata status = %d, want 200 without pkg-1-new.conda", w.Code) + } + for file, want := range map[string]int{"pkg-0-old.conda": http.StatusOK, "pkg-1-new.conda": http.StatusNotFound} { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+file, nil)) + if w.Code != want { + t.Errorf("%s status = %d, want %d (body %q)", file, w.Code, want, w.Body.String()) + } + } + }) + } +} diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index 153c0e6e..dbc0e641 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strings" + "time" ) type filenameDownload struct { @@ -12,6 +13,9 @@ type filenameDownload struct { parseErr string fetchErr string parse func(string) (name, version string) + // publishTime reads a version's publish time from upstream metadata for + // the download-time cooldown check. + publishTime func(r *http.Request, name, version string) (time.Time, bool, error) } func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { @@ -30,6 +34,13 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) + if p.withholdInCooldown(w, downloadVersion{ + ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, + }) { + return + } + downloadURL := d.upstreamURL + r.URL.Path result, err := p.GetOrFetchArtifactFromURL( r.Context(), d.ecosystem, name, version, filename, downloadURL, diff --git a/internal/handler/gem.go b/internal/handler/gem.go index dbb6e08d..beb93bbf 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "regexp" "slices" "strconv" @@ -84,6 +85,7 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse gem filename", fetchErr: "failed to fetch gem", parse: h.parseGemFilename, + publishTime: h.publishTime, }) } @@ -332,6 +334,34 @@ func (h *GemHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return filtered, nil } +// publishTime reads a gem version's created_at from the versions API. The +// download check caches the response, unlike fetchFilteredVersions, so each +// download does not cost an upstream request. version carries the platform +// the way gem filenames do, as in 1.0.0-x86_64-linux. +func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, url.PathEscape(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "gem", "api/v1/versions/"+name, versionsURL) + if err != nil { + return time.Time{}, false, err + } + var versions []gemVersion + if err := json.Unmarshal(body, &versions); err != nil { + return time.Time{}, false, err + } + for _, v := range versions { + number := v.Number + if v.Platform != "" && v.Platform != "ruby" { + number += "-" + v.Platform + } + if number != version { + continue + } + createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) + return createdAt, err == nil, nil + } + return time.Time{}, false, nil +} + // proxyCached forwards a metadata request with caching. func (h *GemHandler) proxyCached(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 363314e8..44a033f1 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -276,6 +276,10 @@ type Proxy struct { // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache + // publishIndexes keeps publish times read out of large metadata + // documents for the download cooldown check; see publishIndex. + publishIndexes publishIndexCache + // now is the clock rewritten metadata is dated by, and started the time // the process started; see metadataDate. now func() time.Time diff --git a/internal/handler/helm.go b/internal/handler/helm.go index b8000698..1b2d92bc 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -119,6 +119,10 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } + if h.withholdChart(w, r, repository, upstreamURL, digest) { + return + } + cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) if err != nil { h.proxy.Logger.Error("failed to check Helm chart cache", "error", err) @@ -352,6 +356,72 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } +// withholdChart checks a chart against the cooldown before it is served. +// findChartDownload already leaves out charts in cooldown, but a cached chart +// may have been fetched under an earlier policy. Charts are cached under the +// repository and digest rather than the chart name the policy is keyed by, so +// the name and publish time come from the index, indexed once per document +// (see publishIndex). +func (h *HelmHandler) withholdChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + res := h.fetchIndex(r, repository, upstreamURL) + if res.err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", + "repository", repository, "digest", digest, "error", res.err) + return false + } + index, err := publishIndex(h.proxy, "helm\x00"+h.indexCacheKey(repository, upstreamURL), res, + func(body []byte, _ string) (map[string]helmChartTime, error) { + return h.indexChartTimes(upstreamURL, body) + }) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not read Helm index for download check", + "repository", repository, "digest", digest, "error", err) + return false + } + chart, ok := index[digest] + if !ok { + return false + } + return h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: helmMetadataEcosystem, policyName: chart.name, version: digest, + lookup: func() (time.Time, bool, error) { return chart.created, !chart.created.IsZero(), nil }, + }) +} + +// helmChartTime is the chart a release digest belongs to and when it was +// created. +type helmChartTime struct { + name string + created time.Time +} + +// indexChartTimes maps each release digest in an index to its chart. +func (h *HelmHandler) indexChartTimes(upstreamURL string, body []byte) (map[string]helmChartTime, error) { + _, entries, err := parseHelmIndex(body) + if err != nil { + return nil, err + } + index := make(map[string]helmChartTime) + for i := 0; i < len(entries.Content); i += 2 { + chartName := entries.Content[i].Value + releases := entries.Content[i+1] + if releases.Kind != yaml.SequenceNode { + continue + } + for _, release := range releases.Content { + chart, err := h.parseChartRelease(chartName, upstreamURL, release) + if err != nil { + continue + } + index[chart.digest] = helmChartTime{name: chartName, created: chart.created} + } + } + return index, nil +} + func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && !h.proxy.Cooldown.IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 2d86686c..c8262e00 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" "time" @@ -78,6 +79,7 @@ func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse tarball filename", fetchErr: "failed to fetch package", parse: h.parseTarballFilename, + publishTime: h.publishTime, }) } @@ -253,6 +255,29 @@ func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[st return filtered, nil } +// publishTime reads a release's inserted_at from the Hex HTTP API. The +// download check caches the response, unlike fetchFilteredVersions, so each +// download does not cost an upstream request. +func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, url.PathEscape(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "hex", "api/packages/"+name, apiURL) + if err != nil { + return time.Time{}, false, err + } + var pkg hexPackageAPI + if err := json.Unmarshal(body, &pkg); err != nil { + return time.Time{}, false, err + } + for _, release := range pkg.Releases { + if release.Version != version { + continue + } + insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) + return insertedAt, err == nil, nil + } + return time.Time{}, false, nil +} + // filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper, // filters releases from the Package payload, and re-encodes as gzipped protobuf // (without the original signature since the payload has changed). diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 3c812983..e6bfff4e 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -94,6 +94,13 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("pub download request", "name", name, "version", version) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "pub", policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + }) { + return + } + downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) h.discardVersionPage(name, version, filename) @@ -134,6 +141,33 @@ func (h *PubHandler) discardVersionPage(name, version, filename string) { } } +// publishTime reads a version's published time from the package metadata, +// sharing the metadata cache with handlePackageMetadata. +func (h *PubHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pub", name, upstreamURL) + if err != nil { + return time.Time{}, false, err + } + var metadata struct { + Versions []struct { + Version string `json:"version"` + Published string `json:"published"` + } `json:"versions"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + return time.Time{}, false, err + } + for _, v := range metadata.Versions { + if v.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, v.Published) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // handlePackageMetadata proxies package metadata and rewrites archive URLs. func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name")