From 673a581f9eac1eae7f2d017dfeddcf105a557c23 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 25 Oct 2026 21:48:59 +0100 Subject: [PATCH 1/2] Mirror Swift registry releases proxy mirror and mirror jobs now take pkg:swift PURLs whose namespace is the registry scope, such as pkg:swift/apple/swift-nio@2.62.0 for apple.swift-nio. The source archive is fetched from upstream.swift, checked against the SHA-256 checksum in the release metadata and cached under the key the archive route serves. PURLs whose namespace is a source repository URL don't name a registry identity and are refused. Unversioned PURLs list releases from upstream.swift. SwiftPM reads the release list and Package.swift before it downloads an archive, and the handler passed both through uncached, so a mirrored release still couldn't be resolved with the upstream down. With cache_metadata the handler now keeps them in the metadata cache along with their Link header, which carries release list pages and the version-specific manifests, and serves them when the upstream fails. Manifests are keyed by tools version, so the 5.8 of an alternate link and the 5.8.0 SwiftPM requests share a key. The mirror stores every release list page and each manifest a release links. --- docs/architecture.md | 1 + docs/configuration.md | 6 +- internal/handler/relay_test.go | 2 +- internal/handler/swift.go | 118 ++++++-------- internal/handler/swift_cache.go | 131 +++++++++++++++ internal/handler/swift_cache_test.go | 76 +++++++++ internal/handler/swift_mirror.go | 125 +++++++++++++++ internal/handler/swift_mirror_test.go | 22 +++ internal/mirror/mirror.go | 4 + internal/mirror/swift.go | 64 ++++++++ internal/mirror/swift_test.go | 16 ++ internal/mirror/versions.go | 6 + internal/server/mirror_swift_test.go | 205 ++++++++++++++++++++++++ internal/server/mirror_versions.go | 2 + internal/server/mirror_versions_test.go | 3 +- 15 files changed, 710 insertions(+), 71 deletions(-) create mode 100644 internal/handler/swift_cache.go create mode 100644 internal/handler/swift_cache_test.go create mode 100644 internal/handler/swift_mirror.go create mode 100644 internal/handler/swift_mirror_test.go create mode 100644 internal/mirror/swift.go create mode 100644 internal/mirror/swift_test.go create mode 100644 internal/server/mirror_swift_test.go diff --git a/docs/architecture.md b/docs/architecture.md index e7ce8c8f..5bb1b120 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -304,6 +304,7 @@ HTTP protocol handlers for each registry type. **SwiftHandler:** - Proxies the Swift Package Registry v1 read endpoints - Rewrites release URLs and caches source archives +- With `cache_metadata`, caches release metadata, release lists and manifests (with their `Link` header) and serves them when the upstream fails **GitHandler:** - Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP diff --git a/docs/configuration.md b/docs/configuration.md index 269d5bdb..0966090d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1328,6 +1328,8 @@ Some metadata never changes once published, and is cached like an artifact, with Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. +Swift release lists and manifests are cached with the upstream's `Link` header, which carries the release list's pages and the version-specific manifests a client chooses from. + The `proxy mirror` command always enables metadata caching regardless of this setting. ### Metadata TTL @@ -1565,6 +1567,8 @@ A conda version is mirrored with every build anaconda.org lists for it, across a A Composer release is mirrored with the dist archive `upstream.composer` lists for it, checked against its shasum when one is listed, and cached under the name `/composer/files/...` serves it as. Give the version as the repository lists it, for example `pkg:composer/monolog/monolog@3.8.1`. Branches such as `dev-main` aren't mirrored, since their archive changes with every push, and neither are packages in `upstream.composer_routes`. +A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`. A PURL whose namespace is a source repository, such as `pkg:swift/github.com/apple/swift-argument-parser`, doesn't name a registry identity and is refused. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | @@ -1585,7 +1589,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 ### Unversioned PURLs -A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. +A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. ### Container images diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go index a7a8fb79..ec60460e 100644 --- a/internal/handler/relay_test.go +++ b/internal/handler/relay_test.go @@ -53,7 +53,7 @@ func TestRelayRoutes(t *testing.T) { "/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt", "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", "/gem/info/demo", "/conda/conda-forge/noarch/repodata.json", - "/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift", + "/hex/packages/demo", "/v2/library/demo/manifests/latest", "/v2/library/demo/tags/list", "/npm/-/npm/v1/keys", } { diff --git a/internal/handler/swift.go b/internal/handler/swift.go index d7d403f5..7de0cd12 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -104,23 +104,23 @@ func (h *SwiftHandler) handlePackageReleases(w http.ResponseWriter, r *http.Requ scope, name = canonicalSwiftPackage(scope, name) upstreamURL := h.buildUpstreamURL(scope, name, "", "", r.URL.RawQuery) - body, contentType, responseHeaders, err := h.fetchMetadataWithHeaders( - r.Context(), upstreamURL, requestAccept(r, swiftAcceptJSON), + doc, err := h.fetchDocument( + r.Context(), swiftReleasesCacheKey(scope, name, r.URL.RawQuery), upstreamURL, requestAccept(r, swiftAcceptJSON), ) if err != nil { h.writeMetadataError(w, err) return } - rewritten, err := h.rewriteReleaseURLs(scope, name, body) + rewritten, err := h.rewriteReleaseURLs(scope, name, doc.body) if err != nil { h.proxy.Logger.Warn("failed to rewrite Swift release URLs", "error", err) - rewritten = body + rewritten = doc.body } - for _, link := range responseHeaders.Values("Link") { - w.Header().Add("Link", h.rewriteLinkHeader(link, upstreamURL)) + if doc.link != "" { + w.Header().Set("Link", h.rewriteLinkHeader(doc.link, upstreamURL)) } - writeSwiftMetadata(w, r, rewritten, contentType) + writeSwiftMetadata(w, r, rewritten, doc.contentType) } func (h *SwiftHandler) handleRelease(w http.ResponseWriter, r *http.Request) { @@ -161,7 +161,17 @@ func (h *SwiftHandler) handleManifest(w http.ResponseWriter, r *http.Request) { scope, name = canonicalSwiftPackage(scope, name) upstreamURL := h.buildUpstreamURL(scope, name, version, "Package.swift", r.URL.RawQuery) - h.proxySwiftResource(w, r, upstreamURL, swiftAcceptManifest) + doc, err := h.fetchDocument( + r.Context(), swiftManifestCacheKey(scope, name, version, r.URL.RawQuery), upstreamURL, requestAccept(r, swiftAcceptManifest), + ) + if err != nil { + h.writeMetadataError(w, err) + return + } + if doc.link != "" { + w.Header().Set("Link", h.rewriteLinkHeader(doc.link, upstreamURL)) + } + writeSwiftMetadata(w, r, doc.body, doc.contentType) } func (h *SwiftHandler) handleIdentifiers(w http.ResponseWriter, r *http.Request) { @@ -194,14 +204,13 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques } scope, name = canonicalSwiftPackage(scope, name) - packageName := scope + "/" + name - filename := fmt.Sprintf("%s-%s.zip", name, version) - upstreamURL := h.buildUpstreamURL(scope, name, version+".zip", "", r.URL.RawQuery) - packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", packageName, version) - if packagePURL == "" || versionPURL == "" { - h.writeArtifactError(w, fmt.Errorf("%w: swift %q", errUnsupportedPackageIdentity, packageName)) + key, err := swiftSourceArchiveKey(scope, name, version) + if err != nil { + h.writeArtifactError(w, err) return } + filename, packagePURL, versionPURL := key.filename, key.packagePURL, key.versionPURL + upstreamURL := h.buildUpstreamURL(scope, name, version+".zip", "", r.URL.RawQuery) archiveInfo, infoErr := h.fetchArchiveInfo(r.Context(), scope, name, version) if infoErr != nil { h.writeArtifactError(w, fmt.Errorf("fetching release metadata: %w", infoErr)) @@ -215,10 +224,7 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques headers := make(http.Header) headers.Set("Accept", requestAccept(r, swiftAcceptArchive)) - result, err := h.proxy.getOrFetchArtifactFromURLWithCachePURLs( - r.Context(), "swift", packageName, version, filename, packagePURL, versionPURL, - upstreamURL, headers, archiveInfo.checksum, "", - ) + result, err := h.fetchSourceArchive(r.Context(), key, version, upstreamURL, headers, archiveInfo) if err != nil { h.writeArtifactError(w, err) return @@ -229,6 +235,32 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques ServeArtifactRequest(w, r, result) } +// swiftArchiveKey is the cache key a release's source archive is stored +// under. +type swiftArchiveKey struct { + packageName, filename, packagePURL, versionPURL string +} + +// swiftSourceArchiveKey returns the cache key of the source archive of the +// canonical (lowercased) scope and name at version. +func swiftSourceArchiveKey(scope, name, version string) (swiftArchiveKey, error) { + key := swiftArchiveKey{packageName: scope + "/" + name, filename: fmt.Sprintf("%s-%s.zip", name, version)} + key.packagePURL, key.versionPURL = packageurl.MakeCacheStrings("swift", key.packageName, version) + if key.packagePURL == "" || key.versionPURL == "" { + return swiftArchiveKey{}, fmt.Errorf("%w: swift %q", errUnsupportedPackageIdentity, key.packageName) + } + return key, nil +} + +// fetchSourceArchive returns the source archive under key from the cache, or +// fetches it from upstreamURL, verified against the checksum in info. +func (h *SwiftHandler) fetchSourceArchive(ctx context.Context, key swiftArchiveKey, version, upstreamURL string, headers http.Header, info swiftArchiveInfo) (*CacheResult, error) { + return h.proxy.getOrFetchArtifactFromURLWithCachePURLs( + ctx, "swift", key.packageName, version, key.filename, key.packagePURL, key.versionPURL, + upstreamURL, headers, info.checksum, "", + ) +} + func (h *SwiftHandler) handleSourceArchiveHead( w http.ResponseWriter, r *http.Request, @@ -424,56 +456,6 @@ func swiftDigestHeader(checksum string) string { return "sha-256=" + base64.StdEncoding.EncodeToString(digest) } -func (h *SwiftHandler) proxySwiftResource(w http.ResponseWriter, r *http.Request, upstreamURL, defaultAccept string) { - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - writeSwiftProblem(w, http.StatusInternalServerError, "failed to create upstream request") - return - } - req.Header.Set("Accept", requestAccept(r, defaultAccept)) - for _, name := range []string{"If-Modified-Since", "If-None-Match"} { - if value := r.Header.Get(name); value != "" { - req.Header.Set(name, value) - } - } - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - writeSwiftProblem(w, http.StatusBadGateway, "upstream request failed") - return - } - defer func() { _ = resp.Body.Close() }() - - h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { - copySwiftResponseHeaders(dst, src) - if location := src.Get("Location"); location != "" { - dst.Set("Location", h.rewriteRegistryURL(location, upstreamURL)) - } - for _, link := range src.Values("Link") { - dst.Add("Link", h.rewriteLinkHeader(link, upstreamURL)) - } - if dst.Get("Content-Version") == "" { - dst.Set("Content-Version", swiftContentVersion) - } - }) -} - -func copySwiftResponseHeaders(dst, src http.Header) { - for _, name := range []string{ - "Cache-Control", "Content-Disposition", "Content-Language", headerContentLength, - headerContentType, "Content-Version", "Digest", headerETag, headerLastModified, - "Retry-After", "Vary", "Warning", "X-Swift-Package-Signature", - "X-Swift-Package-Signature-Format", - } { - for _, value := range src.Values(name) { - dst.Add(name, value) - } - } -} - func (h *SwiftHandler) rewriteLinkHeader(value, upstreamRequestURL string) string { var result strings.Builder for len(value) > 0 { diff --git a/internal/handler/swift_cache.go b/internal/handler/swift_cache.go new file mode 100644 index 00000000..e1493311 --- /dev/null +++ b/internal/handler/swift_cache.go @@ -0,0 +1,131 @@ +package handler + +import ( + "bytes" + "context" + "database/sql" + "errors" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// With cache_metadata, release lists and manifests are kept in the metadata +// cache along with the upstream Link header, which carries the pagination of +// release lists and the version-specific manifests a client picks from. They +// are served from the cache within metadata_ttl, and after it when the +// upstream fails, as release metadata is. + +// swiftDocument is a release list or manifest as the upstream sent it. +type swiftDocument struct { + body []byte + contentType string + // link is the upstream Link header, its values joined, with the + // upstream's URLs. + link string + fetchedAt time.Time +} + +// fetchDocument returns the document at upstreamURL, from the metadata cache +// under cacheKey while it is fresh or when the upstream fails, and otherwise +// from upstream, storing it. +func (h *SwiftHandler) fetchDocument(ctx context.Context, cacheKey, upstreamURL, accept string) (*swiftDocument, error) { + cached := h.loadDocument(ctx, cacheKey) + if cached != nil && h.proxy.MetadataTTL > 0 && time.Since(cached.fetchedAt) < h.proxy.MetadataTTL { + return cached, nil + } + body, contentType, headers, err := h.fetchMetadataWithHeaders(ctx, upstreamURL, accept) + if err != nil { + if cached == nil || errors.Is(err, ErrUpstreamNotFound) { + return nil, err + } + h.proxy.Logger.Warn("upstream Swift metadata fetch failed, serving cached copy", + "url", upstreamURL, "error", err) + return cached, nil + } + doc := &swiftDocument{ + body: body, + contentType: contentType, + link: strings.Join(headers.Values("Link"), ", "), + fetchedAt: time.Now(), + } + h.storeDocument(ctx, cacheKey, doc) + return doc, nil +} + +func (h *SwiftHandler) documentsCached() bool { + return h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil +} + +// loadDocument returns the cached document under cacheKey, or nil. +func (h *SwiftHandler) loadDocument(ctx context.Context, cacheKey string) *swiftDocument { + if !h.documentsCached() { + return nil + } + entry, err := h.proxy.DB.GetMetadataCache("swift", cacheKey) + if err != nil || entry == nil || !entry.FetchedAt.Valid { + return nil + } + reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil + } + defer func() { _ = reader.Close() }() + body, err := h.proxy.ReadMetadata(reader) + if err != nil { + return nil + } + h.proxy.recordMetadataRead(entry) + return &swiftDocument{ + body: body, + contentType: entry.ContentType.String, + link: entry.Link.String, + fetchedAt: entry.FetchedAt.Time, + } +} + +// storeDocument caches doc under cacheKey. A failure is logged; the +// document is still served. +func (h *SwiftHandler) storeDocument(ctx context.Context, cacheKey string, doc *swiftDocument) { + if !h.documentsCached() { + return + } + storagePath := metadataStoragePath("swift", cacheKey) + size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) + if err == nil { + err = h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "swift", + Name: cacheKey, + StoragePath: storagePath, + Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, + ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, + }) + } + if err != nil { + h.proxy.Logger.Warn("failed to cache Swift metadata", "key", cacheKey, "error", err) + } +} + +func swiftReleasesCacheKey(scope, name, rawQuery string) string { + return swiftMetadataCacheKey("releases", scope, name, rawQuery) +} + +// swiftManifestCacheKey keys a manifest by its swift-version as a tools +// version, so the 5.8 of a manifest's alternate link and the 5.8.0 a client +// requests for it share a key. +func swiftManifestCacheKey(scope, name, version, rawQuery string) string { + if query, err := url.ParseQuery(rawQuery); err == nil { + if tools := query.Get("swift-version"); tools != "" { + for strings.Count(tools, ".") > 0 && strings.HasSuffix(tools, ".0") { + tools = strings.TrimSuffix(tools, ".0") + } + query.Set("swift-version", tools) + } + rawQuery = query.Encode() + } + return swiftMetadataCacheKey("manifest", scope, name, version, rawQuery) +} diff --git a/internal/handler/swift_cache_test.go b/internal/handler/swift_cache_test.go new file mode 100644 index 00000000..06692c0d --- /dev/null +++ b/internal/handler/swift_cache_test.go @@ -0,0 +1,76 @@ +package handler + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing.T) { + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down.Load() { + http.Error(w, "down", http.StatusServiceUnavailable) + return + } + w.Header().Set("Link", `; rel="next"`) + if strings.HasSuffix(r.URL.Path, "/Package.swift") { + w.Header().Set("Content-Type", "text/x-swift") + _, _ = io.WriteString(w, "// swift-tools-version:5.9") + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"releases":{"1.0.0":{}}}`) + })) + defer upstream.Close() + + for _, cacheMetadata := range []bool{true, false} { + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = cacheMetadata + routes := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + down.Store(false) + for _, path := range []string{"/apple/example", "/apple/example/1.0.0/Package.swift"} { + if w := get(path); w.Code != http.StatusOK { + t.Fatalf("%s: status = %d with the upstream up", path, w.Code) + } + } + + down.Store(true) + for path, body := range map[string]string{"/apple/example": `"1.0.0"`, "/apple/example/1.0.0/Package.swift": "swift-tools-version:5.9"} { + w := get(path) + if !cacheMetadata { + if w.Code == http.StatusOK { + t.Errorf("%s: served without cache_metadata while the upstream is down", path) + } + continue + } + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), body) { + t.Errorf("%s: status = %d, body %q; want the cached copy", path, w.Code, w.Body.String()) + } + if link := w.Header().Get("Link"); link != `; rel="next"` { + t.Errorf("%s: Link = %q, want the cached Link rewritten", path, link) + } + } + } +} + +func TestSwiftManifestCacheKeyIgnoresToolsVersionZeros(t *testing.T) { + key := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8") + if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8.0"); got != key { + t.Error("5.8.0 and 5.8 have different keys") + } + if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.9"); got == key { + t.Error("5.9 and 5.8 share a key") + } + if got := swiftManifestCacheKey("apple", "example", "1.0.0", ""); got == key { + t.Error("Package.swift and the 5.8 manifest share a key") + } +} diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go new file mode 100644 index 00000000..3374cc9d --- /dev/null +++ b/internal/handler/swift_mirror.go @@ -0,0 +1,125 @@ +package handler + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" +) + +// MirrorRelease caches the source archive of release version of the registry +// package scope.name, under the key the source archive route serves it from, +// verified against the SHA-256 checksum its release metadata lists. The +// release metadata is cached as that route reads it, and with cache_metadata +// so are the package's release list and the release's manifests, which a +// client reads before it downloads the archive. A client can then resolve and +// fetch the release without the upstream. +func (h *SwiftHandler) MirrorRelease(ctx context.Context, scope, name, version string) (*CacheResult, error) { + if !validSwiftPackageReference(scope, name, version) { + return nil, fmt.Errorf("invalid Swift registry release %s.%s@%s", scope, name, version) + } + scope, name = canonicalSwiftPackage(scope, name) + key, err := swiftSourceArchiveKey(scope, name, version) + if err != nil { + return nil, err + } + if h.documentsCached() { + if err := h.mirrorReleaseList(ctx, scope, name); err != nil { + return nil, fmt.Errorf("fetching release list: %w", err) + } + if err := h.mirrorManifests(ctx, scope, name, version); err != nil { + return nil, fmt.Errorf("fetching manifest: %w", err) + } + } + info, err := h.fetchArchiveInfo(ctx, scope, name, version) + if err != nil { + return nil, fmt.Errorf("fetching release metadata: %w", err) + } + headers := http.Header{"Accept": {swiftAcceptArchive}} + return h.fetchSourceArchive(ctx, key, version, h.buildUpstreamURL(scope, name, version+".zip", "", ""), headers, info) +} + +// maxSwiftReleasePages bounds the release list pages a mirror follows. +const maxSwiftReleasePages = 100 + +// mirrorReleaseList caches every page of the release list, under the keys +// the release list route reads for the page links it rewrites. +func (h *SwiftHandler) mirrorReleaseList(ctx context.Context, scope, name string) error { + query := "" + seen := map[string]bool{} + for page := 0; page < maxSwiftReleasePages && !seen[query]; page++ { + seen[query] = true + upstreamURL := h.buildUpstreamURL(scope, name, "", "", query) + doc, err := h.fetchDocument(ctx, swiftReleasesCacheKey(scope, name, query), upstreamURL, swiftAcceptJSON) + if err != nil { + return err + } + next := swiftLinkQueries(doc.link, "next", upstreamURL) + if len(next) == 0 { + return nil + } + query = next[0] + } + return nil +} + +// mirrorManifests caches Package.swift of a release and the version-specific +// manifests its Link header lists, under the keys the manifest route reads +// for the alternate links it rewrites. +func (h *SwiftHandler) mirrorManifests(ctx context.Context, scope, name, version string) error { + upstreamURL := h.buildUpstreamURL(scope, name, version, "Package.swift", "") + doc, err := h.fetchDocument(ctx, swiftManifestCacheKey(scope, name, version, ""), upstreamURL, swiftAcceptManifest) + if err != nil { + return err + } + for _, query := range swiftLinkQueries(doc.link, "alternate", upstreamURL) { + alternateURL := h.buildUpstreamURL(scope, name, version, "Package.swift", query) + if _, err := h.fetchDocument(ctx, swiftManifestCacheKey(scope, name, version, query), alternateURL, swiftAcceptManifest); err != nil { + return fmt.Errorf("%s: %w", query, err) + } + } + return nil +} + +// swiftLinkQueries returns the queries of the links with relation rel in +// link, a Link header, that point at requestURL's own path with a query, +// such as the next page of a release list or a version-specific manifest. +func swiftLinkQueries(link, rel, requestURL string) []string { + base, err := url.Parse(requestURL) + if err != nil { + return nil + } + var queries []string + for _, value := range strings.Split(link, ",") { + target, params, ok := strings.Cut(strings.TrimSpace(value), ";") + if !ok || !strings.HasPrefix(target, "<") || !strings.HasSuffix(target, ">") || !swiftLinkHasRel(params, rel) { + continue + } + ref, err := url.Parse(strings.TrimSpace(target[1 : len(target)-1])) + if err != nil { + continue + } + resolved := base.ResolveReference(ref) + if resolved.Host != base.Host || resolved.EscapedPath() != base.EscapedPath() || resolved.RawQuery == "" { + continue + } + queries = append(queries, resolved.RawQuery) + } + return queries +} + +func swiftLinkHasRel(params, rel string) bool { + for _, param := range strings.Split(params, ";") { + key, value, ok := strings.Cut(strings.TrimSpace(param), "=") + if !ok || !strings.EqualFold(strings.TrimSpace(key), "rel") { + continue + } + for _, r := range strings.Fields(strings.Trim(strings.TrimSpace(value), `"`)) { + if strings.EqualFold(r, rel) { + return true + } + } + } + return false +} diff --git a/internal/handler/swift_mirror_test.go b/internal/handler/swift_mirror_test.go new file mode 100644 index 00000000..8359a5ed --- /dev/null +++ b/internal/handler/swift_mirror_test.go @@ -0,0 +1,22 @@ +package handler + +import ( + "reflect" + "testing" +) + +func TestSwiftLinkQueries(t *testing.T) { + const manifest = "https://registry.example/swift/apple/example/1.0.0/Package.swift" + link := `; rel="alternate"; filename="Package@swift-5.8.swift", ` + + `; rel=alternate, ` + + `; rel="alternate", ` + + `; rel="alternate", ` + + `; rel="canonical"` + + got := swiftLinkQueries(link, "alternate", manifest) + + want := []string{"swift-version=5.8", "swift-version=5.7"} + if !reflect.DeepEqual(got, want) { + t.Errorf("swiftLinkQueries() = %q, want %q", got, want) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index 6bd8d183..cca5d234 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -236,6 +236,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorComposer(ctx, pv, tracker) return } + if pv.Ecosystem == "swift" { + m.mirrorSwift(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go new file mode 100644 index 00000000..516276dd --- /dev/null +++ b/internal/mirror/swift.go @@ -0,0 +1,64 @@ +package mirror + +import ( + "context" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" +) + +// mirrorSwift mirrors the source archive of a Swift registry release, with +// its release metadata, from the registry the proxy serves Swift packages +// from. +func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + scope, name, err := swiftScopeAndName(pv.Name) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + upstream, err := m.registryURL("swift", pv.Name) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + result, err := handler.NewSwiftHandler(m.proxy, "", upstream).MirrorRelease(ctx, scope, name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + m.recordResult(tracker, pv, result) +} + +// registryURL returns the registry base URL Versions lists purlType packages +// from, which is the configured upstream, or empty for the default one. +func (m *Mirror) registryURL(purlType, name string) (string, error) { + if r, ok := m.Versions.(RegistryVersions); ok && r.URL != nil { + return r.URL(purlType, name) + } + return "", nil +} + +// swiftScopeAndName splits the name of a pkg:swift PURL, its namespace and +// name, into a registry scope and package name. The namespace of a registry +// package is its scope, as in pkg:swift/apple/swift-nio for the identity +// apple.swift-nio. A namespace naming a source repository, such as +// pkg:swift/github.com/apple/swift-nio, isn't a registry identity and is +// refused. +func swiftScopeAndName(name string) (scope, pkg string, _ error) { + scope, pkg, ok := strings.Cut(name, "/") + if !ok || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { + return "", "", fmt.Errorf("pkg:swift/%s names a source repository; mirroring needs a registry scope as the namespace, such as pkg:swift/apple/swift-nio for apple.swift-nio", name) + } + return scope, pkg, nil +} + +// swiftIdentity returns the registry identity, scope.name, of the name of a +// pkg:swift PURL. +func swiftIdentity(name string) (string, error) { + scope, pkg, err := swiftScopeAndName(name) + if err != nil { + return "", err + } + return scope + "." + pkg, nil +} diff --git a/internal/mirror/swift_test.go b/internal/mirror/swift_test.go new file mode 100644 index 00000000..2d056155 --- /dev/null +++ b/internal/mirror/swift_test.go @@ -0,0 +1,16 @@ +package mirror + +import "testing" + +func TestSwiftIdentity(t *testing.T) { + for name, want := range map[string]string{ + "apple/swift-nio": "apple.swift-nio", + "github.com/apple/swift-nio": "", + "swift-nio": "", + } { + got, err := swiftIdentity(name) + if got != want || (want == "") != (err != nil) { + t.Errorf("swiftIdentity(%q) = %q, %v; want %q", name, got, err, want) + } + } +} diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 1b68a6dd..973ff69b 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -57,6 +57,12 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + if ecosystem == "swift" { + var err error + if name, err = swiftIdentity(name); err != nil { + return nil, err + } + } purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { diff --git a/internal/server/mirror_swift_test.go b/internal/server/mirror_swift_test.go new file mode 100644 index 00000000..d10b682a --- /dev/null +++ b/internal/server/mirror_swift_test.go @@ -0,0 +1,205 @@ +package server + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/mirror" + "github.com/git-pkgs/proxy/internal/storage" +) + +// swiftUpstream serves the Swift registry API for acme.widget with releases +// 1.0.0, on the first page of the release list, and 1.1.0, on the second. +// Each release has a Package.swift that links a manifest for Swift 5.8. +type swiftUpstream struct { + *httptest.Server + // checksum overrides the archive checksum the release metadata lists. + checksum string + hits atomic.Int32 + archiveHits atomic.Int32 +} + +func swiftArchive(version string) string { return "zip of acme.widget " + version } + +func newSwiftUpstream(t *testing.T) *swiftUpstream { + t.Helper() + u := &swiftUpstream{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.hits.Add(1) + w.Header().Set("Content-Version", "1") + rest, ok := strings.CutPrefix(r.URL.Path, "/acme/widget") + switch { + case !ok: + http.NotFound(w, r) + case rest == "": + page := map[string]string{"": "1.0.0", "page=2": "1.1.0"}[r.URL.RawQuery] + if page == "" { + http.NotFound(w, r) + return + } + if r.URL.RawQuery == "" { + w.Header().Set("Link", fmt.Sprintf(`<%s/acme/widget?page=2>; rel="next"`, u.URL)) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"releases": map[string]any{ + page: map[string]string{"url": u.URL + "/acme/widget/" + page}, + }}) + case strings.HasSuffix(rest, ".zip"): + u.archiveHits.Add(1) + _, _ = io.WriteString(w, swiftArchive(strings.TrimSuffix(rest[1:], ".zip"))) + case strings.HasSuffix(rest, "/Package.swift"): + version := strings.TrimSuffix(rest[1:], "/Package.swift") + w.Header().Set("Content-Type", "text/x-swift") + if r.URL.Query().Get("swift-version") == "5.8" { + _, _ = io.WriteString(w, "// swift-tools-version:5.8\n// "+version) + return + } + w.Header().Set("Link", fmt.Sprintf(`<%s/acme/widget/%s/Package.swift?swift-version=5.8>; rel="alternate"; filename="Package@swift-5.8.swift"; swift-tools-version="5.8"`, u.URL, version)) + _, _ = io.WriteString(w, "// swift-tools-version:5.9\n// "+version) + default: + version := rest[1:] + sum := sha256.Sum256([]byte(swiftArchive(version))) + checksum := hex.EncodeToString(sum[:]) + if u.checksum != "" { + checksum = u.checksum + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": "acme.widget", "version": version, + "resources": []map[string]string{{"name": "source-archive", "type": "application/zip", "checksum": checksum}}, + }) + } + })) + t.Cleanup(u.Close) + return u +} + +// mirrorSwift mirrors purls from upstream the way proxy mirror does, with +// metadata caching on. +func mirrorSwift(t *testing.T, upstream *swiftUpstream, purls ...string) (*mirror.Progress, *handler.Proxy) { + t.Helper() + cfg := config.Default() + cfg.Upstream.Swift = upstream.URL + cfg.Upstream.AllowLoopback = true + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatal(err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatal(err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy, _ := NewMirrorProxy(cfg, db, store, logger) + proxy.CacheMetadata = true + m := mirror.New(proxy, db, store, logger, 1) + m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) + progress, err := m.Run(context.Background(), &mirror.PURLSource{PURLs: purls, Versions: m.Versions}) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + return progress, proxy +} + +func swiftRegistryGet(t *testing.T, routes http.Handler, path, accept string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Accept", accept) + rec := httptest.NewRecorder() + routes.ServeHTTP(rec, req) + return rec +} + +func TestMirrorSwiftReleaseIsServedFromCache(t *testing.T) { + upstream := newSwiftUpstream(t) + + progress, proxy := mirrorSwift(t, upstream, "pkg:swift/Acme/Widget@1.0.0") + + if progress.Completed != 1 || upstream.archiveHits.Load() != 1 { + t.Fatalf("completed = %d, archive hits = %d, errors = %v", progress.Completed, upstream.archiveHits.Load(), progress.Errors) + } + + // SwiftPM resolves and downloads the release with the upstream down: + // the release list and its pages, the release metadata, both manifests + // and the archive come from the cache. + upstream.Close() + routes := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL).Routes() + const jsonType = "application/vnd.swift.registry.v1+json" + const swiftType = "application/vnd.swift.registry.v1+swift" + checks := []struct { + path, accept, body, link string + }{ + {"/acme/widget", jsonType, "http://proxy.test/swift/acme/widget/1.0.0", ""}, + {"/acme/widget?page=2", jsonType, "http://proxy.test/swift/acme/widget/1.1.0", ""}, + {"/acme/widget/1.0.0", jsonType, `"source-archive"`, ""}, + {"/acme/widget/1.0.0/Package.swift", swiftType, "swift-tools-version:5.9", ""}, + {"/acme/widget/1.0.0/Package.swift?swift-version=5.8.0", swiftType, "swift-tools-version:5.8", ""}, + {"/acme/widget/1.0.0.zip", "application/vnd.swift.registry.v1+zip", swiftArchive("1.0.0"), ""}, + } + for _, check := range checks { + rec := swiftRegistryGet(t, routes, check.path, check.accept) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), check.body) { + t.Errorf("%s: status = %d, body %q; want 200 with %q", check.path, rec.Code, strings.TrimSpace(rec.Body.String()), check.body) + } + if link := rec.Header().Get("Link"); !strings.Contains(link, check.link) { + t.Errorf("%s: Link = %q, want %q", check.path, link, check.link) + } + } +} + +func TestMirrorSwiftRefusesChecksumMismatch(t *testing.T) { + upstream := newSwiftUpstream(t) + upstream.checksum = strings.Repeat("0", 64) + + progress, proxy := mirrorSwift(t, upstream, "pkg:swift/acme/widget@1.0.0") + + if progress.Failed != 1 || progress.Completed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v; want the archive refused", progress.Completed, progress.Failed, progress.Errors) + } + upstream.Close() + routes := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL).Routes() + if rec := swiftRegistryGet(t, routes, "/acme/widget/1.0.0.zip", "application/vnd.swift.registry.v1+zip"); rec.Code == http.StatusOK { + t.Error("the refused archive was cached") + } +} + +func TestMirrorSwiftListsReleases(t *testing.T) { + upstream := newSwiftUpstream(t) + + progress, _ := mirrorSwift(t, upstream, "pkg:swift/acme/widget") + + if progress.Total != 2 || progress.Completed != 2 || upstream.archiveHits.Load() != 2 { + t.Errorf("total = %d, completed = %d, archive hits = %d, errors = %v; want both releases mirrored", + progress.Total, progress.Completed, upstream.archiveHits.Load(), progress.Errors) + } +} + +func TestMirrorSwiftRefusesRepositoryNamespaces(t *testing.T) { + upstream := newSwiftUpstream(t) + + progress, _ := mirrorSwift(t, upstream, "pkg:swift/github.com/acme/widget@1.0.0") + + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "registry scope") { + t.Errorf("failed = %d, errors = %v; want the repository PURL refused", progress.Failed, progress.Errors) + } + if upstream.hits.Load() != 0 { + t.Error("the upstream was asked for a repository PURL") + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 404388d7..09ab3b7d 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -80,6 +80,8 @@ func mirrorVersionsURL(u config.UpstreamConfig, purlType, name string) (string, return "", nil case "julia": return publicOnly("upstream.julia", u.Julia, d.Julia) + case "swift": + return upstreamOr(u.Swift, d.Swift), nil default: return "", fmt.Errorf("listing %s versions isn't supported; give a version", purlType) } diff --git a/internal/server/mirror_versions_test.go b/internal/server/mirror_versions_test.go index 898c2128..143fac85 100644 --- a/internal/server/mirror_versions_test.go +++ b/internal/server/mirror_versions_test.go @@ -99,6 +99,7 @@ func TestMirrorVersionsURL(t *testing.T) { custom.Cargo = "https://index.example.com" custom.Conda = "https://conda.example.com" custom.Julia = "https://julia.example.com" + custom.Swift = "https://swift.example.com/registry/" tests := []struct { name string @@ -129,7 +130,7 @@ func TestMirrorVersionsURL(t *testing.T) { {"julia default", config.Default().Upstream, "julia", "JSON", "", ""}, {"julia server", custom, "julia", "JSON", "", "upstream.julia"}, {"helm", custom, "helm", "nginx", "", "give a version"}, - {"swift", custom, "swift", "apple/swift-nio", "", "give a version"}, + {"swift", custom, "swift", "apple.swift-nio", "https://swift.example.com/registry", ""}, {"apk", custom, "apk", "curl", "", "give a version"}, } for _, tt := range tests { From 8ea75a30a313e8ae8cc252a95d0a4d2d5f3ec29e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 26 Oct 2026 16:43:42 +0100 Subject: [PATCH 2/2] Check cached Swift documents against their digest and resolve source PURLs Release lists and manifests record their sha256 digest and are read back through storedMetadataEntry and openMetadataEntry, so a corrupted copy is fetched again. pkg:swift PURLs naming a source repository are looked up through the registry's /identifiers. --- docs/configuration.md | 2 +- internal/handler/swift_cache.go | 33 +++++++++---------- internal/handler/swift_cache_test.go | 48 +++++++++++++++++++++++++++ internal/handler/swift_mirror.go | 46 ++++++++++++++++++++++---- internal/mirror/swift.go | 45 +++++++++++++------------ internal/mirror/swift_test.go | 33 +++++++++++++++++-- internal/mirror/versions.go | 49 ++++++++++++++++++++++++---- internal/server/mirror_swift_test.go | 30 +++++++++++++---- 8 files changed, 224 insertions(+), 62 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index afd08d2f..69398650 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1571,7 +1571,7 @@ A Composer release is mirrored with the dist archive `upstream.composer` lists f A Helm chart is mirrored from a repository in `upstream.helm`, named as the namespace, for example `pkg:helm/prometheus-community/prometheus@27.1.0` with `prometheus-community` configured. The chart is fetched from the first URL the repository's `index.yaml` lists for that version, checked against the listed digest, and cached under the repository and digest `/helm/{name}/charts/...` serves it from. The index is cached as well, so with [`cache_metadata`](#metadata-caching) on `helm pull` works while the repository is down. Without a version every version in the index is mirrored. Charts named without a repository, or from one that isn't configured, are refused, and so are charts the index lists in an OCI registry: helm pulls those through `/v2`, so mirror them as `pkg:oci` images. -A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`. A PURL whose namespace is a source repository, such as `pkg:swift/github.com/apple/swift-argument-parser`, doesn't name a registry identity and is refused. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. +A Swift release is mirrored from `upstream.swift` with its source archive, checked against the SHA-256 checksum in the release metadata and cached under the key `/swift/{scope}/{name}/{version}.zip` serves. Give the registry scope as the namespace, for example `pkg:swift/apple/swift-argument-parser@1.5.0` for `apple.swift-argument-parser`, or the source repository, as SBOM tools write it: `pkg:swift/github.com/apple/swift-argument-parser@1.5.0` is looked up through the upstream's `/identifiers`, and the first identity it lists is mirrored. The release metadata, the package's release list and the release's `Package.swift` with its version-specific manifests are stored in the metadata cache too, so `swift package resolve` works with the upstream unreachable. Serving them, and the archive, offline needs `cache_metadata` on the server, since the archive is served with the checksum from the release metadata. | Flag | Default | Description | |------|---------|-------------| diff --git a/internal/handler/swift_cache.go b/internal/handler/swift_cache.go index e1493311..d7781791 100644 --- a/internal/handler/swift_cache.go +++ b/internal/handler/swift_cache.go @@ -12,12 +12,6 @@ import ( "github.com/git-pkgs/proxy/internal/database" ) -// With cache_metadata, release lists and manifests are kept in the metadata -// cache along with the upstream Link header, which carries the pagination of -// release lists and the version-specific manifests a client picks from. They -// are served from the cache within metadata_ttl, and after it when the -// upstream fails, as release metadata is. - // swiftDocument is a release list or manifest as the upstream sent it. type swiftDocument struct { body []byte @@ -30,7 +24,9 @@ type swiftDocument struct { // fetchDocument returns the document at upstreamURL, from the metadata cache // under cacheKey while it is fresh or when the upstream fails, and otherwise -// from upstream, storing it. +// from upstream, storing it. The upstream Link header is cached with it, +// since it carries the pagination of release lists and the version-specific +// manifests a client picks from. func (h *SwiftHandler) fetchDocument(ctx context.Context, cacheKey, upstreamURL, accept string) (*swiftDocument, error) { cached := h.loadDocument(ctx, cacheKey) if cached != nil && h.proxy.MetadataTTL > 0 && time.Since(cached.fetchedAt) < h.proxy.MetadataTTL { @@ -64,11 +60,11 @@ func (h *SwiftHandler) loadDocument(ctx context.Context, cacheKey string) *swift if !h.documentsCached() { return nil } - entry, err := h.proxy.DB.GetMetadataCache("swift", cacheKey) - if err != nil || entry == nil || !entry.FetchedAt.Valid { + entry := h.proxy.storedMetadataEntry("swift", cacheKey, true) + if entry == nil { return nil } - reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath) + reader, err := h.proxy.openMetadataEntry(ctx, "swift", cacheKey, entry) if err != nil { return nil } @@ -93,16 +89,17 @@ func (h *SwiftHandler) storeDocument(ctx context.Context, cacheKey string, doc * return } storagePath := metadataStoragePath("swift", cacheKey) - size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) + size, hash, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(doc.body)) if err == nil { err = h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ - Ecosystem: "swift", - Name: cacheKey, - StoragePath: storagePath, - Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, - ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, - Size: sql.NullInt64{Int64: size, Valid: true}, - FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, + Ecosystem: "swift", + Name: cacheKey, + StoragePath: storagePath, + Link: sql.NullString{String: doc.link, Valid: doc.link != ""}, + ContentType: sql.NullString{String: doc.contentType, Valid: doc.contentType != ""}, + ContentDigest: sql.NullString{String: "sha256:" + hash, Valid: hash != ""}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: doc.fetchedAt, Valid: true}, }) } if err != nil { diff --git a/internal/handler/swift_cache_test.go b/internal/handler/swift_cache_test.go index 06692c0d..2b194a55 100644 --- a/internal/handler/swift_cache_test.go +++ b/internal/handler/swift_cache_test.go @@ -1,12 +1,14 @@ package handler import ( + "context" "io" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" + "time" ) func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing.T) { @@ -62,6 +64,52 @@ func TestSwiftReleaseListAndManifestServedFromCacheWhenUpstreamFails(t *testing. } } +// TestSwiftCorruptManifestRefetched checks that a cached manifest whose +// stored bytes no longer match the recorded digest is fetched again, even +// within metadata_ttl. +func TestSwiftCorruptManifestRefetched(t *testing.T) { + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "text/x-swift") + _, _ = io.WriteString(w, "// swift-tools-version:5.9") + })) + defer upstream.Close() + + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + routes := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes() + get := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.0.0/Package.swift", nil)) + return w + } + + get() + entry, err := db.GetMetadataCache("swift", swiftManifestCacheKey("apple", "example", "1.0.0", "")) + if err != nil || entry == nil { + t.Fatalf("manifest not cached: %v", err) + } + if !strings.HasPrefix(entry.ContentDigest.String, "sha256:") { + t.Fatalf("ContentDigest = %q, want a sha256 digest", entry.ContentDigest.String) + } + if w := get(); hits.Load() != 1 || w.Code != http.StatusOK { + t.Fatalf("fresh cached manifest: status %d, %d upstream fetches; want it served from the cache", w.Code, hits.Load()) + } + + if _, _, err := store.Store(context.Background(), entry.StoragePath, strings.NewReader("// tampered")); err != nil { + t.Fatal(err) + } + w := get() + if hits.Load() != 2 { + t.Errorf("upstream fetches = %d, want the corrupt manifest fetched again", hits.Load()) + } + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "swift-tools-version:5.9") { + t.Errorf("status = %d, body %q; want the upstream manifest", w.Code, w.Body.String()) + } +} + func TestSwiftManifestCacheKeyIgnoresToolsVersionZeros(t *testing.T) { key := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8") if got := swiftManifestCacheKey("apple", "example", "1.0.0", "swift-version=5.8.0"); got != key { diff --git a/internal/handler/swift_mirror.go b/internal/handler/swift_mirror.go index 3374cc9d..d3944e3b 100644 --- a/internal/handler/swift_mirror.go +++ b/internal/handler/swift_mirror.go @@ -2,19 +2,16 @@ package handler import ( "context" + "encoding/json" + "errors" "fmt" "net/http" "net/url" "strings" ) -// MirrorRelease caches the source archive of release version of the registry -// package scope.name, under the key the source archive route serves it from, -// verified against the SHA-256 checksum its release metadata lists. The -// release metadata is cached as that route reads it, and with cache_metadata -// so are the package's release list and the release's manifests, which a -// client reads before it downloads the archive. A client can then resolve and -// fetch the release without the upstream. +// MirrorRelease caches release version of scope.name so that a client can +// resolve and fetch the release without the upstream. func (h *SwiftHandler) MirrorRelease(ctx context.Context, scope, name, version string) (*CacheResult, error) { if !validSwiftPackageReference(scope, name, version) { return nil, fmt.Errorf("invalid Swift registry release %s.%s@%s", scope, name, version) @@ -123,3 +120,38 @@ func swiftLinkHasRel(params, rel string) bool { } return false } + +// ResolveIdentity returns the registry scope and name the upstream lists for +// the source repository at sourceURL. The answer is cached as the identifiers +// route caches it. +func (h *SwiftHandler) ResolveIdentity(ctx context.Context, sourceURL string) (scope, name string, _ error) { + query := url.Values{"url": {sourceURL}}.Encode() + body, _, err := h.proxy.FetchOrCacheMetadata(ctx, "swift", swiftMetadataCacheKey("identifiers", query), + h.upstreamURL+"/identifiers?"+query, swiftAcceptJSON) + if errors.Is(err, ErrUpstreamNotFound) { + return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + } + if err != nil { + return "", "", err + } + return SwiftIdentityOf(body, sourceURL) +} + +// SwiftIdentityOf returns the scope and name of the first identity in body, +// a registry's answer to an identifiers lookup for sourceURL. +func SwiftIdentityOf(body []byte, sourceURL string) (scope, name string, _ error) { + var answer struct { + Identifiers []string `json:"identifiers"` + } + if err := json.Unmarshal(body, &answer); err != nil { + return "", "", fmt.Errorf("parsing the Swift registry's identifiers for %s: %w", sourceURL, err) + } + if len(answer.Identifiers) == 0 { + return "", "", fmt.Errorf("the Swift registry has no package for %s", sourceURL) + } + scope, name, ok := strings.Cut(answer.Identifiers[0], ".") + if !ok || scope == "" || name == "" { + return "", "", fmt.Errorf("the Swift registry answered %s with the invalid identity %q", sourceURL, answer.Identifiers[0]) + } + return scope, name, nil +} diff --git a/internal/mirror/swift.go b/internal/mirror/swift.go index 516276dd..4616c8f2 100644 --- a/internal/mirror/swift.go +++ b/internal/mirror/swift.go @@ -12,17 +12,23 @@ import ( // its release metadata, from the registry the proxy serves Swift packages // from. func (m *Mirror) mirrorSwift(ctx context.Context, pv PackageVersion, tracker *progressTracker) { - scope, name, err := swiftScopeAndName(pv.Name) + upstream, err := m.registryURL("swift", pv.Name) if err != nil { m.recordFailure(tracker, pv, err) return } - upstream, err := m.registryURL("swift", pv.Name) + h := handler.NewSwiftHandler(m.proxy, "", upstream) + var scope, name string + if source := swiftSourceURL(pv.Name); source != "" { + scope, name, err = h.ResolveIdentity(ctx, source) + } else { + scope, name, err = swiftScopeAndName(pv.Name) + } if err != nil { m.recordFailure(tracker, pv, err) return } - result, err := handler.NewSwiftHandler(m.proxy, "", upstream).MirrorRelease(ctx, scope, name, pv.Version) + result, err := h.MirrorRelease(ctx, scope, name, pv.Version) if err != nil { m.recordFailure(tracker, pv, err) return @@ -39,26 +45,23 @@ func (m *Mirror) registryURL(purlType, name string) (string, error) { return "", nil } -// swiftScopeAndName splits the name of a pkg:swift PURL, its namespace and -// name, into a registry scope and package name. The namespace of a registry -// package is its scope, as in pkg:swift/apple/swift-nio for the identity -// apple.swift-nio. A namespace naming a source repository, such as -// pkg:swift/github.com/apple/swift-nio, isn't a registry identity and is -// refused. -func swiftScopeAndName(name string) (scope, pkg string, _ error) { - scope, pkg, ok := strings.Cut(name, "/") - if !ok || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { - return "", "", fmt.Errorf("pkg:swift/%s names a source repository; mirroring needs a registry scope as the namespace, such as pkg:swift/apple/swift-nio for apple.swift-nio", name) +// swiftSourceURL returns the repository URL of a pkg:swift name whose +// namespace starts with a host, as in pkg:swift/github.com/apple/swift-nio, +// or "" for a name whose namespace is a registry scope. +func swiftSourceURL(name string) string { + if host, _, ok := strings.Cut(name, "/"); ok && strings.Contains(host, ".") { + return "https://" + name } - return scope, pkg, nil + return "" } -// swiftIdentity returns the registry identity, scope.name, of the name of a -// pkg:swift PURL. -func swiftIdentity(name string) (string, error) { - scope, pkg, err := swiftScopeAndName(name) - if err != nil { - return "", err +// swiftScopeAndName splits the name of a pkg:swift PURL whose namespace is +// a registry scope, as in pkg:swift/apple/swift-nio for the identity +// apple.swift-nio. +func swiftScopeAndName(name string) (scope, pkg string, _ error) { + scope, pkg, ok := strings.Cut(name, "/") + if !ok || scope == "" || pkg == "" || strings.ContainsAny(scope, ".:") || strings.Contains(pkg, "/") { + return "", "", fmt.Errorf("pkg:swift/%s needs a registry scope or a source repository as the namespace, such as pkg:swift/apple/swift-nio or pkg:swift/github.com/apple/swift-nio", name) } - return scope + "." + pkg, nil + return scope, pkg, nil } diff --git a/internal/mirror/swift_test.go b/internal/mirror/swift_test.go index 2d056155..4ae0764a 100644 --- a/internal/mirror/swift_test.go +++ b/internal/mirror/swift_test.go @@ -1,14 +1,41 @@ package mirror -import "testing" +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/registries" +) func TestSwiftIdentity(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/identifiers" || r.Header.Get("Accept") != "application/vnd.swift.registry.v1+json" { + http.Error(w, "unexpected request", http.StatusBadRequest) + return + } + switch r.URL.Query().Get("url") { + case "https://github.com/apple/swift-nio": + _, _ = w.Write([]byte(`{"identifiers":["apple.swift-nio","mirror.swift-nio"]}`)) + case "https://github.com/acme/empty": + _, _ = w.Write([]byte(`{"identifiers":[]}`)) + default: + http.NotFound(w, r) + } + })) + defer registry.Close() + for name, want := range map[string]string{ "apple/swift-nio": "apple.swift-nio", - "github.com/apple/swift-nio": "", + "github.com/apple/swift-nio": "apple.swift-nio", + "github.com/acme/empty": "", + "github.com/acme/unknown": "", "swift-nio": "", + "apple/": "", + "apple/swift-nio/extra": "", } { - got, err := swiftIdentity(name) + got, err := swiftIdentity(context.Background(), registries.DefaultClient(), registry.URL, name) if got != want || (want == "") != (err != nil) { t.Errorf("swiftIdentity(%q) = %q, %v; want %q", name, got, err, want) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index fce9c2a8..ad22039d 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -2,9 +2,15 @@ package mirror import ( "context" + "encoding/json" + "errors" "fmt" + "net/http" + "net/url" "strings" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" _ "github.com/git-pkgs/registries/all" @@ -58,12 +64,6 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { - if ecosystem == "swift" { - var err error - if name, err = swiftIdentity(name); err != nil { - return nil, err - } - } purlType := purl.EcosystemToPURLType(ecosystem) var baseURL string if r.URL != nil { @@ -76,6 +76,12 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str if client == nil { client = registries.DefaultClient() } + if ecosystem == "swift" { + var err error + if name, err = swiftIdentity(ctx, client, baseURL, name); err != nil { + return nil, err + } + } reg, err := registries.New(purlType, baseURL, client) if err != nil { return nil, err @@ -87,3 +93,34 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str } return reg.FetchVersions(ctx, name) } + +// swiftIdentity returns the registry identity, scope.name, of the name of a +// pkg:swift PURL, looking a source repository up on the registry at baseURL. +func swiftIdentity(ctx context.Context, client *registries.Client, baseURL, name string) (string, error) { + source := swiftSourceURL(name) + if source == "" { + scope, pkg, err := swiftScopeAndName(name) + if err != nil { + return "", err + } + return scope + "." + pkg, nil + } + if baseURL == "" { + baseURL = config.DefaultSwiftUpstream + } + var body json.RawMessage + _, err := client.GetJSONWithHeaders(ctx, + strings.TrimSuffix(baseURL, "/")+"/identifiers?"+url.Values{"url": {source}}.Encode(), + http.Header{"Accept": {"application/vnd.swift.registry.v1+json"}}, &body) + if httpErr := (*registries.HTTPError)(nil); errors.As(err, &httpErr) && httpErr.IsNotFound() { + return "", fmt.Errorf("the Swift registry has no package for %s", source) + } + if err != nil { + return "", err + } + scope, pkg, err := handler.SwiftIdentityOf(body, source) + if err != nil { + return "", err + } + return scope + "." + pkg, nil +} diff --git a/internal/server/mirror_swift_test.go b/internal/server/mirror_swift_test.go index d10b682a..a7050c37 100644 --- a/internal/server/mirror_swift_test.go +++ b/internal/server/mirror_swift_test.go @@ -42,6 +42,13 @@ func newSwiftUpstream(t *testing.T) *swiftUpstream { w.Header().Set("Content-Version", "1") rest, ok := strings.CutPrefix(r.URL.Path, "/acme/widget") switch { + case r.URL.Path == "/identifiers": + if r.URL.Query().Get("url") != "https://github.com/acme/widget" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"identifiers":["acme.widget"]}`) case !ok: http.NotFound(w, r) case rest == "": @@ -191,15 +198,26 @@ func TestMirrorSwiftListsReleases(t *testing.T) { } } -func TestMirrorSwiftRefusesRepositoryNamespaces(t *testing.T) { +func TestMirrorSwiftResolvesRepositoryPURLs(t *testing.T) { upstream := newSwiftUpstream(t) - progress, _ := mirrorSwift(t, upstream, "pkg:swift/github.com/acme/widget@1.0.0") + progress, proxy := mirrorSwift(t, upstream, "pkg:swift/github.com/acme/widget@1.0.0", "pkg:swift/github.com/acme/widget", "pkg:swift/github.com/acme/other@1.0.0") - if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "registry scope") { - t.Errorf("failed = %d, errors = %v; want the repository PURL refused", progress.Failed, progress.Errors) + // The listed PURL's 1.0.0 is the versioned one, so it's mirrored once. + if progress.Completed != 2 || progress.Failed != 1 || upstream.archiveHits.Load() != 2 { + t.Fatalf("completed = %d, failed = %d, archive hits = %d, errors = %v; want both acme.widget releases mirrored and the unknown repository refused", + progress.Completed, progress.Failed, upstream.archiveHits.Load(), progress.Errors) } - if upstream.hits.Load() != 0 { - t.Error("the upstream was asked for a repository PURL") + if len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "no package for https://github.com/acme/other") { + t.Errorf("errors = %v; want the unknown repository named", progress.Errors) + } + + // The lookup is cached too, for clients that resolve a repository URL + // through the registry. + upstream.Close() + routes := handler.NewSwiftHandler(proxy, "http://proxy.test", upstream.URL).Routes() + rec := swiftRegistryGet(t, routes, "/identifiers?url=https%3A%2F%2Fgithub.com%2Facme%2Fwidget", "application/vnd.swift.registry.v1+json") + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "acme.widget") { + t.Errorf("identifiers: status = %d, body %q; want the cached lookup", rec.Code, rec.Body.String()) } }