From 4db59ba245adf0efa65f6e483901ef10a205befc Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 10:26:25 +0100 Subject: [PATCH] fix(api): keep server-side fetches on the hosts they are meant for The CityBikes client built a request URL by appending a link from the upstream index to its base, so the link could name another host; it now accepts only a network path on the CityBikes origin. The private feed host setting applied to every safe fetch by default, including the extension store; feed downloads now opt in and everything else refuses private hosts. --- .../__tests__/extension-store.test.ts | 20 ++++ apps/api/src/services/extension-store.ts | 51 +++++---- docs/docs/install/configuration.md | 1 + .../__tests__/citybikes-client.test.ts | 107 ++++++++++++++++++ .../__tests__/contributor-acquisition.test.ts | 25 +--- .../bike-sharing/__tests__/fixtures.ts | 26 +++++ .../providers/citybikes-client.ts | 52 ++++++++- integrations/ride-gofs/catalog.ts | 4 +- integrations/ride-gofs/feed.ts | 5 +- .../core/src/__tests__/safe-download.test.ts | 42 +++++++ packages/core/src/mobility-http-transport.ts | 20 +++- packages/core/src/server.ts | 1 + packages/core/src/utils/safe-download.ts | 9 +- 13 files changed, 306 insertions(+), 57 deletions(-) create mode 100644 integrations/bike-sharing/__tests__/citybikes-client.test.ts create mode 100644 integrations/bike-sharing/__tests__/fixtures.ts diff --git a/apps/api/src/services/__tests__/extension-store.test.ts b/apps/api/src/services/__tests__/extension-store.test.ts index c26fabbac..bd11f8db7 100644 --- a/apps/api/src/services/__tests__/extension-store.test.ts +++ b/apps/api/src/services/__tests__/extension-store.test.ts @@ -243,6 +243,26 @@ describe("resolveExtensionManifest (real safeFetchJson SSRF path)", () => { }); }); +describe("extension fetches ignore the private feed mirror allowlist", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("refuses a manifest host the operator allowed as a private feed mirror", async () => { + vi.stubEnv("OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS", "feeds.example.test"); + lookupMock.mockResolvedValue([{ address: "10.0.0.5", family: 4 }]); + const fetchMock = stubFetchSequence(); + + await expect( + fetchManifestMeta("https://feeds.example.test/extension.json"), + ).resolves.toBeNull(); + await expect( + resolveExtensionManifest( + entry({ id: "e", name: "E", version: "1", manifest: "https://feeds.example.test/e.json" }), + ), + ).rejects.toThrow(/not allowed \(private/); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + describe("immutable verified trust", () => { it("treats only an exact 40-hex commit path as immutable", () => { expect( diff --git a/apps/api/src/services/extension-store.ts b/apps/api/src/services/extension-store.ts index 6a7b3ce58..1246de13e 100644 --- a/apps/api/src/services/extension-store.ts +++ b/apps/api/src/services/extension-store.ts @@ -46,6 +46,15 @@ const EXTENSION_REVOCATION_FEED_URL = envString( "https://raw.githubusercontent.com/openmapx/community-extensions/main/revocations.json", ); +// Shared by every catalog, manifest and revocation request. Their URLs come +// from catalog content, so private hosts stay refused even when the operator +// allows private feed mirrors; that setting covers feed downloads only. +const EXTENSION_FETCH_OPTIONS = { + headers: { "User-Agent": USER_AGENT_ADMIN }, + timeoutMs: 15_000, + allowPrivateHosts: [] as string[], +}; + const COMMIT_PATH_SEGMENT = /(^|\/)[a-f0-9]{40}(\/|$)/; /** @@ -145,10 +154,10 @@ interface RevocationSnapshot { async function fetchRevocations(): Promise<{ snapshot: RevocationSnapshot; stale: boolean }> { const cached = await readCachedRevocations(); try { - const data = await safeFetchJson(EXTENSION_REVOCATION_FEED_URL, { - headers: { "User-Agent": USER_AGENT_ADMIN }, - timeoutMs: 15_000, - }); + const data = await safeFetchJson( + EXTENSION_REVOCATION_FEED_URL, + EXTENSION_FETCH_OPTIONS, + ); // Validate the whole document before using any of it. A document of the // wrong shape is a failed refresh, not evidence that nothing is revoked, and // it must not replace the last good snapshot. @@ -401,10 +410,7 @@ async function fetchCatalogFromUrl(url: string): Promise { } catch { throw new ExtensionStoreError("catalog_acquisition_failed"); } - const data = await safeFetchJson(canonical, { - headers: { "User-Agent": USER_AGENT_ADMIN }, - timeoutMs: 15_000, - }); + const data = await safeFetchJson(canonical, EXTENSION_FETCH_OPTIONS); // Accept either a bare array of entries or `{ extensions, removed, critical }`. // Validate the whole document before consuming it: malformed acquisition is // not evidence of an empty catalog, and must not clear cached denials. @@ -440,10 +446,10 @@ export async function fetchManifestMeta( url: string, ): Promise<{ version?: string; platform?: string } | null> { try { - const data = await safeFetchJson<{ version?: unknown; platform?: unknown }>(url, { - headers: { "User-Agent": USER_AGENT_ADMIN }, - timeoutMs: 15_000, - }); + const data = await safeFetchJson<{ version?: unknown; platform?: unknown }>( + url, + EXTENSION_FETCH_OPTIONS, + ); return { version: typeof data.version === "string" && data.version ? data.version : undefined, platform: typeof data.platform === "string" && data.platform ? data.platform : undefined, @@ -642,10 +648,10 @@ export async function resolveExtensionManifest( // Verify the digest over the exact received bytes BEFORE they are parsed or // cached. Parsing first would expose the parser to unverified content and // would let a cache hold bytes that were never authorized. - const text = await safeFetchText(normalizeExtensionLocator(entry.manifest), { - headers: { "User-Agent": USER_AGENT_ADMIN }, - timeoutMs: 15_000, - }); + const text = await safeFetchText( + normalizeExtensionLocator(entry.manifest), + EXTENSION_FETCH_OPTIONS, + ); const digest = createHash("sha256").update(text, "utf8").digest("hex"); if (digest !== entry.manifestSha256.toLowerCase()) { throw new ExtensionStoreError("digest_mismatch"); @@ -655,10 +661,10 @@ export async function resolveExtensionManifest( if (entry.trust === "verified") { throw new ExtensionStoreError("missing_digest"); } - raw = await safeFetchJson(normalizeExtensionLocator(entry.manifest), { - headers: { "User-Agent": USER_AGENT_ADMIN }, - timeoutMs: 15_000, - }); + raw = await safeFetchJson( + normalizeExtensionLocator(entry.manifest), + EXTENSION_FETCH_OPTIONS, + ); } else { raw = { id: entry.id, @@ -711,10 +717,7 @@ export function normalizeExtensionLocator(url: string): string { /** Direct content has no catalog authorization to bind against or trust to inherit. */ export async function resolveDirectExtensionManifest(url: string): Promise { - const raw = await safeFetchJson(normalizeExtensionLocator(url), { - headers: { "User-Agent": USER_AGENT_ADMIN }, - timeoutMs: 15_000, - }); + const raw = await safeFetchJson(normalizeExtensionLocator(url), EXTENSION_FETCH_OPTIONS); return validatedManifest(raw); } diff --git a/docs/docs/install/configuration.md b/docs/docs/install/configuration.md index b564cb86b..fad7e5c3d 100644 --- a/docs/docs/install/configuration.md +++ b/docs/docs/install/configuration.md @@ -202,6 +202,7 @@ required pieces of this group are listed under [Core / required](#core--required | `DATA_MANAGER_URL` | Base URL `app-api` uses to reach the data-manager. Compose overrides this to the service-network DNS name automatically. | Default `http://localhost:4000` (`pnpm dev`); compose sets `http://data-manager:4000` | | `DATA_MANAGER_PLAINTEXT_HOSTS` | Comma-separated hostnames that may be reached over plain `http://` in addition to `localhost`, `127.0.0.1`, `::1` and `data-manager`. `DATA_MANAGER_URL` must otherwise be `https://`; credentials, path prefixes and query strings are always rejected. | Unset | | `OPENMAPX_FETCH_JSON_MAX_BYTES` | Ceiling in bytes for upstream JSON responses fetched by integrations that do not set their own limit. Lower it to tighten memory use, or raise it for a large regional feed; a raised value is logged once at first use. | 8 MiB | +| `OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS` | Comma-separated hostnames (exact or `*.suffix`) that transit and mobility feed downloads may reach even though they resolve to a private address, for feed mirrors on a LAN. It applies to feed downloads only; the extension store, health checks and other server-side fetches still refuse private addresses. | Unset | | `OPENMAPX_RELEASE` | Select `main`, `preview`, an exact prerelease such as `v0.1.0-alpha.1`, a metadata `sha256:<64hex>` digest, or explicit `local` opt-out. Missing selections fail; stable/latest are unavailable. Existing valid overlays remain deliberate pins until reselected or cleared. | `main` | | `OPENMAPX_RELEASE_MANIFEST_REPOSITORY` | Trusted manifest repository without tag/digest. Its namespace must contain every one of the six images; forwarded to the operations agent. | `ghcr.io/openmapx/release-manifest` | | `OPENMAPX_ENABLED_SERVICES` | Pin a specific service selection (comma- or space-separated) for CI/repeatable deploys. Wins over the `service-selection.json` written by the admin UI / CLI. | Optional. Default: `service-selection.json` | diff --git a/integrations/bike-sharing/__tests__/citybikes-client.test.ts b/integrations/bike-sharing/__tests__/citybikes-client.test.ts new file mode 100644 index 000000000..799b4670c --- /dev/null +++ b/integrations/bike-sharing/__tests__/citybikes-client.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { searchCityBikes } from "../providers/citybikes-client.js"; +import { memoryCache } from "./fixtures.js"; + +const bbox = { west: 13.3, south: 52.4, east: 13.5, north: 52.6 }; +const location = { longitude: 13.4, latitude: 52.5, city: "Berlin", country: "DE" }; +const INDEX_CACHE_KEY = "shared-mobility:citybikes:networks"; + +function network(id: string, href: unknown) { + return { id, name: id, href, company: ["Operator"], location }; +} + +function detail(id: string) { + return { + network: { + id, + name: id, + company: ["Operator"], + stations: [ + { + id: "one", + name: "Station", + longitude: 13.4, + latitude: 52.5, + free_bikes: 2, + empty_slots: 5, + }, + ], + }, + }; +} + +function stubUpstream(networks: unknown[]) { + const http = vi.fn(async (input: unknown) => { + const url = new URL(String(input)); + const body = url.search.includes("fields=") + ? { networks } + : detail(url.pathname.split("/").pop() ?? ""); + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + }); + vi.stubGlobal("fetch", http); + return http; +} + +function requestedUrls(http: ReturnType): string[] { + return http.mock.calls.map(([input]) => String(input)); +} + +describe("CityBikes network detail links", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("fetches only network detail paths on the CityBikes origin", async () => { + const http = stubUpstream([ + network("at-sign", "@other.example/v2/networks/a"), + network("dot", ".other.example/v2/networks/a"), + network("scheme-relative", "//other.example/v2/networks/a"), + network("absolute", "https://other.example/v2/networks/a"), + network("query", "/v2/networks/a?x=1"), + network("other-path", "/v2/other"), + network("not-a-string", 42), + network("berlin", "/v2/networks/berlin"), + ]); + + const stations = await searchCityBikes(bbox, memoryCache()); + + expect(stations.map((s) => s.sources)).toEqual([["citybikes/berlin"]]); + const urls = requestedUrls(http); + expect(urls).toHaveLength(2); + expect(urls[1]).toBe("https://api.citybik.es/v2/networks/berlin"); + for (const url of urls) expect(new URL(url).origin).toBe("https://api.citybik.es"); + }); + + it("does not cache index entries with unusable detail links", async () => { + stubUpstream([ + network("absolute", "https://other.example/v2/networks/a"), + network("berlin", "/v2/networks/berlin"), + ]); + const cache = memoryCache(); + + await searchCityBikes(bbox, cache); + + const cached = await cache.get<{ id: string }[]>(INDEX_CACHE_KEY); + expect(cached?.map((n) => n.id)).toEqual(["berlin"]); + }); + + it("skips an unusable entry already held in the index cache", async () => { + const http = stubUpstream([]); + const cache = memoryCache(); + await cache.set(INDEX_CACHE_KEY, [ + network("absolute", "https://other.example/v2/networks/a"), + network("berlin", "/v2/networks/berlin"), + ]); + + const stations = await searchCityBikes(bbox, cache); + + expect(stations.map((s) => s.sources)).toEqual([["citybikes/berlin"]]); + expect(requestedUrls(http)).toEqual(["https://api.citybik.es/v2/networks/berlin"]); + }); +}); diff --git a/integrations/bike-sharing/__tests__/contributor-acquisition.test.ts b/integrations/bike-sharing/__tests__/contributor-acquisition.test.ts index a9ac130c1..69ce2a2be 100644 --- a/integrations/bike-sharing/__tests__/contributor-acquisition.test.ts +++ b/integrations/bike-sharing/__tests__/contributor-acquisition.test.ts @@ -1,31 +1,8 @@ -import type { CacheClient } from "@openmapx/mobility-core/cache"; import { afterEach, expect, it, vi } from "vitest"; import { searchCityBikes } from "../providers/citybikes-client.js"; import { searchNextbike } from "../providers/nextbike-client.js"; +import { memoryCache } from "./fixtures.js"; -function memoryCache(): CacheClient { - const values = new Map(); - return { - get: async (key: string) => (values.get(key) as T) ?? null, - set: async (key, value) => { - values.set(key, structuredClone(value)); - }, - del: async (key) => { - values.delete(key); - }, - withCache: async function ( - key: string, - _ttl: number, - load: (signal: AbortSignal) => Promise, - ) { - const previous = await this.get(key); - if (previous !== null) return previous; - const data = await load(new AbortController().signal); - await this.set(key, data); - return data; - }, - }; -} const bbox = { west: 13.3, south: 52.4, east: 13.5, north: 52.6 }; afterEach(() => vi.unstubAllGlobals()); diff --git a/integrations/bike-sharing/__tests__/fixtures.ts b/integrations/bike-sharing/__tests__/fixtures.ts new file mode 100644 index 000000000..7e5ed0b32 --- /dev/null +++ b/integrations/bike-sharing/__tests__/fixtures.ts @@ -0,0 +1,26 @@ +import type { CacheClient } from "@openmapx/mobility-core/cache"; + +/** In-memory read-through cache with the host's clone-on-write semantics. */ +export function memoryCache(): CacheClient { + const values = new Map(); + return { + get: async (key: string) => (values.get(key) as T) ?? null, + set: async (key, value) => { + values.set(key, structuredClone(value)); + }, + del: async (key) => { + values.delete(key); + }, + withCache: async function ( + key: string, + _ttl: number, + load: (signal: AbortSignal) => Promise, + ) { + const previous = await this.get(key); + if (previous !== null) return previous; + const data = await load(new AbortController().signal); + await this.set(key, data); + return data; + }, + }; +} diff --git a/integrations/bike-sharing/providers/citybikes-client.ts b/integrations/bike-sharing/providers/citybikes-client.ts index ece2e4a43..4b438d3e3 100644 --- a/integrations/bike-sharing/providers/citybikes-client.ts +++ b/integrations/bike-sharing/providers/citybikes-client.ts @@ -15,6 +15,8 @@ import { type CacheClient, TTL, withCache } from "@openmapx/mobility-core/cache" import type { SharedMobilityStation } from "@openmapx/mobility-core/shared-mobility"; const CITYBIKES_BASE = "https://api.citybik.es"; +const CITYBIKES_ORIGIN = new URL(CITYBIKES_BASE).origin; +const NETWORK_PATH = /^\/v2\/networks\/[A-Za-z0-9][A-Za-z0-9._-]*$/; const HEADERS = { Accept: "application/json", }; @@ -68,6 +70,32 @@ async function fetchJson(url: string): Promise { }); } +/** + * Resolve a network's `href` from the upstream index to a detail URL. Only a + * network detail path on the CityBikes origin is accepted; anything else + * returns null so the network is skipped rather than fetched. + */ +function cityBikesNetworkUrl(href: unknown): URL | null { + if (typeof href !== "string") return null; + let url: URL; + try { + url = new URL(href, CITYBIKES_BASE); + } catch { + return null; + } + if ( + url.origin !== CITYBIKES_ORIGIN || + url.username || + url.password || + url.search || + url.hash || + !NETWORK_PATH.test(url.pathname) + ) { + return null; + } + return url; +} + /** Cached index of all CityBikes networks. */ async function getNetworkIndex(cache: CacheClient): Promise { return withCache( @@ -78,7 +106,20 @@ async function getNetworkIndex(cache: CacheClient): Promise const data = await fetchJson<{ networks: CityBikesNetwork[] }>( `${CITYBIKES_BASE}/v2/networks?fields=id,name,href,company,location`, ); - return data.networks; + // Drop entries whose detail link leaves the CityBikes API before they are + // cached, so one bad entry cannot be reused for the whole cache lifetime. + const skipped: string[] = []; + const networks = data.networks.filter((network) => { + const usable = typeof network.id === "string" && cityBikesNetworkUrl(network.href) !== null; + if (!usable) skipped.push(String(network.id)); + return usable; + }); + if (skipped.length > 0) { + console.warn( + `[citybikes] skipped ${skipped.length} network(s) with an unusable detail link: ${skipped.slice(0, 20).join(", ")}`, + ); + } + return networks; }, ); } @@ -110,13 +151,20 @@ export async function fetchNetworkStations( cache: CacheClient, contributors?: readonly Attribution[], ): Promise { + // Checked again here because an index cached before this check existed may + // still hold entries that were never validated. + const detailUrl = cityBikesNetworkUrl(network.href); + if (!detailUrl) { + console.warn(`[citybikes] skipped network ${String(network.id)}: unusable detail link`); + return []; + } const cacheKey = `shared-mobility:citybikes:contributors-v1:${network.id}`; const stations = await withCache( cache, cacheKey, TTL.sharedMobility.stations, async () => { - const data = await fetchJson(`${CITYBIKES_BASE}${network.href}`); + const data = await fetchJson(detailUrl.href); const operator = network.company?.[0] ?? network.name; return data.network.stations.map( diff --git a/integrations/ride-gofs/catalog.ts b/integrations/ride-gofs/catalog.ts index 5aed4e723..63f4efd9f 100644 --- a/integrations/ride-gofs/catalog.ts +++ b/integrations/ride-gofs/catalog.ts @@ -1,4 +1,4 @@ -import { safeFetchJson } from "@openmapx/core/server"; +import { privateFeedHostAllowlist, safeFetchJson } from "@openmapx/core/server"; import { type CacheWriter, captureCacheWriter, @@ -181,6 +181,8 @@ export function createGofsCatalog(ctx: IntegrationContext, fetchJson?: GofsFetch ? fetchJson(url, headers) : safeFetchJson(url, { headers, + // The catalog and its system feeds are feed downloads. + allowPrivateHosts: privateFeedHostAllowlist(), allowedRedirectHosts: headers && Object.keys(headers).length > 0 ? [new URL(url).hostname] : undefined, }); diff --git a/integrations/ride-gofs/feed.ts b/integrations/ride-gofs/feed.ts index 9c3379611..7c6d8caa8 100644 --- a/integrations/ride-gofs/feed.ts +++ b/integrations/ride-gofs/feed.ts @@ -1,4 +1,4 @@ -import { safeFetchJson } from "@openmapx/core/server"; +import { privateFeedHostAllowlist, safeFetchJson } from "@openmapx/core/server"; import { captureCacheWriter, type IntegrationContext } from "@openmapx/integration-framework"; import { type GofsRealtimeBookingEntry, @@ -25,10 +25,13 @@ export type GofsFetchJson = (url: string, headers?: Record) => P * feed's own host: `safeFetchJson` follows redirects, and without * `allowedRedirectHosts` a 302 from the feed would hand the API key to whatever * host the `Location` names. + * + * GOFS feeds are feed downloads, so the operator's private feed mirrors apply. */ const defaultFetchJson: GofsFetchJson = (url, headers) => safeFetchJson(url, { headers, + allowPrivateHosts: privateFeedHostAllowlist(), allowedRedirectHosts: headers && Object.keys(headers).length > 0 ? [new URL(url).hostname] : undefined, }); diff --git a/packages/core/src/__tests__/safe-download.test.ts b/packages/core/src/__tests__/safe-download.test.ts index 686aaf8ee..8b084bc13 100644 --- a/packages/core/src/__tests__/safe-download.test.ts +++ b/packages/core/src/__tests__/safe-download.test.ts @@ -43,6 +43,7 @@ vi.mock("undici", async (importOriginal) => { // mocked module to drive the mock. import { lookup as dnsLookup } from "node:dns/promises"; import { fetch as undiciFetch } from "undici"; +import { mobilityHttpTransport } from "../mobility-http-transport"; import { createPinnedFetchTransport } from "../utils/pinned-fetch"; import { assertResolvesToPublicIp, @@ -1026,6 +1027,47 @@ describe("safeFetchJson", () => { /HTTP\(S\)/, ); }); + + describe("operator feed mirror allowlist", () => { + afterEach(() => vi.unstubAllEnvs()); + + const MIRROR = "https://feeds.example.test/x.json"; + + it("is not applied when the caller names no private hosts", async () => { + vi.stubEnv("OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS", "feeds.example.test"); + lookupMock.mockResolvedValue([{ address: "10.0.0.5", family: 4 }]); + const fetchMock = stubFetchSequence(); + + await expect(safeFetchJson(MIRROR)).rejects.toThrow(/not allowed \(private/); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("still applies to mobility feed downloads", async () => { + vi.stubEnv("OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS", "feeds.example.test"); + lookupMock.mockResolvedValue([{ address: "10.0.0.5", family: 4 }]); + stubFetchSequence( + makeResponse({ + status: 200, + headers: { "content-type": "application/json" }, + bodyText: JSON.stringify({ ok: true }), + }), + ); + + await expect(mobilityHttpTransport.fetchJson(MIRROR)).resolves.toEqual({ ok: true }); + expect(undiciFetchMock).toHaveBeenCalledTimes(1); + }); + + it("lets a mobility caller replace the feed list with its own", async () => { + vi.stubEnv("OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS", "feeds.example.test"); + lookupMock.mockResolvedValue([{ address: "10.0.0.5", family: 4 }]); + const fetchMock = stubFetchSequence(); + + await expect( + mobilityHttpTransport.fetchJson(MIRROR, { allowPrivateHosts: [] }), + ).rejects.toThrow(/not allowed \(private/); + expect(fetchMock).not.toHaveBeenCalled(); + }); + }); }); describe("safeDownload", () => { diff --git a/packages/core/src/mobility-http-transport.ts b/packages/core/src/mobility-http-transport.ts index 8057e84d5..8a7f85bca 100644 --- a/packages/core/src/mobility-http-transport.ts +++ b/packages/core/src/mobility-http-transport.ts @@ -1,4 +1,7 @@ -import type { MobilityHttpTransport } from "@openmapx/mobility-core/json-transport"; +import type { + MobilityHttpRequestOptions, + MobilityHttpTransport, +} from "@openmapx/mobility-core/json-transport"; import { hostMatchesAllowlist, privateFeedHostAllowlist, @@ -7,10 +10,21 @@ import { } from "./utils/safe-download.js"; import { USER_AGENT } from "./utils/userAgent.js"; +/** + * Mobility requests are feed downloads, so they honour the operator's private + * feed mirrors unless the caller names its own private hosts. + */ +function withFeedHosts(options: MobilityHttpRequestOptions | undefined) { + return { + ...options, + allowPrivateHosts: options?.allowPrivateHosts ?? privateFeedHostAllowlist(), + }; +} + export const mobilityHttpTransport: MobilityHttpTransport = { userAgent: USER_AGENT, - fetchJson: (url, options) => safeFetchJson(url, options), - fetchText: (url, options) => safeFetchText(url, options), + fetchJson: (url, options) => safeFetchJson(url, withFeedHosts(options)), + fetchText: (url, options) => safeFetchText(url, withFeedHosts(options)), hostMatchesAllowlist, privateFeedHostAllowlist, }; diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 46f2c0ecd..535750289 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -115,6 +115,7 @@ export { // Safe downloader (DNS-aware SSRF protection for server-side fetches). export { assertResolvesToPublicIp, + privateFeedHostAllowlist, resolvePublicConnectionAddresses, type SafeDownloadOptions, type SafeDownloadResult, diff --git a/packages/core/src/utils/safe-download.ts b/packages/core/src/utils/safe-download.ts index 4d49b335b..4e64e8f2a 100644 --- a/packages/core/src/utils/safe-download.ts +++ b/packages/core/src/utils/safe-download.ts @@ -564,7 +564,9 @@ export interface SafeFetchJsonOptions { truncate?: boolean; /** * Hostnames (exact or "*.suffix") permitted to resolve to a private address. - * Defaults to `privateFeedHostAllowlist()` when omitted. + * Defaults to none. Transit and mobility feed downloads opt in to the + * operator's feed mirrors by passing `privateFeedHostAllowlist()`; other + * callers must not, because that setting covers feed mirrors only. */ allowPrivateHosts?: string[]; /** @@ -652,6 +654,9 @@ async function resolvePublicHostOrThrow(hostname: string): Promise( ): Promise { const timeoutMs = opts.timeoutMs ?? 15_000; const maxRedirects = opts.maxRedirects ?? 5; - const allowPrivateHosts = opts.allowPrivateHosts ?? privateFeedHostAllowlist(); + const allowPrivateHosts = opts.allowPrivateHosts ?? []; const operationDeadlineAt = Date.now() + timeoutMs; const timeoutSignal = AbortSignal.timeout(timeoutMs); const operationSignal = opts.signal