Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions apps/api/src/services/__tests__/extension-store.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,26 @@ describe("resolveExtensionManifest (real safeFetchJson SSRF path)", () => {
});
});

describe("extension fetches ignore the private feed mirror allowlist", () => {
afterEach(() => vi.unstubAllEnvs());

it("refuses a manifest host the operator allowed as a private feed mirror", async () => {
vi.stubEnv("OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS", "feeds.example.test");
lookupMock.mockResolvedValue([{ address: "10.0.0.5", family: 4 }]);
const fetchMock = stubFetchSequence();

await expect(
fetchManifestMeta("https://feeds.example.test/extension.json"),
).resolves.toBeNull();
await expect(
resolveExtensionManifest(
entry({ id: "e", name: "E", version: "1", manifest: "https://feeds.example.test/e.json" }),
),
).rejects.toThrow(/not allowed \(private/);
expect(fetchMock).not.toHaveBeenCalled();
});
});

describe("immutable verified trust", () => {
it("treats only an exact 40-hex commit path as immutable", () => {
expect(
Expand Down
51 changes: 27 additions & 24 deletions apps/api/src/services/extension-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,15 @@ const EXTENSION_REVOCATION_FEED_URL = envString(
"https://raw.githubusercontent.com/openmapx/community-extensions/main/revocations.json",
);

// Shared by every catalog, manifest and revocation request. Their URLs come
// from catalog content, so private hosts stay refused even when the operator
// allows private feed mirrors; that setting covers feed downloads only.
const EXTENSION_FETCH_OPTIONS = {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
allowPrivateHosts: [] as string[],
};

const COMMIT_PATH_SEGMENT = /(^|\/)[a-f0-9]{40}(\/|$)/;

/**
Expand Down Expand Up @@ -145,10 +154,10 @@ interface RevocationSnapshot {
async function fetchRevocations(): Promise<{ snapshot: RevocationSnapshot; stale: boolean }> {
const cached = await readCachedRevocations();
try {
const data = await safeFetchJson<unknown>(EXTENSION_REVOCATION_FEED_URL, {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
});
const data = await safeFetchJson<unknown>(
EXTENSION_REVOCATION_FEED_URL,
EXTENSION_FETCH_OPTIONS,
);
// Validate the whole document before using any of it. A document of the
// wrong shape is a failed refresh, not evidence that nothing is revoked, and
// it must not replace the last good snapshot.
Expand Down Expand Up @@ -401,10 +410,7 @@ async function fetchCatalogFromUrl(url: string): Promise<FetchedCatalog> {
} catch {
throw new ExtensionStoreError("catalog_acquisition_failed");
}
const data = await safeFetchJson<unknown>(canonical, {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
});
const data = await safeFetchJson<unknown>(canonical, EXTENSION_FETCH_OPTIONS);
// Accept either a bare array of entries or `{ extensions, removed, critical }`.
// Validate the whole document before consuming it: malformed acquisition is
// not evidence of an empty catalog, and must not clear cached denials.
Expand Down Expand Up @@ -440,10 +446,10 @@ export async function fetchManifestMeta(
url: string,
): Promise<{ version?: string; platform?: string } | null> {
try {
const data = await safeFetchJson<{ version?: unknown; platform?: unknown }>(url, {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
});
const data = await safeFetchJson<{ version?: unknown; platform?: unknown }>(
url,
EXTENSION_FETCH_OPTIONS,
);
return {
version: typeof data.version === "string" && data.version ? data.version : undefined,
platform: typeof data.platform === "string" && data.platform ? data.platform : undefined,
Expand Down Expand Up @@ -642,10 +648,10 @@ export async function resolveExtensionManifest(
// Verify the digest over the exact received bytes BEFORE they are parsed or
// cached. Parsing first would expose the parser to unverified content and
// would let a cache hold bytes that were never authorized.
const text = await safeFetchText(normalizeExtensionLocator(entry.manifest), {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
});
const text = await safeFetchText(
normalizeExtensionLocator(entry.manifest),
EXTENSION_FETCH_OPTIONS,
);
const digest = createHash("sha256").update(text, "utf8").digest("hex");
if (digest !== entry.manifestSha256.toLowerCase()) {
throw new ExtensionStoreError("digest_mismatch");
Expand All @@ -655,10 +661,10 @@ export async function resolveExtensionManifest(
if (entry.trust === "verified") {
throw new ExtensionStoreError("missing_digest");
}
raw = await safeFetchJson<unknown>(normalizeExtensionLocator(entry.manifest), {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
});
raw = await safeFetchJson<unknown>(
normalizeExtensionLocator(entry.manifest),
EXTENSION_FETCH_OPTIONS,
);
} else {
raw = {
id: entry.id,
Expand Down Expand Up @@ -711,10 +717,7 @@ export function normalizeExtensionLocator(url: string): string {

/** Direct content has no catalog authorization to bind against or trust to inherit. */
export async function resolveDirectExtensionManifest(url: string): Promise<ExtensionManifest> {
const raw = await safeFetchJson<unknown>(normalizeExtensionLocator(url), {
headers: { "User-Agent": USER_AGENT_ADMIN },
timeoutMs: 15_000,
});
const raw = await safeFetchJson<unknown>(normalizeExtensionLocator(url), EXTENSION_FETCH_OPTIONS);
return validatedManifest(raw);
}

Expand Down
1 change: 1 addition & 0 deletions docs/docs/install/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ required pieces of this group are listed under [Core / required](#core--required
| `DATA_MANAGER_URL` | Base URL `app-api` uses to reach the data-manager. Compose overrides this to the service-network DNS name automatically. | Default `http://localhost:4000` (`pnpm dev`); compose sets `http://data-manager:4000` |
| `DATA_MANAGER_PLAINTEXT_HOSTS` | Comma-separated hostnames that may be reached over plain `http://` in addition to `localhost`, `127.0.0.1`, `::1` and `data-manager`. `DATA_MANAGER_URL` must otherwise be `https://`; credentials, path prefixes and query strings are always rejected. | Unset |
| `OPENMAPX_FETCH_JSON_MAX_BYTES` | Ceiling in bytes for upstream JSON responses fetched by integrations that do not set their own limit. Lower it to tighten memory use, or raise it for a large regional feed; a raised value is logged once at first use. | 8 MiB |
| `OPENMAPX_ALLOW_PRIVATE_FEED_HOSTS` | Comma-separated hostnames (exact or `*.suffix`) that transit and mobility feed downloads may reach even though they resolve to a private address, for feed mirrors on a LAN. It applies to feed downloads only; the extension store, health checks and other server-side fetches still refuse private addresses. | Unset |
| `OPENMAPX_RELEASE` | Select `main`, `preview`, an exact prerelease such as `v0.1.0-alpha.1`, a metadata `sha256:<64hex>` digest, or explicit `local` opt-out. Missing selections fail; stable/latest are unavailable. Existing valid overlays remain deliberate pins until reselected or cleared. | `main` |
| `OPENMAPX_RELEASE_MANIFEST_REPOSITORY` | Trusted manifest repository without tag/digest. Its namespace must contain every one of the six images; forwarded to the operations agent. | `ghcr.io/openmapx/release-manifest` |
| `OPENMAPX_ENABLED_SERVICES` | Pin a specific service selection (comma- or space-separated) for CI/repeatable deploys. Wins over the `service-selection.json` written by the admin UI / CLI. | Optional. Default: `service-selection.json` |
Expand Down
107 changes: 107 additions & 0 deletions integrations/bike-sharing/__tests__/citybikes-client.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { searchCityBikes } from "../providers/citybikes-client.js";
import { memoryCache } from "./fixtures.js";

const bbox = { west: 13.3, south: 52.4, east: 13.5, north: 52.6 };
const location = { longitude: 13.4, latitude: 52.5, city: "Berlin", country: "DE" };
const INDEX_CACHE_KEY = "shared-mobility:citybikes:networks";

function network(id: string, href: unknown) {
return { id, name: id, href, company: ["Operator"], location };
}

function detail(id: string) {
return {
network: {
id,
name: id,
company: ["Operator"],
stations: [
{
id: "one",
name: "Station",
longitude: 13.4,
latitude: 52.5,
free_bikes: 2,
empty_slots: 5,
},
],
},
};
}

function stubUpstream(networks: unknown[]) {
const http = vi.fn(async (input: unknown) => {
const url = new URL(String(input));
const body = url.search.includes("fields=")
? { networks }
: detail(url.pathname.split("/").pop() ?? "");
return new Response(JSON.stringify(body), {
headers: { "content-type": "application/json" },
});
});
vi.stubGlobal("fetch", http);
return http;
}

function requestedUrls(http: ReturnType<typeof vi.fn>): string[] {
return http.mock.calls.map(([input]) => String(input));
}

describe("CityBikes network detail links", () => {
beforeEach(() => {
vi.spyOn(console, "warn").mockImplementation(() => {});
});
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});

it("fetches only network detail paths on the CityBikes origin", async () => {
const http = stubUpstream([
network("at-sign", "@other.example/v2/networks/a"),
network("dot", ".other.example/v2/networks/a"),
network("scheme-relative", "//other.example/v2/networks/a"),
network("absolute", "https://other.example/v2/networks/a"),
network("query", "/v2/networks/a?x=1"),
network("other-path", "/v2/other"),
network("not-a-string", 42),
network("berlin", "/v2/networks/berlin"),
]);

const stations = await searchCityBikes(bbox, memoryCache());

expect(stations.map((s) => s.sources)).toEqual([["citybikes/berlin"]]);
const urls = requestedUrls(http);
expect(urls).toHaveLength(2);
expect(urls[1]).toBe("https://api.citybik.es/v2/networks/berlin");
for (const url of urls) expect(new URL(url).origin).toBe("https://api.citybik.es");
});

it("does not cache index entries with unusable detail links", async () => {
stubUpstream([
network("absolute", "https://other.example/v2/networks/a"),
network("berlin", "/v2/networks/berlin"),
]);
const cache = memoryCache();

await searchCityBikes(bbox, cache);

const cached = await cache.get<{ id: string }[]>(INDEX_CACHE_KEY);
expect(cached?.map((n) => n.id)).toEqual(["berlin"]);
});

it("skips an unusable entry already held in the index cache", async () => {
const http = stubUpstream([]);
const cache = memoryCache();
await cache.set(INDEX_CACHE_KEY, [
network("absolute", "https://other.example/v2/networks/a"),
network("berlin", "/v2/networks/berlin"),
]);

const stations = await searchCityBikes(bbox, cache);

expect(stations.map((s) => s.sources)).toEqual([["citybikes/berlin"]]);
expect(requestedUrls(http)).toEqual(["https://api.citybik.es/v2/networks/berlin"]);
});
});
Original file line number Diff line number Diff line change
@@ -1,31 +1,8 @@
import type { CacheClient } from "@openmapx/mobility-core/cache";
import { afterEach, expect, it, vi } from "vitest";
import { searchCityBikes } from "../providers/citybikes-client.js";
import { searchNextbike } from "../providers/nextbike-client.js";
import { memoryCache } from "./fixtures.js";

function memoryCache(): CacheClient {
const values = new Map<string, unknown>();
return {
get: async <T>(key: string) => (values.get(key) as T) ?? null,
set: async (key, value) => {
values.set(key, structuredClone(value));
},
del: async (key) => {
values.delete(key);
},
withCache: async function <T>(
key: string,
_ttl: number,
load: (signal: AbortSignal) => Promise<T>,
) {
const previous = await this.get<T>(key);
if (previous !== null) return previous;
const data = await load(new AbortController().signal);
await this.set(key, data);
return data;
},
};
}
const bbox = { west: 13.3, south: 52.4, east: 13.5, north: 52.6 };
afterEach(() => vi.unstubAllGlobals());

Expand Down
26 changes: 26 additions & 0 deletions integrations/bike-sharing/__tests__/fixtures.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import type { CacheClient } from "@openmapx/mobility-core/cache";

/** In-memory read-through cache with the host's clone-on-write semantics. */
export function memoryCache(): CacheClient {
const values = new Map<string, unknown>();
return {
get: async <T>(key: string) => (values.get(key) as T) ?? null,
set: async (key, value) => {
values.set(key, structuredClone(value));
},
del: async (key) => {
values.delete(key);
},
withCache: async function <T>(
key: string,
_ttl: number,
load: (signal: AbortSignal) => Promise<T>,
) {
const previous = await this.get<T>(key);
if (previous !== null) return previous;
const data = await load(new AbortController().signal);
await this.set(key, data);
return data;
},
};
}
52 changes: 50 additions & 2 deletions integrations/bike-sharing/providers/citybikes-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import { type CacheClient, TTL, withCache } from "@openmapx/mobility-core/cache"
import type { SharedMobilityStation } from "@openmapx/mobility-core/shared-mobility";

const CITYBIKES_BASE = "https://api.citybik.es";
const CITYBIKES_ORIGIN = new URL(CITYBIKES_BASE).origin;
const NETWORK_PATH = /^\/v2\/networks\/[A-Za-z0-9][A-Za-z0-9._-]*$/;
const HEADERS = {
Accept: "application/json",
};
Expand Down Expand Up @@ -68,6 +70,32 @@ async function fetchJson<T>(url: string): Promise<T> {
});
}

/**
* Resolve a network's `href` from the upstream index to a detail URL. Only a
* network detail path on the CityBikes origin is accepted; anything else
* returns null so the network is skipped rather than fetched.
*/
function cityBikesNetworkUrl(href: unknown): URL | null {
if (typeof href !== "string") return null;
let url: URL;
try {
url = new URL(href, CITYBIKES_BASE);
} catch {
return null;
}
if (
url.origin !== CITYBIKES_ORIGIN ||
url.username ||
url.password ||
url.search ||
url.hash ||
!NETWORK_PATH.test(url.pathname)
) {
return null;
}
return url;
}

/** Cached index of all CityBikes networks. */
async function getNetworkIndex(cache: CacheClient): Promise<CityBikesNetwork[]> {
return withCache<CityBikesNetwork[]>(
Expand All @@ -78,7 +106,20 @@ async function getNetworkIndex(cache: CacheClient): Promise<CityBikesNetwork[]>
const data = await fetchJson<{ networks: CityBikesNetwork[] }>(
`${CITYBIKES_BASE}/v2/networks?fields=id,name,href,company,location`,
);
return data.networks;
// Drop entries whose detail link leaves the CityBikes API before they are
// cached, so one bad entry cannot be reused for the whole cache lifetime.
const skipped: string[] = [];
const networks = data.networks.filter((network) => {
const usable = typeof network.id === "string" && cityBikesNetworkUrl(network.href) !== null;
if (!usable) skipped.push(String(network.id));
return usable;
});
if (skipped.length > 0) {
console.warn(
`[citybikes] skipped ${skipped.length} network(s) with an unusable detail link: ${skipped.slice(0, 20).join(", ")}`,
);
}
return networks;
},
);
}
Expand Down Expand Up @@ -110,13 +151,20 @@ export async function fetchNetworkStations(
cache: CacheClient,
contributors?: readonly Attribution[],
): Promise<SharedMobilityStation[]> {
// Checked again here because an index cached before this check existed may
// still hold entries that were never validated.
const detailUrl = cityBikesNetworkUrl(network.href);
if (!detailUrl) {
console.warn(`[citybikes] skipped network ${String(network.id)}: unusable detail link`);
return [];
}
const cacheKey = `shared-mobility:citybikes:contributors-v1:${network.id}`;
const stations = await withCache<SharedMobilityStation[]>(
cache,
cacheKey,
TTL.sharedMobility.stations,
async () => {
const data = await fetchJson<CityBikesNetworkDetail>(`${CITYBIKES_BASE}${network.href}`);
const data = await fetchJson<CityBikesNetworkDetail>(detailUrl.href);
const operator = network.company?.[0] ?? network.name;

return data.network.stations.map(
Expand Down
Loading
Loading