From 45bc2f03c3f31d45b1d9125cf7da8b857fb5e5b8 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 10:42:12 +0100 Subject: [PATCH 1/2] fix(api): run one privacy export per account at a time Any number of exports for one account could be prepared and streamed at once. An account now holds one export slot, the process allows four in total, and the route joins the expensive rate-limit class. A retry after an interrupted download is accepted once the earlier attempt has settled. --- apps/api/src/privacy/export-admission.ts | 47 ++++ apps/api/src/privacy/export-response.test.ts | 30 +++ apps/api/src/privacy/export-response.ts | 4 + .../privacy-export-admission.test.ts | 252 ++++++++++++++++++ apps/api/src/routes/privacy-requests.ts | 31 ++- apps/api/src/server-wiring.test.ts | 2 + apps/api/src/server-wiring.ts | 4 +- .../auth/PrivacyDataSection.test.tsx | 22 ++ .../components/auth/PrivacyDataSection.tsx | 16 ++ docs/docs/features/accounts-and-privacy.md | 2 + docs/docs/install/configuration.md | 2 +- packages/core/src/api/index.ts | 2 + packages/core/src/api/privacy.ts | 10 +- packages/i18n/locales/de.json | 2 + packages/i18n/locales/en.json | 2 + 15 files changed, 422 insertions(+), 6 deletions(-) create mode 100644 apps/api/src/privacy/export-admission.ts create mode 100644 apps/api/src/privacy/export-response.test.ts create mode 100644 apps/api/src/routes/__tests__/privacy-export-admission.test.ts diff --git a/apps/api/src/privacy/export-admission.ts b/apps/api/src/privacy/export-admission.ts new file mode 100644 index 000000000..bbf0411ca --- /dev/null +++ b/apps/api/src/privacy/export-admission.ts @@ -0,0 +1,47 @@ +import { createHash } from "node:crypto"; +import { RollingStreamBudget, type RollingStreamBudgetLease } from "@openmapx/core/server"; +import { PrivacyRequestError } from "./errors.js"; + +/** Exports this API process prepares and streams at once, across all accounts. */ +const MAX_CONCURRENT_EXPORTS = 4; + +/** + * Admission for self-service exports: one export per account at a time and a + * small ceiling for the whole process. Collecting an account's data and + * streaming the archive is the most expensive thing a signed-in person can + * ask for, so a second export for the same account is refused while the first + * is still running. A retry after an interrupted download is accepted as soon + * as the earlier attempt has settled. + * + * The slots live in this process only. A restart frees them together with the + * streams they guarded; with several API replicas each replica counts its own. + * The byte budget of the underlying lease is unused. + */ +export function createPrivacyExportAdmission( + maxConcurrent = MAX_CONCURRENT_EXPORTS, +): RollingStreamBudget { + return new RollingStreamBudget({ + maxConcurrentPerKey: 1, + maxConcurrentGlobal: maxConcurrent, + maxBytesPerKey: Number.MAX_SAFE_INTEGER, + windowMs: 60_000, + }); +} + +export const privacyExportAdmission = createPrivacyExportAdmission(); + +/** + * Take the export slot for an account, or throw the refusal the route maps to + * its response: 409 `EXPORT_IN_PROGRESS` while the account already holds one, + * 503 `PRIVACY_SERVICE_UNAVAILABLE` while the process is at its ceiling. The + * account id is digested so the slot map holds no account identifier. + */ +export function acquireExportSlot( + admission: RollingStreamBudget, + userId: string, +): RollingStreamBudgetLease { + const result = admission.tryAcquire(createHash("sha256").update(userId).digest("hex")); + if (result.allowed) return result.lease; + if (result.reason === "key-concurrency") throw new PrivacyRequestError("EXPORT_IN_PROGRESS", 409); + throw new PrivacyRequestError("PRIVACY_SERVICE_UNAVAILABLE", 503); +} diff --git a/apps/api/src/privacy/export-response.test.ts b/apps/api/src/privacy/export-response.test.ts new file mode 100644 index 000000000..438f652cf --- /dev/null +++ b/apps/api/src/privacy/export-response.test.ts @@ -0,0 +1,30 @@ +import { PassThrough, type Writable } from "node:stream"; +import { describe, expect, it, vi } from "vitest"; +import { pumpExport } from "./export-response.js"; + +describe("pumpExport", () => { + it("ends the archive when the response closed before streaming began", async () => { + const response = new PassThrough(); + response.destroy(); + await new Promise((resolve) => setImmediate(resolve)); + let sink: Writable | undefined; + const dispose = vi.fn(async () => {}); + const { delivered } = pumpExport( + { + filename: "export.zip", + // Waits for the sink to fail, as the real archive writer does. + write: (target) => + new Promise((_resolve, reject) => { + sink = target; + if (target.destroyed) reject(new Error("sink closed")); + else target.once("close", () => reject(new Error("sink closed"))); + }), + dispose, + }, + response, + ); + expect(await delivered).toBe(false); + expect(sink?.destroyed).toBe(true); + expect(dispose).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/api/src/privacy/export-response.ts b/apps/api/src/privacy/export-response.ts index 9de845927..3e26c35bf 100644 --- a/apps/api/src/privacy/export-response.ts +++ b/apps/api/src/privacy/export-response.ts @@ -30,6 +30,10 @@ export function pumpExport( response.once("close", () => { if (!response.writableFinished) body.destroy(new Error("export client disconnected")); }); + // A client that left while the export was being prepared closed the + // response before the listener above existed; end the archive now. + if (response.destroyed && !response.writableFinished) + body.destroy(new Error("export client disconnected")); const written = prepared.write(body); const delivered = Promise.allSettled([written, finished(response)]).then( async ([archive, sent]) => { diff --git a/apps/api/src/routes/__tests__/privacy-export-admission.test.ts b/apps/api/src/routes/__tests__/privacy-export-admission.test.ts new file mode 100644 index 000000000..d4421a958 --- /dev/null +++ b/apps/api/src/routes/__tests__/privacy-export-admission.test.ts @@ -0,0 +1,252 @@ +import { request as httpRequest } from "node:http"; +import type { AddressInfo } from "node:net"; +import type { Writable } from "node:stream"; +import Fastify, { type FastifyInstance } from "fastify"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const { getSession, prepareSubjectExport } = vi.hoisted(() => ({ + getSession: vi.fn(), + prepareSubjectExport: vi.fn(), +})); + +vi.mock("../../auth.js", () => ({ auth: { api: { getSession } } })); +vi.mock("../../db/index.js", () => ({ db: {} })); +vi.mock("../../utils/require-auth.js", () => ({ + requireAuthHook: vi.fn(async (request: { headers: Record; userId?: string }) => { + request.userId = String(request.headers["x-test-user"]); + }), + getUserId: (request: { userId?: string }) => request.userId, +})); +vi.mock("../../privacy/session-assurance.js", () => ({ hasFreshAssurance: async () => true })); +vi.mock("../../privacy/request-service.js", () => ({ PrivacyRequestService: class {} })); +vi.mock("../../privacy/subject-export.js", () => ({ prepareSubjectExport })); + +import { createPrivacyExportAdmission } from "../../privacy/export-admission.js"; +import { privacyRequestsRoute } from "../privacy-requests.js"; + +const PAYLOAD = { locale: "en", timeZone: "UTC" }; + +/** An export that writes its first bytes and then waits until told to finish. + * Like the real one, it rejects once the response goes away. */ +function heldExport() { + let finish!: () => void; + const gate = new Promise((resolve) => { + finish = resolve; + }); + const prepared = { + filename: "openmapx-data-export.zip", + async write(sink: Writable) { + if (sink.destroyed) throw new Error("sink closed"); + const closed = new Promise((_resolve, reject) => + sink.once("close", () => reject(new Error("sink closed"))), + ); + closed.catch(() => undefined); + sink.write("PK"); + await Promise.race([gate, closed]); + sink.end(); + }, + dispose: vi.fn(async () => {}), + }; + return { prepared, finish }; +} + +function instantExport() { + return { + filename: "openmapx-data-export.zip", + async write(sink: Writable) { + sink.end("PK"); + }, + dispose: async () => {}, + }; +} + +function fakeService() { + return { + beginSelfServiceExport: vi.fn(async (input: { userId: string }) => ({ + id: `request-${input.userId}`, + userId: input.userId, + })), + finishExport: vi.fn(async () => ({})), + recordExportFailure: vi.fn(async () => {}), + }; +} + +let app: FastifyInstance | undefined; + +async function exportApp(maxConcurrent?: number) { + const service = fakeService(); + app = Fastify(); + await app.register(privacyRequestsRoute, { + prefix: "/api", + // The route only calls the export methods on the service. + service: service as never, + exportAdmission: createPrivacyExportAdmission(maxConcurrent), + }); + await app.listen({ port: 0, host: "127.0.0.1" }); + return { app, service, port: (app.server.address() as AddressInfo).port }; +} + +function inject(user: string) { + return (app as FastifyInstance).inject({ + method: "POST", + url: "/api/privacy/export", + headers: { "x-test-user": user }, + payload: PAYLOAD, + }); +} + +/** Start an export over a real socket; resolves once the first bytes arrive. */ +function startStreaming(port: number, user: string, onSent?: () => void) { + const outgoing = httpRequest({ + host: "127.0.0.1", + port, + method: "POST", + path: "/api/privacy/export", + headers: { "content-type": "application/json", "x-test-user": user }, + }); + outgoing.on("error", () => undefined); + const firstBytes = new Promise<{ status: number; rest: Promise; abort: () => void }>( + (resolve) => { + outgoing.on("response", (incoming) => { + incoming.on("error", () => undefined); + let text = ""; + const rest = new Promise((done) => { + incoming.on("end", () => done(text)); + }); + incoming.once("data", (chunk: Buffer) => { + text += chunk.toString(); + incoming.on("data", (more: Buffer) => { + text += more.toString(); + }); + resolve({ + status: incoming.statusCode ?? 0, + rest, + abort: () => { + incoming.destroy(); + outgoing.destroy(); + }, + }); + }); + }); + }, + ); + outgoing.end(JSON.stringify(PAYLOAD), onSent); + return { firstBytes, outgoing }; +} + +getSession.mockImplementation(async ({ headers }: { headers: Headers }) => { + const id = headers.get("x-test-user"); + return { user: { id, twoFactorEnabled: false }, session: { id: `session-${id}` } }; +}); + +afterEach(async () => { + await app?.close(); + app = undefined; + prepareSubjectExport.mockReset(); +}); + +describe("POST /api/privacy/export admission", () => { + it("refuses a second export for the same account while the first is streaming, then accepts a retry", async () => { + const { service, port } = await exportApp(); + const held = heldExport(); + prepareSubjectExport.mockResolvedValueOnce(held.prepared); + const first = await startStreaming(port, "alice").firstBytes; + expect(first.status).toBe(200); + + const second = await inject("alice"); + expect(second.statusCode).toBe(409); + expect(second.json()).toEqual({ code: "EXPORT_IN_PROGRESS" }); + expect(service.beginSelfServiceExport).toHaveBeenCalledTimes(1); + expect(service.recordExportFailure).not.toHaveBeenCalled(); + + held.finish(); + expect(await first.rest).toBe("PK"); + await vi.waitFor(() => expect(service.finishExport).toHaveBeenCalledTimes(1)); + + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + await vi.waitFor(async () => expect((await inject("alice")).statusCode).toBe(200)); + }); + + it("does not block another account", async () => { + const { port } = await exportApp(); + const held = heldExport(); + prepareSubjectExport.mockResolvedValueOnce(held.prepared); + const alice = await startStreaming(port, "alice").firstBytes; + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + expect((await inject("bob")).statusCode).toBe(200); + held.finish(); + await alice.rest; + }); + + it("asks for a later retry once the process ceiling is reached", async () => { + const { service, port } = await exportApp(1); + const held = heldExport(); + prepareSubjectExport.mockResolvedValueOnce(held.prepared); + const alice = await startStreaming(port, "alice").firstBytes; + const bob = await inject("bob"); + expect(bob.statusCode).toBe(503); + expect(bob.json()).toEqual({ code: "PRIVACY_SERVICE_UNAVAILABLE" }); + expect(service.beginSelfServiceExport).toHaveBeenCalledTimes(1); + held.finish(); + await alice.rest; + }); + + it("frees the slot after preparation fails", async () => { + const { service } = await exportApp(); + prepareSubjectExport.mockRejectedValueOnce(new Error("source down")); + const failed = await inject("alice"); + expect(failed.statusCode).toBe(503); + expect(service.recordExportFailure).toHaveBeenCalledWith( + "request-alice", + "export-preparation-failed", + ); + + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + expect((await inject("alice")).statusCode).toBe(200); + }); + + it("frees the slot after the client aborts the download", async () => { + const { service, port } = await exportApp(); + const held = heldExport(); + prepareSubjectExport.mockResolvedValueOnce(held.prepared); + const first = await startStreaming(port, "alice").firstBytes; + first.abort(); + await vi.waitFor(() => + expect(service.recordExportFailure).toHaveBeenCalledWith( + "request-alice", + "export-interrupted", + ), + ); + expect(service.finishExport).not.toHaveBeenCalled(); + expect(held.prepared.dispose).toHaveBeenCalled(); + + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + await vi.waitFor(async () => expect((await inject("alice")).statusCode).toBe(200)); + }); + + it("frees the slot when the client leaves while the export is being prepared", async () => { + const { service, port } = await exportApp(); + const held = heldExport(); + let release!: (value: unknown) => void; + prepareSubjectExport.mockReturnValueOnce( + new Promise((resolve) => { + release = resolve; + }), + ); + const { outgoing } = startStreaming(port, "alice"); + await vi.waitFor(() => expect(prepareSubjectExport).toHaveBeenCalledTimes(1)); + outgoing.destroy(); + // Give the server a moment to see the closed socket before preparation ends. + await new Promise((resolve) => setTimeout(resolve, 50)); + release(held.prepared); + + await vi.waitFor(() => + expect(service.recordExportFailure).toHaveBeenCalledWith( + "request-alice", + "export-interrupted", + ), + ); + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + await vi.waitFor(async () => expect((await inject("alice")).statusCode).toBe(200)); + }); +}); diff --git a/apps/api/src/routes/privacy-requests.ts b/apps/api/src/routes/privacy-requests.ts index 7c3e44473..d152c4d64 100644 --- a/apps/api/src/routes/privacy-requests.ts +++ b/apps/api/src/routes/privacy-requests.ts @@ -1,3 +1,4 @@ +import type { RollingStreamBudget, RollingStreamBudgetLease } from "@openmapx/core/server"; import { fromNodeHeaders } from "better-auth/node"; import { eq } from "drizzle-orm"; import type { FastifyPluginAsync } from "fastify"; @@ -6,6 +7,7 @@ import { auth } from "../auth.js"; import { db } from "../db/index.js"; import { dataSubjectRequestTask } from "../db/schema.js"; import { PrivacyRequestError, privacyErrorReply } from "../privacy/errors.js"; +import { acquireExportSlot, privacyExportAdmission } from "../privacy/export-admission.js"; import { exportResponseHeaders, pumpExport } from "../privacy/export-response.js"; import { selfServiceExportSchema } from "../privacy/request-contracts.js"; import { PrivacyRequestService } from "../privacy/request-service.js"; @@ -26,6 +28,8 @@ export interface PrivacyRequestsRouteOptions { database?: typeof db; /** Collector settings for the export; tests replace the external sources. */ exportSources?: Pick; + /** Export slots; tests pass their own so cases stay independent. */ + exportAdmission?: RollingStreamBudget; } function publicRequest(row: Record): Record { @@ -76,6 +80,7 @@ export const privacyRequestsRoute: FastifyPluginAsync options.service ?? new PrivacyRequestService({ database }); + const exportAdmission = options.exportAdmission ?? privacyExportAdmission; /** * The self-service export. A recent interactive sign-in is the identity @@ -89,6 +94,7 @@ export const privacyRequestsRoute: FastifyPluginAsync undefined); + slot?.release(); return privacyErrorReply(reply, error); } const exportRequestId = requestId; - const { body, delivered } = pumpExport(prepared, reply.raw); + const heldSlot = slot; + let delivered: Promise; + let body: ReturnType["body"]; + try { + ({ body, delivered } = pumpExport(prepared, reply.raw)); + } catch (error) { + // Nothing was sent yet; without this the slot would stay taken. + await prepared.dispose().catch(() => undefined); + await service + .recordExportFailure(exportRequestId, "export-preparation-failed") + .catch(() => undefined); + heldSlot.release(); + return privacyErrorReply(reply, error); + } // After the headers the request stays in progress unless every byte - // arrives, so an interrupted download can simply be retried. + // arrives, so an interrupted download can simply be retried. The slot is + // freed only once the outcome is recorded, so a retry never overlaps the + // earlier attempt's bookkeeping. void delivered .then(async (ok) => { if (ok) await service.finishExport(exportRequestId, { kind: "subject", id: userId }); @@ -134,7 +158,8 @@ export const privacyRequestsRoute: FastifyPluginAsync request.log.error({ err: error }, "failed to record the privacy export outcome"), - ); + ) + .finally(() => heldSlot.release()); reply.headers(exportResponseHeaders(prepared.filename)); return reply.send(body); }); diff --git a/apps/api/src/server-wiring.test.ts b/apps/api/src/server-wiring.test.ts index 97cde593c..cc06ac0de 100644 --- a/apps/api/src/server-wiring.test.ts +++ b/apps/api/src/server-wiring.test.ts @@ -369,6 +369,7 @@ describe("makeRateLimitTierHook", () => { app.get("/api/offline/packages/glyphs/:version/:fontstack/:range.pbf", ok); app.get("/api/offline/packages/:packageId/archive", ok); app.post("/api/offline/packages/prepare", ok); + app.post("/api/privacy/export", ok); app.get("/api/isochrone", ok); app.get("/api/places/:id", ok); app.get("/api/timeline/day/:date", ok); @@ -389,6 +390,7 @@ describe("makeRateLimitTierHook", () => { { url: "/api/offline/packages/glyphs/glyphs-v1/Noto%20Sans/0-255.pbf", tier: "tile" }, { url: `/api/offline/packages/omp2-${"a".repeat(64)}/archive`, tier: "public" }, { method: "POST", url: "/api/offline/packages/prepare", tier: "expensive" }, + { method: "POST", url: "/api/privacy/export", tier: "expensive" }, { url: "/api/isochrone?x=1", tier: "expensive" }, { url: "/api/places/node%2F1", tier: "expensive" }, { diff --git a/apps/api/src/server-wiring.ts b/apps/api/src/server-wiring.ts index df375864c..e2b3c9645 100644 --- a/apps/api/src/server-wiring.ts +++ b/apps/api/src/server-wiring.ts @@ -401,6 +401,7 @@ export const EXPENSIVE_ROUTE_PREFIXES = [ "/api/image-proxy", "/api/winter-sports", "/api/offline/packages/prepare", + "/api/privacy/export", ] as const; export function isTimelineApiRequest(request: FastifyRequest): boolean { @@ -544,7 +545,8 @@ const BODY_CAPABLE_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); // - `/api/auth/*` → strict (credential stuffing, email spam) // - exactly `GET /api/status` → 60/minute (dependency snapshot refresh) // - tile / map asset routes → generous (bursty, cacheable, CDN-friendly) -// - expensive public routes → tight (Valhalla, Overpass, place and image fan-out) +// - expensive public routes → tight (Valhalla, Overpass, place and image fan-out, +// offline package preparation, the personal data export) // - every other matched route → broad floor, including timeline day reads, // which add a per-user bucket after sign-in export function makeRateLimitTierHook(limits: RateLimitTiers) { diff --git a/apps/web/src/components/auth/PrivacyDataSection.test.tsx b/apps/web/src/components/auth/PrivacyDataSection.test.tsx index da2ec3922..513aada27 100644 --- a/apps/web/src/components/auth/PrivacyDataSection.test.tsx +++ b/apps/web/src/components/auth/PrivacyDataSection.test.tsx @@ -139,6 +139,28 @@ it("retries once and then explains the second-factor requirement instead of redi expect(sessionStorage.getItem(PENDING_EXPORT_KEY)).toBeNull(); }); +it("explains that another download for the account is still running", async () => { + getPrivacyDataRequests.mockResolvedValue({ requests: [] }); + requestPrivacyExport.mockResolvedValue( + Response.json({ code: "EXPORT_IN_PROGRESS" }, { status: 409 }), + ); + render(); + await screen.findByText("privacyData.noRequests"); + fireEvent.click(screen.getByRole("button", { name: "privacyData.downloadExport" })); + expect(await screen.findByRole("alert")).toHaveTextContent("privacyData.exportInProgress"); +}); + +it("asks the person to try again later when the server cannot start a download", async () => { + getPrivacyDataRequests.mockResolvedValue({ requests: [] }); + requestPrivacyExport.mockResolvedValue( + Response.json({ code: "PRIVACY_SERVICE_UNAVAILABLE" }, { status: 503 }), + ); + render(); + await screen.findByText("privacyData.noRequests"); + fireEvent.click(screen.getByRole("button", { name: "privacyData.downloadExport" })); + expect(await screen.findByRole("alert")).toHaveTextContent("privacyData.exportUnavailable"); +}); + it("does not retry an intent that was already retried", async () => { getPrivacyDataRequests.mockResolvedValue({ requests: [] }); sessionStorage.setItem(PENDING_EXPORT_KEY, JSON.stringify({ at: Date.now(), retried: true })); diff --git a/apps/web/src/components/auth/PrivacyDataSection.tsx b/apps/web/src/components/auth/PrivacyDataSection.tsx index 3f2c1c139..8f107c421 100644 --- a/apps/web/src/components/auth/PrivacyDataSection.tsx +++ b/apps/web/src/components/auth/PrivacyDataSection.tsx @@ -14,8 +14,10 @@ import ListItemIcon from "@mui/material/ListItemIcon"; import ListItemText from "@mui/material/ListItemText"; import Typography from "@mui/material/Typography"; import { + EXPORT_IN_PROGRESS, FRESH_SIGN_IN_REQUIRED, getPrivacyDataRequests, + PRIVACY_SERVICE_UNAVAILABLE, type PrivacyRequestView, requestPrivacyExport, withdrawPrivacyDataRequest, @@ -155,6 +157,20 @@ export const PrivacyDataSection = forwardRef< return; } } + if (response.status === 409 || response.status === 503) { + const body = (await response.json().catch(() => null)) as { code?: string } | null; + const message = + body?.code === EXPORT_IN_PROGRESS + ? t("privacyData.exportInProgress") + : body?.code === PRIVACY_SERVICE_UNAVAILABLE + ? t("privacyData.exportUnavailable") + : null; + if (message) { + if (retry) writePendingExport(null); + if (mounted.current) setError(message); + return; + } + } if (!response.ok) throw new Error("export failed"); await saveResponse(response); if (retry) writePendingExport(null); diff --git a/docs/docs/features/accounts-and-privacy.md b/docs/docs/features/accounts-and-privacy.md index 020008e52..2dac4c710 100644 --- a/docs/docs/features/accounts-and-privacy.md +++ b/docs/docs/features/accounts-and-privacy.md @@ -49,6 +49,8 @@ See [Reviews](./reviews.md) for key export and recovery limits. and then continues the download. 3. The archive is created on demand and downloaded straight to your device. Nothing is kept on the server, so you can download again at any time. + One download per account runs at a time; a second one started while the + first is still running is refused until it finishes. 4. The request stays listed with its status and response deadline. Some sources can only be checked by the operator; the archive's report names them, and the request is completed once the operator has checked them. diff --git a/docs/docs/install/configuration.md b/docs/docs/install/configuration.md index fad7e5c3d..ba795e57a 100644 --- a/docs/docs/install/configuration.md +++ b/docs/docs/install/configuration.md @@ -540,7 +540,7 @@ workload; a manifest default is not a hardware sizing guarantee. | `RATE_LIMIT_PUBLIC_WINDOW_MS` | Refill window for the public budget. | Default `60000` (1 min) | | `RATE_LIMIT_TILE_MAX` | Per-client budget for map tiles, the MapTiler and traffic proxies and offline glyphs, which arrive in bursts of 30 to 60 per viewport change. | Default `1800` | | `RATE_LIMIT_TILE_WINDOW_MS` | Refill window for the tile budget. | Default `60000` (1 min) | -| `RATE_LIMIT_EXPENSIVE_MAX` | Per-client budget for routes that fan out to costly upstreams: place details, image proxy, isochrones, route optimization, EV plans, transit plans, reverse geocoding. | Default `60` | +| `RATE_LIMIT_EXPENSIVE_MAX` | Per-client budget for routes that fan out to costly upstreams: place details, image proxy, isochrones, route optimization, EV plans, transit plans, reverse geocoding, personal data export. | Default `60` | | `RATE_LIMIT_EXPENSIVE_WINDOW_MS` | Refill window for the expensive budget. | Default `60000` (1 min) | | `RATE_LIMIT_AUTH_MAX` | Per-client budget for sign-in, sign-up and other `/api/auth` requests. Better Auth applies its own limits in addition. | Default `10` | | `RATE_LIMIT_AUTH_WINDOW_MS` | Refill window for the auth budget. | Default `60000` (1 min) | diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts index 1a430c972..bfc08fc60 100644 --- a/packages/core/src/api/index.ts +++ b/packages/core/src/api/index.ts @@ -42,8 +42,10 @@ export { type PlaceDetailsRequestInput, } from "./placeDetails"; export { + EXPORT_IN_PROGRESS, getPrivacyDataRequest, getPrivacyDataRequests, + PRIVACY_SERVICE_UNAVAILABLE, type PrivacyRequestDetail, type PrivacyRequestKind, type PrivacyRequestState, diff --git a/packages/core/src/api/privacy.ts b/packages/core/src/api/privacy.ts index ec4972e84..401222b0b 100644 --- a/packages/core/src/api/privacy.ts +++ b/packages/core/src/api/privacy.ts @@ -60,10 +60,18 @@ export function withdrawPrivacyDataRequest(requestId: string, version: number) { ); } +/** The API refuses an export while another export for the same account is still running. */ +export const EXPORT_IN_PROGRESS = "EXPORT_IN_PROGRESS"; + +/** The API cannot start an export right now, for example while the server is busy. */ +export const PRIVACY_SERVICE_UNAVAILABLE = "PRIVACY_SERVICE_UNAVAILABLE"; + /** * Start the self-service export. The response body is the ZIP archive itself, * so the caller reads it as a stream or blob. A 401 with the code - * `FRESH_SIGN_IN_REQUIRED` means the person must sign in again first. + * `FRESH_SIGN_IN_REQUIRED` means the person must sign in again first; a 409 + * with `EXPORT_IN_PROGRESS` means another export for the account is still + * running; a 503 with `PRIVACY_SERVICE_UNAVAILABLE` means it can be retried later. */ export function requestPrivacyExport( kind: PrivacyRequestKind = "access_and_portability", diff --git a/packages/i18n/locales/de.json b/packages/i18n/locales/de.json index 1e549676d..f229c3bfd 100644 --- a/packages/i18n/locales/de.json +++ b/packages/i18n/locales/de.json @@ -2300,6 +2300,8 @@ }, "downloadExport": "Meine Daten herunterladen", "exportFailed": "Das Archiv konnte nicht heruntergeladen werden. Bitte versuchen Sie es erneut.", + "exportInProgress": "Ein Download Ihrer Daten läuft bereits. Bitte warten Sie, bis er abgeschlossen ist, und versuchen Sie es dann erneut.", + "exportUnavailable": "Downloads sind vorübergehend nicht verfügbar. Bitte versuchen Sie es in einigen Minuten erneut.", "secondFactorRequired": "Bitte melden Sie sich erneut mit Ihrem Passwort und dem zweiten Faktor Ihres Kontos an (Authenticator-App oder Wiederherstellungscode). Ein Passkey oder eine externe Anmeldung reicht nicht aus, solange die Zwei-Faktor-Authentifizierung aktiviert ist." }, "mangroveConnectTitle": "Mit Mangrove verbinden", diff --git a/packages/i18n/locales/en.json b/packages/i18n/locales/en.json index 6b3059045..3a284aedb 100644 --- a/packages/i18n/locales/en.json +++ b/packages/i18n/locales/en.json @@ -2300,6 +2300,8 @@ }, "downloadExport": "Download my data", "exportFailed": "The archive could not be downloaded. Please try again.", + "exportInProgress": "A download of your data is already running. Please wait for it to finish, then try again.", + "exportUnavailable": "Downloads are temporarily unavailable. Please try again in a few minutes.", "secondFactorRequired": "Please sign in again with your password and your account's second factor (authenticator app or recovery code). A passkey or external sign-in is not enough to download your data while two-factor authentication is enabled." }, "mangroveConnectTitle": "Connect to Mangrove", From 78867848b45c277ed2fb634c8e78e60234fc5aee Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 10:53:55 +0100 Subject: [PATCH 2/2] fix(api): end a privacy export that stalls or runs too long An export slot was held until the stream settled, with no limit on how long that could take. The download now ends after 60 seconds without the client taking data and after one hour in total, and collecting the managed timeline stops when the client leaves or after 20 minutes. --- apps/api/src/privacy/export-response.test.ts | 126 +++++++++++++++++- apps/api/src/privacy/export-response.ts | 86 +++++++++++- .../privacy-export-admission.test.ts | 68 +++++++++- apps/api/src/routes/privacy-requests.ts | 42 ++++-- docs/docs/features/accounts-and-privacy.md | 3 +- 5 files changed, 308 insertions(+), 17 deletions(-) diff --git a/apps/api/src/privacy/export-response.test.ts b/apps/api/src/privacy/export-response.test.ts index 438f652cf..1118e3516 100644 --- a/apps/api/src/privacy/export-response.test.ts +++ b/apps/api/src/privacy/export-response.test.ts @@ -1,6 +1,52 @@ import { PassThrough, type Writable } from "node:stream"; -import { describe, expect, it, vi } from "vitest"; -import { pumpExport } from "./export-response.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ExportDeadlineError, pumpExport } from "./export-response.js"; + +const CHUNK = Buffer.alloc(16 * 1024, 1); + +/** Writes chunks for as long as the sink accepts them, honouring backpressure, + * and rejects once the sink fails, as the real archive writer does. */ +function endlessExport(pauseMs = 0) { + return { + filename: "export.zip", + dispose: vi.fn(async () => {}), + write: (sink: Writable) => + new Promise((_resolve, reject) => { + let stopped = false; + const stop = () => { + stopped = true; + reject(new Error("sink closed")); + }; + sink.once("close", stop); + sink.once("error", () => undefined); + const next = () => { + if (stopped) return; + if (sink.write(CHUNK)) setTimeout(next, pauseMs); + else sink.once("drain", next); + }; + next(); + }), + }; +} + +function shortExport() { + return { + filename: "export.zip", + dispose: vi.fn(async () => {}), + write: async (sink: Writable) => { + sink.write(CHUNK); + sink.end(CHUNK); + }, + }; +} + +function deadlineOf(body: PassThrough): Promise { + return new Promise((resolve) => body.once("error", resolve)); +} + +afterEach(() => { + vi.useRealTimers(); +}); describe("pumpExport", () => { it("ends the archive when the response closed before streaming began", async () => { @@ -9,7 +55,7 @@ describe("pumpExport", () => { await new Promise((resolve) => setImmediate(resolve)); let sink: Writable | undefined; const dispose = vi.fn(async () => {}); - const { delivered } = pumpExport( + const { body, delivered } = pumpExport( { filename: "export.zip", // Waits for the sink to fail, as the real archive writer does. @@ -23,8 +69,82 @@ describe("pumpExport", () => { }, response, ); + // Fastify listens on the body once it is sent; stand in for it here. + body.on("error", () => undefined); expect(await delivered).toBe(false); expect(sink?.destroyed).toBe(true); expect(dispose).toHaveBeenCalledTimes(1); }); + + it("ends a download whose client stops reading", async () => { + // A response nobody reads: once its small buffer is full, no more bytes move. + const response = new PassThrough({ highWaterMark: 1024 }); + const prepared = endlessExport(); + const { body, delivered } = pumpExport(prepared, response, { idleMs: 50, totalMs: 60_000 }); + const reason = deadlineOf(body); + body.pipe(response); + expect(await delivered).toBe(false); + expect(await reason).toEqual(new ExportDeadlineError("idle")); + expect(response.destroyed).toBe(true); + expect(prepared.dispose).toHaveBeenCalledTimes(1); + }); + + it("ends a download that runs past the longest download time", async () => { + // The client reads every byte, but the archive only trickles in. + const response = new PassThrough(); + response.resume(); + const prepared = endlessExport(5); + const { body, delivered } = pumpExport(prepared, response, { idleMs: 60_000, totalMs: 100 }); + const reason = deadlineOf(body); + body.pipe(response); + expect(await delivered).toBe(false); + expect(await reason).toEqual(new ExportDeadlineError("total")); + expect(prepared.dispose).toHaveBeenCalledTimes(1); + }); + + it("delivers a normal download and leaves no timer behind", async () => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const response = new PassThrough(); + const read = new Promise((resolve) => { + let bytes = 0; + response.on("data", (chunk: Buffer) => { + bytes += chunk.byteLength; + }); + response.once("end", () => resolve(bytes)); + }); + const { body, delivered } = pumpExport(shortExport(), response, { + idleMs: 1_000, + totalMs: 5_000, + }); + // Both deadlines are armed and neither keeps the process alive. + expect(vi.getTimerCount()).toBe(2); + body.pipe(response); + expect(await delivered).toBe(true); + expect(await read).toBe(CHUNK.byteLength * 2); + expect(vi.getTimerCount()).toBe(0); + }); + + it("does not keep the process alive while a download runs", () => { + const created: Array> = []; + const original = globalThis.setTimeout; + const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation((( + ...args: Parameters + ) => { + const timer = original(...args); + created.push(timer); + return timer; + }) as typeof setTimeout); + try { + const response = new PassThrough(); + const { body } = pumpExport( + { filename: "export.zip", dispose: async () => {}, write: () => new Promise(() => {}) }, + response, + ); + expect(created).toHaveLength(2); + for (const timer of created) expect(timer.hasRef()).toBe(false); + body.destroy(); + } finally { + spy.mockRestore(); + } + }); }); diff --git a/apps/api/src/privacy/export-response.ts b/apps/api/src/privacy/export-response.ts index 3e26c35bf..ea4e6ad9a 100644 --- a/apps/api/src/privacy/export-response.ts +++ b/apps/api/src/privacy/export-response.ts @@ -15,17 +15,88 @@ export function exportResponseHeaders(filename: string): Record }; } +/** + * Longest time the client may take no bytes before the download is ended. The + * archive is written as fast as the client reads, so a healthy download never + * pauses this long; it matches the request-body deadline. + */ +export const EXPORT_IDLE_MS = 60_000; + +/** + * Longest one download may run. The archive holds at most about 4 GB (the + * 2 GB OpenMapX source spool plus the 2 GB managed timeline limit), which + * still arrives within the hour at about 10 Mbit/s; a typical account is far + * smaller. It ends a client that keeps an export slot by reading a trickle. + */ +export const EXPORT_TOTAL_MS = 60 * 60_000; + +export interface ExportStreamLimits { + idleMs?: number; + totalMs?: number; +} + +/** The download was ended because the client stopped reading or took too long. */ +export class ExportDeadlineError extends Error { + constructor(readonly deadline: "idle" | "total") { + super( + deadline === "idle" + ? "export client stopped reading" + : "export exceeded its longest download time", + ); + this.name = "ExportDeadlineError"; + } +} + +/** Counts a chunk as progress once the response side has taken it. */ +class ProgressPassThrough extends PassThrough { + constructor(private readonly onProgress: () => void) { + super(); + } + + override read(size?: number): ReturnType { + const chunk = super.read(size); + if (chunk !== null) this.onProgress(); + return chunk; + } +} + /** * Stream a prepared export into `body` and settle once both the archive and * the response are finished. It resolves `true` only when every byte reached - * the client; an abort or a source failure after the headers resolves - * `false`, and the short-lived spool is removed either way. + * the client; an abort, a source failure or a missed deadline after the + * headers resolves `false`, and the short-lived spool is removed either way. + * The deadlines end the stream itself, so whatever the caller holds for the + * download is released only once the download has really stopped. */ export function pumpExport( prepared: PreparedSubjectExport, response: Writable, + limits: ExportStreamLimits = {}, ): { body: PassThrough; delivered: Promise } { - const body = new PassThrough(); + const idleMs = limits.idleMs ?? EXPORT_IDLE_MS; + const totalMs = limits.totalMs ?? EXPORT_TOTAL_MS; + let idleTimer: ReturnType | undefined; + const cut = (error: ExportDeadlineError) => { + body.destroy(error); + // Closing the connection settles the response even when the client never + // reads again; the reason travels with the archive stream. + response.destroy(); + }; + const armIdle = () => { + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => cut(new ExportDeadlineError("idle")), idleMs); + idleTimer.unref(); + }; + const body = new ProgressPassThrough(armIdle); + armIdle(); + const totalTimer = setTimeout(() => cut(new ExportDeadlineError("total")), totalMs); + totalTimer.unref(); + const stopTimers = () => { + if (idleTimer) clearTimeout(idleTimer); + clearTimeout(totalTimer); + }; + body.once("close", stopTimers); + // A client that goes away stops reading; end the archive instead of waiting. response.once("close", () => { if (!response.writableFinished) body.destroy(new Error("export client disconnected")); @@ -34,9 +105,16 @@ export function pumpExport( // response before the listener above existed; end the archive now. if (response.destroyed && !response.writableFinished) body.destroy(new Error("export client disconnected")); - const written = prepared.write(body); + let written: Promise; + try { + written = prepared.write(body); + } catch (error) { + stopTimers(); + throw error; + } const delivered = Promise.allSettled([written, finished(response)]).then( async ([archive, sent]) => { + stopTimers(); await prepared.dispose(); const ok = archive.status === "fulfilled" && sent.status === "fulfilled"; if (!ok) body.destroy(); diff --git a/apps/api/src/routes/__tests__/privacy-export-admission.test.ts b/apps/api/src/routes/__tests__/privacy-export-admission.test.ts index d4421a958..ffa65dbe4 100644 --- a/apps/api/src/routes/__tests__/privacy-export-admission.test.ts +++ b/apps/api/src/routes/__tests__/privacy-export-admission.test.ts @@ -73,7 +73,34 @@ function fakeService() { let app: FastifyInstance | undefined; -async function exportApp(maxConcurrent?: number) { +/** An export far larger than any socket buffer, written with backpressure; + * it rejects once the response goes away, like the real archive writer. */ +function endlessExport() { + const chunk = Buffer.alloc(64 * 1024, 1); + return { + filename: "openmapx-data-export.zip", + dispose: vi.fn(async () => {}), + write: (sink: Writable) => + new Promise((_resolve, reject) => { + let stopped = false; + sink.once("close", () => { + stopped = true; + reject(new Error("sink closed")); + }); + const next = () => { + if (stopped) return; + if (sink.write(chunk)) setImmediate(next); + else sink.once("drain", next); + }; + next(); + }), + }; +} + +async function exportApp( + maxConcurrent?: number, + exportStreamLimits?: { idleMs?: number; totalMs?: number }, +) { const service = fakeService(); app = Fastify(); await app.register(privacyRequestsRoute, { @@ -81,6 +108,7 @@ async function exportApp(maxConcurrent?: number) { // The route only calls the export methods on the service. service: service as never, exportAdmission: createPrivacyExportAdmission(maxConcurrent), + ...(exportStreamLimits ? { exportStreamLimits } : {}), }); await app.listen({ port: 0, host: "127.0.0.1" }); return { app, service, port: (app.server.address() as AddressInfo).port }; @@ -224,6 +252,44 @@ describe("POST /api/privacy/export admission", () => { await vi.waitFor(async () => expect((await inject("alice")).statusCode).toBe(200)); }); + it("ends a download whose client stops reading and frees the slot", async () => { + const { service, port } = await exportApp(undefined, { idleMs: 200 }); + const stalled = endlessExport(); + prepareSubjectExport.mockResolvedValueOnce(stalled); + const { outgoing } = startStreaming(port, "alice"); + let incoming: import("node:http").IncomingMessage | undefined; + outgoing.once("response", (response) => { + incoming = response; + // The client keeps the connection open but never reads past this point. + response.once("data", () => response.pause()); + }); + await vi.waitFor( + () => + expect(service.recordExportFailure).toHaveBeenCalledWith( + "request-alice", + "export-interrupted", + ), + { timeout: 10_000 }, + ); + expect(service.finishExport).not.toHaveBeenCalled(); + expect(stalled.dispose).toHaveBeenCalled(); + expect(incoming).toBeDefined(); + + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + await vi.waitFor(async () => expect((await inject("alice")).statusCode).toBe(200)); + outgoing.destroy(); + }); + + it("leaves a normal download alone under short deadlines", async () => { + const { service, port } = await exportApp(undefined, { idleMs: 200, totalMs: 5_000 }); + prepareSubjectExport.mockResolvedValueOnce(instantExport()); + const first = await startStreaming(port, "alice").firstBytes; + expect(first.status).toBe(200); + expect(await first.rest).toBe("PK"); + await vi.waitFor(() => expect(service.finishExport).toHaveBeenCalledTimes(1)); + expect(service.recordExportFailure).not.toHaveBeenCalled(); + }); + it("frees the slot when the client leaves while the export is being prepared", async () => { const { service, port } = await exportApp(); const held = heldExport(); diff --git a/apps/api/src/routes/privacy-requests.ts b/apps/api/src/routes/privacy-requests.ts index d152c4d64..5d18e9581 100644 --- a/apps/api/src/routes/privacy-requests.ts +++ b/apps/api/src/routes/privacy-requests.ts @@ -8,7 +8,11 @@ import { db } from "../db/index.js"; import { dataSubjectRequestTask } from "../db/schema.js"; import { PrivacyRequestError, privacyErrorReply } from "../privacy/errors.js"; import { acquireExportSlot, privacyExportAdmission } from "../privacy/export-admission.js"; -import { exportResponseHeaders, pumpExport } from "../privacy/export-response.js"; +import { + type ExportStreamLimits, + exportResponseHeaders, + pumpExport, +} from "../privacy/export-response.js"; import { selfServiceExportSchema } from "../privacy/request-contracts.js"; import { PrivacyRequestService } from "../privacy/request-service.js"; import { hasFreshAssurance } from "../privacy/session-assurance.js"; @@ -17,6 +21,7 @@ import { type PrepareSubjectExportOptions, prepareSubjectExport, } from "../privacy/subject-export.js"; +import { bindRequestAbort, withDeadline } from "../utils/request-abort.js"; import { getUserId, requireAuthHook } from "../utils/require-auth.js"; import { declareRouteAuth } from "../utils/route-auth.js"; @@ -30,8 +35,17 @@ export interface PrivacyRequestsRouteOptions { exportSources?: Pick; /** Export slots; tests pass their own so cases stay independent. */ exportAdmission?: RollingStreamBudget; + /** Download deadlines; tests shorten them. */ + exportStreamLimits?: ExportStreamLimits; } +/** + * Longest time collecting an export may take before the managed timeline + * transfer is stopped. The operations agent ends its own export after 15 + * minutes, so this never cuts a transfer the agent still considers healthy. + */ +const EXPORT_PREPARATION_DEADLINE_MS = 20 * 60_000; + function publicRequest(row: Record): Record { return { id: row.id, @@ -118,12 +132,24 @@ export const privacyRequestsRoute: FastifyPluginAsync; let body: ReturnType["body"]; try { - ({ body, delivered } = pumpExport(prepared, reply.raw)); + ({ body, delivered } = pumpExport(prepared, reply.raw, options.exportStreamLimits)); } catch (error) { // Nothing was sent yet; without this the slot would stay taken. await prepared.dispose().catch(() => undefined); diff --git a/docs/docs/features/accounts-and-privacy.md b/docs/docs/features/accounts-and-privacy.md index 2dac4c710..91e804788 100644 --- a/docs/docs/features/accounts-and-privacy.md +++ b/docs/docs/features/accounts-and-privacy.md @@ -50,7 +50,8 @@ See [Reviews](./reviews.md) for key export and recovery limits. 3. The archive is created on demand and downloaded straight to your device. Nothing is kept on the server, so you can download again at any time. One download per account runs at a time; a second one started while the - first is still running is refused until it finishes. + first is still running is refused until it finishes. A download that + stalls is ended and can be started again. 4. The request stays listed with its status and response deadline. Some sources can only be checked by the operator; the archive's report names them, and the request is completed once the operator has checked them.