Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions apps/api/src/privacy/export-admission.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { createHash } from "node:crypto";
import { RollingStreamBudget, type RollingStreamBudgetLease } from "@openmapx/core/server";
import { PrivacyRequestError } from "./errors.js";

/** Exports this API process prepares and streams at once, across all accounts. */
const MAX_CONCURRENT_EXPORTS = 4;

/**
* Admission for self-service exports: one export per account at a time and a
* small ceiling for the whole process. Collecting an account's data and
* streaming the archive is the most expensive thing a signed-in person can
* ask for, so a second export for the same account is refused while the first
* is still running. A retry after an interrupted download is accepted as soon
* as the earlier attempt has settled.
*
* The slots live in this process only. A restart frees them together with the
* streams they guarded; with several API replicas each replica counts its own.
* The byte budget of the underlying lease is unused.
*/
export function createPrivacyExportAdmission(
maxConcurrent = MAX_CONCURRENT_EXPORTS,
): RollingStreamBudget {
return new RollingStreamBudget({
maxConcurrentPerKey: 1,
maxConcurrentGlobal: maxConcurrent,
maxBytesPerKey: Number.MAX_SAFE_INTEGER,
windowMs: 60_000,
});
}

export const privacyExportAdmission = createPrivacyExportAdmission();

/**
* Take the export slot for an account, or throw the refusal the route maps to
* its response: 409 `EXPORT_IN_PROGRESS` while the account already holds one,
* 503 `PRIVACY_SERVICE_UNAVAILABLE` while the process is at its ceiling. The
* account id is digested so the slot map holds no account identifier.
*/
export function acquireExportSlot(
admission: RollingStreamBudget,
userId: string,
): RollingStreamBudgetLease {
const result = admission.tryAcquire(createHash("sha256").update(userId).digest("hex"));
if (result.allowed) return result.lease;
if (result.reason === "key-concurrency") throw new PrivacyRequestError("EXPORT_IN_PROGRESS", 409);
throw new PrivacyRequestError("PRIVACY_SERVICE_UNAVAILABLE", 503);
}
150 changes: 150 additions & 0 deletions apps/api/src/privacy/export-response.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
import { PassThrough, type Writable } from "node:stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ExportDeadlineError, pumpExport } from "./export-response.js";

const CHUNK = Buffer.alloc(16 * 1024, 1);

/** Writes chunks for as long as the sink accepts them, honouring backpressure,
* and rejects once the sink fails, as the real archive writer does. */
function endlessExport(pauseMs = 0) {
return {
filename: "export.zip",
dispose: vi.fn(async () => {}),
write: (sink: Writable) =>
new Promise<void>((_resolve, reject) => {
let stopped = false;
const stop = () => {
stopped = true;
reject(new Error("sink closed"));
};
sink.once("close", stop);
sink.once("error", () => undefined);
const next = () => {
if (stopped) return;
if (sink.write(CHUNK)) setTimeout(next, pauseMs);
else sink.once("drain", next);
};
next();
}),
};
}

function shortExport() {
return {
filename: "export.zip",
dispose: vi.fn(async () => {}),
write: async (sink: Writable) => {
sink.write(CHUNK);
sink.end(CHUNK);
},
};
}

function deadlineOf(body: PassThrough): Promise<unknown> {
return new Promise((resolve) => body.once("error", resolve));
}

afterEach(() => {
vi.useRealTimers();
});

describe("pumpExport", () => {
it("ends the archive when the response closed before streaming began", async () => {
const response = new PassThrough();
response.destroy();
await new Promise((resolve) => setImmediate(resolve));
let sink: Writable | undefined;
const dispose = vi.fn(async () => {});
const { body, delivered } = pumpExport(
{
filename: "export.zip",
// Waits for the sink to fail, as the real archive writer does.
write: (target) =>
new Promise<void>((_resolve, reject) => {
sink = target;
if (target.destroyed) reject(new Error("sink closed"));
else target.once("close", () => reject(new Error("sink closed")));
}),
dispose,
},
response,
);
// Fastify listens on the body once it is sent; stand in for it here.
body.on("error", () => undefined);
expect(await delivered).toBe(false);
expect(sink?.destroyed).toBe(true);
expect(dispose).toHaveBeenCalledTimes(1);
});

it("ends a download whose client stops reading", async () => {
// A response nobody reads: once its small buffer is full, no more bytes move.
const response = new PassThrough({ highWaterMark: 1024 });
const prepared = endlessExport();
const { body, delivered } = pumpExport(prepared, response, { idleMs: 50, totalMs: 60_000 });
const reason = deadlineOf(body);
body.pipe(response);
expect(await delivered).toBe(false);
expect(await reason).toEqual(new ExportDeadlineError("idle"));
expect(response.destroyed).toBe(true);
expect(prepared.dispose).toHaveBeenCalledTimes(1);
});

it("ends a download that runs past the longest download time", async () => {
// The client reads every byte, but the archive only trickles in.
const response = new PassThrough();
response.resume();
const prepared = endlessExport(5);
const { body, delivered } = pumpExport(prepared, response, { idleMs: 60_000, totalMs: 100 });
const reason = deadlineOf(body);
body.pipe(response);
expect(await delivered).toBe(false);
expect(await reason).toEqual(new ExportDeadlineError("total"));
expect(prepared.dispose).toHaveBeenCalledTimes(1);
});

it("delivers a normal download and leaves no timer behind", async () => {
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const response = new PassThrough();
const read = new Promise<number>((resolve) => {
let bytes = 0;
response.on("data", (chunk: Buffer) => {
bytes += chunk.byteLength;
});
response.once("end", () => resolve(bytes));
});
const { body, delivered } = pumpExport(shortExport(), response, {
idleMs: 1_000,
totalMs: 5_000,
});
// Both deadlines are armed and neither keeps the process alive.
expect(vi.getTimerCount()).toBe(2);
body.pipe(response);
expect(await delivered).toBe(true);
expect(await read).toBe(CHUNK.byteLength * 2);
expect(vi.getTimerCount()).toBe(0);
});

it("does not keep the process alive while a download runs", () => {
const created: Array<ReturnType<typeof setTimeout>> = [];
const original = globalThis.setTimeout;
const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((
...args: Parameters<typeof setTimeout>
) => {
const timer = original(...args);
created.push(timer);
return timer;
}) as typeof setTimeout);
try {
const response = new PassThrough();
const { body } = pumpExport(
{ filename: "export.zip", dispose: async () => {}, write: () => new Promise(() => {}) },
response,
);
expect(created).toHaveLength(2);
for (const timer of created) expect(timer.hasRef()).toBe(false);
body.destroy();
} finally {
spy.mockRestore();
}
});
});
90 changes: 86 additions & 4 deletions apps/api/src/privacy/export-response.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,106 @@ export function exportResponseHeaders(filename: string): Record<string, string>
};
}

/**
* Longest time the client may take no bytes before the download is ended. The
* archive is written as fast as the client reads, so a healthy download never
* pauses this long; it matches the request-body deadline.
*/
export const EXPORT_IDLE_MS = 60_000;

/**
* Longest one download may run. The archive holds at most about 4 GB (the
* 2 GB OpenMapX source spool plus the 2 GB managed timeline limit), which
* still arrives within the hour at about 10 Mbit/s; a typical account is far
* smaller. It ends a client that keeps an export slot by reading a trickle.
*/
export const EXPORT_TOTAL_MS = 60 * 60_000;

export interface ExportStreamLimits {
idleMs?: number;
totalMs?: number;
}

/** The download was ended because the client stopped reading or took too long. */
export class ExportDeadlineError extends Error {
constructor(readonly deadline: "idle" | "total") {
super(
deadline === "idle"
? "export client stopped reading"
: "export exceeded its longest download time",
);
this.name = "ExportDeadlineError";
}
}

/** Counts a chunk as progress once the response side has taken it. */
class ProgressPassThrough extends PassThrough {
constructor(private readonly onProgress: () => void) {
super();
}

override read(size?: number): ReturnType<PassThrough["read"]> {
const chunk = super.read(size);
if (chunk !== null) this.onProgress();
return chunk;
}
}

/**
* Stream a prepared export into `body` and settle once both the archive and
* the response are finished. It resolves `true` only when every byte reached
* the client; an abort or a source failure after the headers resolves
* `false`, and the short-lived spool is removed either way.
* the client; an abort, a source failure or a missed deadline after the
* headers resolves `false`, and the short-lived spool is removed either way.
* The deadlines end the stream itself, so whatever the caller holds for the
* download is released only once the download has really stopped.
*/
export function pumpExport(
prepared: PreparedSubjectExport,
response: Writable,
limits: ExportStreamLimits = {},
): { body: PassThrough; delivered: Promise<boolean> } {
const body = new PassThrough();
const idleMs = limits.idleMs ?? EXPORT_IDLE_MS;
const totalMs = limits.totalMs ?? EXPORT_TOTAL_MS;
let idleTimer: ReturnType<typeof setTimeout> | undefined;
const cut = (error: ExportDeadlineError) => {
body.destroy(error);
// Closing the connection settles the response even when the client never
// reads again; the reason travels with the archive stream.
response.destroy();
};
const armIdle = () => {
if (idleTimer) clearTimeout(idleTimer);
idleTimer = setTimeout(() => cut(new ExportDeadlineError("idle")), idleMs);
idleTimer.unref();
};
const body = new ProgressPassThrough(armIdle);
armIdle();
const totalTimer = setTimeout(() => cut(new ExportDeadlineError("total")), totalMs);
totalTimer.unref();
const stopTimers = () => {
if (idleTimer) clearTimeout(idleTimer);
clearTimeout(totalTimer);
};
body.once("close", stopTimers);

// A client that goes away stops reading; end the archive instead of waiting.
response.once("close", () => {
if (!response.writableFinished) body.destroy(new Error("export client disconnected"));
});
const written = prepared.write(body);
// A client that left while the export was being prepared closed the
// response before the listener above existed; end the archive now.
if (response.destroyed && !response.writableFinished)
body.destroy(new Error("export client disconnected"));
let written: Promise<void>;
try {
written = prepared.write(body);
} catch (error) {
stopTimers();
throw error;
}
const delivered = Promise.allSettled([written, finished(response)]).then(
async ([archive, sent]) => {
stopTimers();
await prepared.dispose();
const ok = archive.status === "fulfilled" && sent.status === "fulfilled";
if (!ok) body.destroy();
Expand Down
Loading
Loading