diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md index 2aa2dc85..3fd33258 100644 --- a/docs/protocol-v2.md +++ b/docs/protocol-v2.md @@ -850,9 +850,22 @@ already stores the file. the client uploads the files and commits the same session again. A polling client sees `status` `open`, with the refusal in `error`. - Committing a session that has already committed answers 201 with the same body. +- A server MAY let a session whose commit failed on the server's side (`error.statusCode` 500 or 503) be committed again from the same uploads; underlay.org does until shortly before it + deletes the session's uploads, about a day after it would have expired. A failure the client can + fix only by pushing again (422, 409) answers 409. - A client that holds the base SHOULD compute the new version's hash itself and compare it with `hash`. +**One version from any number of uploads** (informative). A session has no size limit beyond the +server's storage: a client can upload a whole corpus into one session over many requests, and +days, and commit it as one version. A server keeps the uploads in storage rather than memory, and +a background commit's status MAY carry `progress` (underlay.org: `runs`, `run_bytes`, +`runs_compacting`, `phase` of `compacting`, `planning`, `units` or `assembling`, `units` done of +`total`, and `progress_at`). An open session expires only after `session_idle_seconds` without an +upload, so a client that stops can pick the same session up again by its id. Each version rewrites +the leaves its changes land in, so one large version costs far less than the same records in many +small ones. + **Clients without a copy** (informative). A client that keeps no copy of the collection reads the base's manifest (Section 11.3.6), compares each of its records with the manifest by (type, id), hash and set, uploads the records that are new, changed or moved between sets, and deletes the @@ -896,10 +909,15 @@ The reference implementation defines these in `packages/protocol/src/constants.t test vectors repeat them. Server limits (Section 11.4) are not protocol constants. Informative: underlay.org advertises -`open_bytes` 8 MiB, `batch_bytes` 16 MiB, `batch_lines` 10,000, `session_idle_seconds` 3,600, -`open_sessions` 20 and `file_bytes` 32 MiB; caps log pages at 1,000 entries and manifest pages at -25,000 records; stores records over 64 KiB out of line; and commits in the background when a -session uploaded more than 100,000 records or a schema change revalidates more than 100,000. +`open_bytes` 8 MiB, `batch_bytes` 8 MiB, `batch_lines` 10,000, `session_idle_seconds` 43,200, +`open_sessions` 20 and `file_bytes` 32 MiB; caps log pages at 1,000 entries, manifest pages at +25,000 records and diff pages at 5,000 records or 4 MiB of records; stores records over 64 KiB +out of line; and commits in the background when a session uploaded more than 100,000 records or a +schema change revalidates more than 100,000. A schema change that can't reject a record (dropping +`required` names, editing annotations) isn't revalidated; any other is revalidated in parallel +jobs, so a type of any size can change its schema. A privacy change that would rewrite the whole +type (records already in both sets, or private records pushed as the type turns public) is refused +past about 2 GiB: push it on its own first. ## 13. Security considerations diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 899bc0e3..87deafd6 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -15,7 +15,17 @@ export { type FsckStepOptions, } from './repo/fsck.js' export * from './repo/log.js' -export { gunzip, gunzipMembers, gunzipText, gzip, isGzip, splitLines } from './repo/gzip.js' +export { schemaRelaxes } from './schema-relax.js' +export { + chunkLines, + gunzip, + gunzipChunks, + gunzipMembers, + gunzipText, + gzip, + isGzip, + splitLines, +} from './repo/gzip.js' export { Lru } from './repo/lru.js' export { type BuildInput, diff --git a/packages/protocol/src/repo/build.ts b/packages/protocol/src/repo/build.ts index b1425fbf..3ddef862 100644 --- a/packages/protocol/src/repo/build.ts +++ b/packages/protocol/src/repo/build.ts @@ -12,7 +12,8 @@ * Moving a type between sets, or removing one, moves or drops its file * references by their per-type counts (file-sets.ts): O(files it references). * On a base whose count trees predate those counts, it reads the type's bodies. - * 3. types whose schema changed are revalidated (O(type size)); + * 3. types whose schema changed are revalidated (O(type size)), unless the + * change can't make a valid record invalid (schemaRelaxes); * 4. file sets from reference-count deltas (file-sets.ts); * 5. the private set object and the root. * @@ -36,6 +37,7 @@ import { type TreeSummary, type VersionRoot, } from '../format.js' +import { schemaRelaxes } from '../schema-relax.js' import { applyFileSet, FileRefDelta, @@ -82,13 +84,18 @@ export interface BuildInput { /** * Trees already merged elsewhere (a parallel commit), per type, with the file * reference deltas and change counts that built them. The types' change - * sources are then ignored. Only when no type changes privacy or schema and - * none is removed. + * sources are then ignored, and so is `validate`: a parallel commit + * revalidates in its own jobs. A type in `moves` changed privacy by moving its + * whole tree to the other set (the serial build's plain moves); its trees + * were merged into the moved tree, and its file references move here. For a + * type made public, `stats` leave out its public changes: every record of the + * new public tree counts as added, as in the serial build. */ prebuilt?: { trees: Record refs: FileRefDelta stats: { added: number; removed: number; updated: number } + moves?: Record } } @@ -258,6 +265,15 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise interface Zlib { gunzip(bytes: Uint8Array, done: (err: Error | null, out: Uint8Array) => void): void + /** A streaming gunzip (a Node Transform): written once, read as it decompresses. */ + createGunzip?(): { end(chunk: Uint8Array): void } & AsyncIterable } const zlib = ( globalThis as { process?: { getBuiltinModule?: (id: 'node:zlib') => Zlib | undefined } } @@ -86,6 +88,38 @@ export async function gunzipMembers( export const gunzipText = async (bytes: Uint8Array) => dec.decode(await gunzip(bytes)) +/** + * A gzip object's decompressed bytes (any number of members) a chunk at a time, + * as zlib produces them, so a large body never sits whole in memory. Runtimes + * without a streaming zlib get it in one chunk. + */ +export async function* gunzipChunks(bytes: Uint8Array): AsyncGenerator { + const stream = zlib?.createGunzip?.() + if (!stream || typeof stream[Symbol.asyncIterator] !== 'function') { + yield await gunzip(bytes) + return + } + stream.end(bytes) + for await (const chunk of stream) yield chunk +} + +/** NDJSON lines from chunks of UTF-8 bytes (a trailing newline ends the last line). */ +export async function* chunkLines(chunks: AsyncIterable): AsyncGenerator { + const decoder = new TextDecoder() + let rest = '' + for await (const chunk of chunks) { + const text = rest + decoder.decode(chunk, { stream: true }) + let start = 0 + for (let nl = text.indexOf('\n'); nl !== -1; nl = text.indexOf('\n', start)) { + yield text.slice(start, nl) + start = nl + 1 + } + rest = text.slice(start) + } + rest += decoder.decode() + if (rest.length > 0) yield rest +} + export const isGzip = (bytes: Uint8Array) => bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b diff --git a/packages/protocol/src/repo/repo.ts b/packages/protocol/src/repo/repo.ts index 5163f93a..00c03e9c 100644 --- a/packages/protocol/src/repo/repo.ts +++ b/packages/protocol/src/repo/repo.ts @@ -41,7 +41,7 @@ import { versionDigest, versionHash, } from '../format.js' -import { gunzipText, gzip, splitLines } from './gzip.js' +import { chunkLines, gunzipChunks, gunzipText, gzip, splitLines } from './gzip.js' import { Lru } from './lru.js' import { type Store, type Cache, noCache } from './types.js' @@ -144,8 +144,23 @@ export class Repo { return `${this.scope}/${key}` } + /** Reads in progress, by what they read: a second caller waits for the first. */ + readonly #pending = new Map>() + + #shared(key: string, read: () => Promise): Promise { + const running = this.#pending.get(key) + if (running) return running as Promise + const p = read().finally(() => this.#pending.delete(key)) + this.#pending.set(key, p) + return p + } + /** An immutable object's bytes: shared cache, then the bucket. `verify` runs before caching. */ - async #immutable( + #immutable(key: string, verify?: (bytes: Uint8Array) => Promise) { + return this.#shared(`bytes:${key}`, () => this.#fetch(key, verify)) + } + + async #fetch( key: string, verify?: (bytes: Uint8Array) => Promise, ): Promise { @@ -165,6 +180,10 @@ export class Repo { const key = keys.node(hash) const hit = this.lru.get(this.#ck(key)) if (typeof hit === 'string') return hit + return this.#shared(`node:${key}`, () => this.#nodeJson(hash, key)) + } + + async #nodeJson(hash: string, key: string): Promise { const bytes = await this.#immutable(key, async (b) => { if (sha256Hex(await gunzipText(b)) !== hash) throw new IntegrityError(`Node ${hash} fails its hash`) @@ -203,6 +222,14 @@ export class Repo { const key = keys.body(leaf.hash) const hit = this.lru.get(this.#ck(key)) if (hit) return (hit as { lines: string[] }).lines + // Two readers of one leaf at once decompress it once. + return this.#shared(`lines:${key}`, () => this.#bodyLines(leaf, key)) + } + + async #bodyLines( + leaf: { hash: string; entries: readonly RecordEntry[] }, + key: string, + ): Promise { const check = async (b: Uint8Array) => { const lines = await this.#resolve(splitLines(await gunzipText(b))) if (lines.length !== leaf.entries.length) @@ -228,6 +255,36 @@ export class Repo { return lines } + /** + * A leaf's body lines one at a time, out-of-line records resolved, without + * holding the body decompressed: for rewriting leaves, where one leaf of large + * records can be tens of MB. Nothing is put in the memory cache. Untrusted + * locations read through bodyLines, which checks every line before use. + */ + async *bodyLineStream(leaf: { + hash: string + entries: readonly RecordEntry[] + }): AsyncGenerator { + const key = keys.body(leaf.hash) + const hit = this.lru.get(this.#ck(key)) + if (hit || !this.trusted) { + yield* hit ? (hit as { lines: string[] }).lines : await this.bodyLines(leaf) + return + } + const bytes = await this.#immutable(key) + if (!bytes) throw new Error(`Missing body for leaf ${leaf.hash}`) + let n = 0 + for await (const line of chunkLines(gunzipChunks(bytes))) { + if (++n > leaf.entries.length) break + yield line.startsWith(REF_PREFIX) ? await this.#outOfLine(line) : line + } + if (n !== leaf.entries.length) { + throw new IntegrityError( + `Body of ${leaf.hash} has ${n} lines for ${leaf.entries.length} entries`, + ) + } + } + /** * A leaf body's stored bytes (gzip members), as written: for serving a run of * leaves without decoding them. Pointer lines (out-of-line records) stay @@ -370,6 +427,21 @@ export class RepoSource implements NodeSource { const lines = await this.repo.bodyLines({ hash, entries }) return entries.map((e, i) => ({ ...e, body: lines[i]! })) as E[] } + + /** A leaf's entries with bodies, one at a time (Repo.bodyLineStream). */ + async *leafStream(hash: string): AsyncGenerator { + const node = await this.node(hash) + if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) + if (this.spec !== (recordTree as TreeSpec)) { + yield* node.entries + return + } + const entries = node.entries as RecordEntry[] + let i = 0 + for await (const body of this.repo.bodyLineStream({ hash, entries })) { + yield { ...entries[i++]!, body } as E + } + } } /** diff --git a/packages/protocol/src/schema-relax.ts b/packages/protocol/src/schema-relax.ts new file mode 100644 index 00000000..09beaeaa --- /dev/null +++ b/packages/protocol/src/schema-relax.ts @@ -0,0 +1,82 @@ +/** + * Is a schema change one that can't make a valid record invalid? Then a commit + * needn't revalidate the type's records, which is O(type size). + * + * `after` relaxes `before` when the two are the same schema except for: + * + * - `required` lists that lose names, where loosening can only accept more + * (not under `not`, `oneOf`, `if`, `definitions` or `$defs`: there a looser + * subschema can reject a record, or be reached from anywhere by `$ref`); + * - annotations: `title`, `description`, `$comment`; + * - the root's `private` flag, which says which set records go in, not what + * they hold. + * + * A schema with a `$ref` gets only the annotation and `private` allowances: a + * reference can reach a subschema from under `not`. Everything else must be + * equal (JCS). False means "revalidate", never "invalid". + */ +import { jcs } from './jcs.js' + +const ANNOTATIONS = new Set(['title', 'description', '$comment']) +/** Keywords whose value is a map of name → subschema, each in a monotone position. */ +const SCHEMA_MAPS = new Set(['properties', 'patternProperties']) +/** Keywords whose value is one subschema in a monotone position. */ +const SCHEMA_ONE = new Set([ + 'items', + 'additionalItems', + 'additionalProperties', + 'contains', + 'propertyNames', + 'then', + 'else', +]) +/** Keywords whose value is a list of subschemas in monotone positions. */ +const SCHEMA_LIST = new Set(['allOf', 'anyOf', 'items']) + +const isObject = (v: unknown): v is Record => + v !== null && typeof v === 'object' && !Array.isArray(v) +const same = (a: unknown, b: unknown) => + a === undefined || b === undefined ? a === b : jcs(a) === jcs(b) + +function hasRef(v: unknown): boolean { + if (Array.isArray(v)) return v.some(hasRef) + if (!isObject(v)) return false + return Object.entries(v).some(([k, x]) => k === '$ref' || hasRef(x)) +} + +export function schemaRelaxes(before: unknown, after: unknown): boolean { + return relaxes(before, after, true, !hasRef(before) && !hasRef(after)) +} + +function relaxes(a: unknown, b: unknown, root: boolean, loosen: boolean): boolean { + if (!isObject(a) || !isObject(b)) return same(a, b) + for (const k of new Set([...Object.keys(a), ...Object.keys(b)])) { + if (ANNOTATIONS.has(k) || (root && k === 'private')) continue + const x = a[k] + const y = b[k] + if (k === 'required' && loosen) { + const was = x === undefined ? [] : x + const now = y === undefined ? [] : y + if (!Array.isArray(was) || !Array.isArray(now)) return false + if (!now.every((n) => was.includes(n))) return false + continue + } + if (SCHEMA_MAPS.has(k) && isObject(x) && isObject(y)) { + if (!same(Object.keys(x).sort(), Object.keys(y).sort())) return false + for (const name of Object.keys(x)) if (!relaxes(x[name], y[name], false, loosen)) return false + continue + } + if (SCHEMA_LIST.has(k) && Array.isArray(x) && Array.isArray(y)) { + if (x.length !== y.length) return false + for (let i = 0; i < x.length; i++) if (!relaxes(x[i], y[i], false, loosen)) return false + continue + } + if (SCHEMA_ONE.has(k) && isObject(x) && isObject(y)) { + if (!relaxes(x, y, false, loosen)) return false + continue + } + // Everything else, including `not`, `oneOf`, `if` and definitions, must be equal. + if (!same(x, y)) return false + } + return true +} diff --git a/packages/protocol/src/tree/assemble.ts b/packages/protocol/src/tree/assemble.ts index 04981c81..81245c03 100644 --- a/packages/protocol/src/tree/assemble.ts +++ b/packages/protocol/src/tree/assemble.ts @@ -25,7 +25,7 @@ import { compareUtf8 } from '../utf8.js' import { type BuilderOptions, TreeBuilder, type TreeSink } from './builder.js' import { emptyStats, type KeyRange, type MergeResult } from './merge.js' import type { NodeDesc } from './node.js' -import { type NodeSource, resolveRoot, rootDesc } from './source.js' +import { leafEntryStream, type NodeSource, resolveRoot, rootDesc } from './source.js' /** * One unit's output: its range and the leaves it produced, in key order. The @@ -95,16 +95,21 @@ export async function assembleTree( } stats.readNodes++ if (desc.level === 0) { - const entries = await source.leafEntries(desc.hash) - let i = 0 + // Read one entry at a time: a leaf of large records is never held whole. + const it = leafEntryStream(source, desc.hash)[Symbol.asyncIterator]() + const nextEntry = async () => { + const r = await it.next() + return r.done ? undefined : r.value + } + let cur = await nextEntry() while (overlaps(desc.lastKey)) { const seg = segments[head]! // Under the rules a segment never starts inside a base leaf with // uncovered entries before it: an unchanged `after` ends its base leaf, // and an inserted one was covered by the previous segment. if (seg.after !== null) { - for (; i < entries.length && compareUtf8(spec.key(entries[i]!), seg.after) <= 0; i++) { - if (!isCovered(spec.key(entries[i]!))) { + for (; cur && compareUtf8(spec.key(cur), seg.after) <= 0; cur = await nextEntry()) { + if (!isCovered(spec.key(cur))) { throw new Error( `assembleTree: segment after ${JSON.stringify(seg.after)} starts inside a base leaf`, ) @@ -112,10 +117,13 @@ export async function assembleTree( } } await emit(seg) - if (all) return + if (all) { + await it.return?.() + return + } } // The tail of a base leaf straddling an inserted `through`. - for (const e of entries) if (!isCovered(spec.key(e))) builder.addEntry(e) + for (; cur; cur = await nextEntry()) if (!isCovered(spec.key(cur))) builder.addEntry(cur) await sink.drain?.() return } diff --git a/packages/protocol/src/tree/merge.ts b/packages/protocol/src/tree/merge.ts index 43c8c3a3..ceda7f2c 100644 --- a/packages/protocol/src/tree/merge.ts +++ b/packages/protocol/src/tree/merge.ts @@ -23,7 +23,7 @@ import { compareUtf8 } from '../utf8.js' import { type BuilderOptions, TreeBuilder, type TreeSink } from './builder.js' import type { NodeDesc } from './node.js' -import { type NodeSource, resolveRoot, rootDesc } from './source.js' +import { leafEntryStream, type NodeSource, resolveRoot, rootDesc } from './source.js' /** Upsert (`entry`) or delete (`entry: null`) of one key. */ export interface Change { @@ -162,8 +162,7 @@ export async function mergeTree( const rewriteLeaf = async (desc: NodeDesc) => { stats.readNodes++ - const entries = await source.leafEntries(desc.hash) - for (const old of entries) { + for await (const old of leafEntryStream(source, desc.hash)) { const oldKey = spec.key(old) // A leaf straddling a unit's range edge: keep only the range's entries. if (range && !inRange(range, oldKey)) continue diff --git a/packages/protocol/src/tree/source.ts b/packages/protocol/src/tree/source.ts index 8c66b6db..07443be6 100644 --- a/packages/protocol/src/tree/source.ts +++ b/packages/protocol/src/tree/source.ts @@ -13,8 +13,21 @@ export interface NodeSource { * Sources whose entries carry no payload return the node's entries. */ leafEntries(hash: string): Promise + /** + * The same entries one at a time, where the source can stream them: a leaf of + * large records is never held whole. Readers fall back to `leafEntries`. + */ + leafStream?(hash: string): AsyncIterable } +/** A leaf's entries with payloads: streamed where the source can. */ +export const leafEntryStream = (source: NodeSource, hash: string): AsyncIterable => + source.leafStream + ? source.leafStream(hash) + : (async function* () { + yield* await source.leafEntries(hash) + })() + /** A source over node JSON held in memory, keyed by hash. */ export class MapSource implements NodeSource { constructor( diff --git a/packages/protocol/test/repo/gzip.test.ts b/packages/protocol/test/repo/gzip.test.ts index c7bda573..f857cc3c 100644 --- a/packages/protocol/test/repo/gzip.test.ts +++ b/packages/protocol/test/repo/gzip.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { gunzipMembers, gunzipText, gzip } from '../../src/repo/gzip.js' +import { chunkLines, gunzipChunks, gunzipMembers, gunzipText, gzip } from '../../src/repo/gzip.js' describe('gunzip', () => { // A large leaf body is several gzip members in one object. workerd's @@ -45,3 +45,29 @@ describe('gunzip where the decoder reads one member (browsers)', () => { await expect(gunzipMembers(new Uint8Array([1, 2, 3]), oneMember)).rejects.toThrow() }) }) + +describe('streaming gunzip', () => { + it('reads every member, and splits lines across chunk and character boundaries', async () => { + const lines = Array.from( + { length: 5000 }, + (_, i) => `{"n":${i},"t":"é😀${'x'.repeat(i % 97)}"}`, + ) + const text = lines.join('\n') + '\n' + const half = text.length >> 1 + const body = new Uint8Array([ + ...(await gzip(text.slice(0, half))), + ...(await gzip(text.slice(half))), + ]) + const got: string[] = [] + for await (const l of chunkLines(gunzipChunks(body))) got.push(l) + expect(got).toEqual(lines) + // Chunks that cut a multi-byte character in two. + const bytes = new TextEncoder().encode('ab😀c\nd') + async function* oneByte() { + for (const b of bytes) yield new Uint8Array([b]) + } + const split: string[] = [] + for await (const l of chunkLines(oneByte())) split.push(l) + expect(split).toEqual(['ab😀c', 'd']) + }) +}) diff --git a/packages/protocol/test/repo/objects.test.ts b/packages/protocol/test/repo/objects.test.ts index 96e6e36d..c44e87fe 100644 --- a/packages/protocol/test/repo/objects.test.ts +++ b/packages/protocol/test/repo/objects.test.ts @@ -132,6 +132,56 @@ describe('record trees in a repository', () => { expect(rows.every((r) => JSON.parse(r.body!).id === r.key)).toBe(true) }) + it('streams a leaf one entry at a time, as bodyLines reads it whole', async () => { + const { repo } = freshRepo() + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + const b = new TreeBuilder(recordTree, sink, { + payloadBytes: recordPayloadBytes, + dropPayload: dropRecordBody, + spillBytes: 20_000, // several members + }) + const big = (id: string) => { + const { hash, canonical } = hashRecord(id, 'T', { id, t: 'é😀'.repeat(OUT_OF_LINE_BYTES) }) + return { key: id, hash, size: utf8ByteLength(canonical), body: canonical } + } + const all = ids(2000) + for (const id of all) { + const e = id === all[7] ? big(id) : record(id) + if (e.size > OUT_OF_LINE_BYTES) e.body = await repo.putOutOfLine(e.hash, e.body!) + b.addEntry(e) + } + const root = b.finish().root! + await sink.flush() + const source = new RepoSource(recordTree, repo) + const node = await source.node(root.hash) + const leaf = node.kind === 'leaf' ? root.hash : (node.children[0]!.hash as string) + const whole = await source.leafEntries(leaf) + const streamed = await collect(source.leafStream(leaf)) + expect(streamed).toEqual(whole) + expect(streamed.some((e) => e.body!.includes('😀'))).toBe(true) + }) + + it('shares a read in progress: simultaneous readers of one leaf fetch it once', async () => { + const { blobs, repo } = freshRepo() + const { root } = await buildRecords(repo, 500) + const source = new RepoSource(recordTree, repo) + const node = await source.node(root.hash) + const leaf = node.kind === 'leaf' ? root.hash : (node.children[0]!.hash as string) + const leafNode = await source.node(leaf) + if (leafNode.kind !== 'leaf') throw new Error('not a leaf') + const entries = leafNode.entries + let gets = 0 + const get = blobs.get.bind(blobs) + blobs.get = async (key, range) => { + if (key === keys.body(leaf)) gets++ + return get(key, range) + } + const reads = await Promise.all([1, 2, 3].map(() => repo.bodyLines({ hash: leaf, entries }))) + expect(gets).toBe(1) + expect(reads[1]).toEqual(reads[0]) + expect(reads[0]!.length).toBe(entries.length) + }) + it('holds back drain while the bodies in flight pass the byte bound', async () => { const blobs = memoryStore() let release!: () => void diff --git a/packages/protocol/test/schema-relax.test.ts b/packages/protocol/test/schema-relax.test.ts new file mode 100644 index 00000000..aa22ca57 --- /dev/null +++ b/packages/protocol/test/schema-relax.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'vitest' + +import { compileSchema } from '../src/format.js' +import { schemaRelaxes } from '../src/schema-relax.js' + +const base = { + type: 'object', + title: 'Work', + properties: { + title: { type: 'array', items: { type: 'string' } }, + author: { + type: 'array', + items: { + type: 'object', + properties: { given: { type: 'string' }, family: { type: 'string' } }, + required: ['family'], + }, + }, + published: { type: 'object', properties: { year: { type: 'integer' } }, required: ['year'] }, + }, + required: ['title', 'author'], +} +const clone = (v: T): T => JSON.parse(JSON.stringify(v)) as T + +describe('schemaRelaxes', () => { + it('allows dropping required names, anywhere loosening can only accept more', () => { + const top = clone(base) + top.required = ['author'] + expect(schemaRelaxes(base, top)).toBe(true) + const nested = clone(base) + nested.properties.author.items.required = [] + expect(schemaRelaxes(base, nested)).toBe(true) + const gone = clone(base) as Record + delete gone.properties.published.required + expect(schemaRelaxes(base, gone)).toBe(true) + }) + + it('refuses anything that can reject a record valid before', () => { + const added = clone(base) + added.required = ['title', 'author', 'published'] + expect(schemaRelaxes(base, added)).toBe(false) + const retyped = clone(base) as Record + // A property named "title" is a schema, not an annotation. + retyped.properties.title = { type: 'string' } + expect(schemaRelaxes(base, retyped)).toBe(false) + const newProp = clone(base) as Record + newProp.properties.doi = { type: 'string' } + expect(schemaRelaxes(base, newProp)).toBe(false) + const nestedPrivate = clone(base) as Record + nestedPrivate.properties.author.private = true + expect(schemaRelaxes(base, nestedPrivate)).toBe(false) + }) + + it('ignores annotations and the root private flag', () => { + const notes = clone(base) as Record + notes.title = 'A work' + notes.description = 'One work' + notes.properties.published.$comment = 'year only' + notes.private = true + expect(schemaRelaxes(base, notes)).toBe(true) + }) + + it('keeps oneOf, not, if and definitions exact', () => { + for (const wrap of ['oneOf', 'not', 'if'] as const) { + const branch = { type: 'object', required: ['a', 'b'] } + const before = { [wrap]: wrap === 'oneOf' ? [branch, { required: ['c'] }] : branch } + const looser = { type: 'object', required: ['a'] } + const after = { [wrap]: wrap === 'oneOf' ? [looser, { required: ['c'] }] : looser } + expect(schemaRelaxes(before, after)).toBe(false) + } + expect( + schemaRelaxes( + { definitions: { x: { required: ['a'] } } }, + { definitions: { x: { required: [] } } }, + ), + ).toBe(false) + }) + + it('gives a schema with $ref only the annotation allowances', () => { + const before = { + properties: { a: { $ref: '#/definitions/x' } }, + required: ['a'], + definitions: { x: {} }, + } + expect(schemaRelaxes(before, { ...before, required: [] })).toBe(false) + expect(schemaRelaxes(before, { ...before, title: 'T' })).toBe(true) + }) + + it('accepts every record the old schema accepts, on samples', () => { + const after = clone(base) + after.required = ['author'] + after.properties.author.items.required = [] + expect(schemaRelaxes(base, after)).toBe(true) + const was = compileSchema(base) + const now = compileSchema(after) + const samples = [ + { title: ['t'], author: [{ family: 'f' }] }, + { title: ['t'], author: [{ given: 'g', family: 'f' }], published: { year: 2001 } }, + { author: [] }, + { title: 'x', author: [{ given: 1 }] }, + ] + for (const s of samples) if (was(s).length === 0) expect(now(s)).toEqual([]) + }) +}) diff --git a/packages/server/drizzle/0024_stream_push.sql b/packages/server/drizzle/0024_stream_push.sql new file mode 100644 index 00000000..f988e142 --- /dev/null +++ b/packages/server/drizzle/0024_stream_push.sql @@ -0,0 +1,6 @@ +ALTER TABLE `commit_units` ADD `attempts` integer DEFAULT 0 NOT NULL;--> statement-breakpoint +ALTER TABLE `push_runs` ADD `bytes` integer DEFAULT 0 NOT NULL;--> statement-breakpoint +ALTER TABLE `push_runs` ADD `claimed_at` integer;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `commit_phase` text;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `progress_at` integer;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `stalls` integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0024_snapshot.json b/packages/server/drizzle/meta/0024_snapshot.json new file mode 100644 index 00000000..d1123282 --- /dev/null +++ b/packages/server/drizzle/meta/0024_snapshot.json @@ -0,0 +1,3560 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "8d26270d-9b7c-4470-bde3-4e4c122b30fb", + "prevId": "e71a382c-fa6e-4adb-8677-16e126744817", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index a018012d..2aa3310a 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -169,6 +169,13 @@ "when": 1791231702996, "tag": "0023_version_push_session", "breakpoints": true + }, + { + "idx": 24, + "version": "6", + "when": 1792066046645, + "tag": "0024_stream_push", + "breakpoints": true } ] } diff --git a/packages/server/scripts/memory.ts b/packages/server/scripts/memory.ts new file mode 100644 index 00000000..bd2a98a5 --- /dev/null +++ b/packages/server/scripts/memory.ts @@ -0,0 +1,371 @@ +/** + * Memory of every push job on large records: uploads, compactions, the commit's + * planning, units and assembly, and indexing, run in this process on records + * shaped like bibliographic metadata (several KB, non-ASCII text, long reference + * lists). Prints, per job type, how many ran and the most live memory one held + * above where it started: JS heap plus array buffers, after a forced GC, sampled + * every few storage operations. Stored objects are on disk (a file store), and + * the shared cache is off, as the Cache API sits outside a Worker's isolate; the + * isolate's memory cache is the bulk Worker's 8 MB. + * + * node --expose-gc --import tsx scripts/memory.ts [options] + * + * --records N records in the first push (default 90000, about 600 MB) + * --update N records the second push changes, spread over the first's (default 20000) + * --commit-runs N compactConfig.commitRuns (default 4, so the commit compacts first) + * --piece-mb N compactConfig.pieceBytes in MB (default 128, so compactions split) + * --unit-mb N parallelConfig.unitBytes and aboveBytes in MB (default: production's) + * --sample N storage operations between samples (default 25) + * + * Budgets not named here are the production ones (parallelConfig, runConfig). + * A Worker has 128 MB per isolate, shared by the invocations it runs at once. + */ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { + ed25519Signer, + fileStore, + generateSigningKey, + memoryStore, + newSalt, + noCache, + type Store, +} from '@underlay/protocol' +import { and, asc, eq, lte, sql } from 'drizzle-orm' + +import '../src/handlers.js' +import { createApp } from '../src/app.js' +import { D1_MAX_BOUND_PARAMS, openNodeDb } from '../src/db/node.js' +import * as schema from '../src/db/schema.js' +import { BULK_JOBS, runJob, SqliteJobs } from '../src/jobs.js' +import type { JobMessage, Ports } from '../src/ports.js' +import { compactConfig } from '../src/push/compact.js' +import { MAX_BATCH_BYTES, MAX_BATCH_LINES } from '../src/push/delta.js' +import { parallelConfig } from '../src/push/parallel.js' +import { createStores } from '../src/stores.js' + +const args = process.argv.slice(2) +const opt = (name: string, def: number) => { + const i = args.indexOf(`--${name}`) + return i >= 0 ? Number(args[i + 1]) : def +} +const RECORDS = opt('records', 90_000) +const UPDATE = opt('update', 20_000) +const SAMPLE = opt('sample', 25) +compactConfig.commitRuns = opt('commit-runs', 4) +compactConfig.pieceBytes = opt('piece-mb', 128) * 1024 * 1024 +if (args.includes('--unit-mb')) + parallelConfig.unitBytes = parallelConfig.aboveBytes = opt('unit-mb', 0) * 1024 * 1024 + +const gc = (globalThis as { gc?: () => void }).gc +if (!gc) { + console.error('Run with node --expose-gc --import tsx scripts/memory.ts') + process.exit(2) +} +const MB = 1024 * 1024 +let parts = { heap: 0, buffers: 0 } +const live = () => { + gc() + const m = process.memoryUsage() + parts = { heap: m.heapUsed, buffers: m.arrayBuffers } + return m.heapUsed + m.arrayBuffers +} + +// --- Sampling ------------------------------------------------------------------------ + +let ops = 0 +let peak = 0 +let peakParts = { heap: 0, buffers: 0 } +const sample = () => { + if (++ops % SAMPLE !== 0) return + const m = live() + if (m > peak) { + peak = m + peakParts = parts + } +} +/** A store that samples memory every SAMPLE operations. */ +function sampled(store: Store): Store { + return new Proxy(store, { + get(target, prop, receiver) { + const v = Reflect.get(target, prop, receiver) + if ( + typeof v !== 'function' || + !['get', 'put', 'head', 'list', 'delete'].includes(String(prop)) + ) + return v + return async (...a: unknown[]) => { + sample() + const out = await (v as (...x: unknown[]) => Promise).apply(target, a) + sample() + return out + } + }, + }) +} + +const stats = new Map< + string, + { n: number; peak: number; heap: number; buffers: number; ms: number } +>() +async function measure(label: string, fn: () => Promise) { + const base = live() + const baseParts = parts + peak = base + peakParts = parts + const t = Date.now() + await fn() + sample() + const s = stats.get(label) ?? { n: 0, peak: 0, heap: 0, buffers: 0, ms: 0 } + s.n++ + if (peak - base > s.peak) { + s.peak = peak - base + s.heap = peakParts.heap - baseParts.heap + s.buffers = peakParts.buffers - baseParts.buffers + } + s.ms += Date.now() - t + stats.set(label, s) +} + +// --- Records shaped like bibliographic metadata --------------------------------------- + +// mulberry32: small, seedable, and good enough that ids don't collide. +let seed = 1 +const rand = () => { + seed = (seed + 0x6d2b79f5) | 0 + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296 +} +const pick = (xs: T[]) => xs[Math.floor(rand() * xs.length)]! +const WORDS = [ + 'analysis', + 'protein', + 'Étude', + 'über', + 'Gesellschaft', + 'climate', + 'résumé', + 'quantum', + '研究', + 'データ', + 'модель', + 'ação', + 'network', + 'learning', + 'cell', + 'cancer', + 'São', + 'Zürich', +] +const words = (n: number) => Array.from({ length: n }, () => pick(WORDS)).join(' ') +const token = () => + Math.floor(rand() * 36 ** 6).toString(36) + Math.floor(rand() * 36 ** 6).toString(36) + +function work(doi: string, version = 0) { + const authors = Array.from({ length: 2 + Math.floor(rand() * 18) }, () => ({ + given: words(1), + family: words(1), + sequence: 'additional', + affiliation: [{ name: words(4) }], + })) + const refs = Array.from({ length: Math.floor(rand() * 50) }, (_, i) => ({ + key: `ref${i}`, + DOI: `10.${1000 + Math.floor(rand() * 9000)}/${token()}`, + 'article-title': words(8), + author: words(2), + year: String(1950 + Math.floor(rand() * 75)), + unstructured: words(20), + })) + return { + id: doi, + // The type follows the DOI, so an update lands on the same record. + type: doi.charCodeAt(doi.length - 1) % 5 ? 'journal-article' : 'book-chapter', + data: { + DOI: doi, + title: [words(10)], + author: authors, + reference: refs, + abstract: rand() < 0.5 ? words(150) : undefined, + published: { 'date-parts': [[2000 + Math.floor(rand() * 26), 1 + Math.floor(rand() * 12)]] }, + version, + }, + } +} + +// --- Ports ----------------------------------------------------------------------------- + +const dir = await mkdtemp(join(tmpdir(), 'ul-memory-')) +const db = await openNodeDb(`file:${join(dir, 'db.sqlite')}`, { + maxBoundParams: D1_MAX_BOUND_PARAMS, +}) +const bucket = sampled(fileStore(join(dir, 'bucket'))) +const signer = await ed25519Signer(await generateSigningKey()) +const ports: Ports = { + db, + stores: createStores(db, noCache, { + bucket, + // File bytes aren't part of this; the store only has to presign. + files: memoryStore() as never, + repoPrefix: 'repo', + internalPrefix: 'internal', + memoryBytes: 8 * MB, + }), + cache: noCache, + signer: async () => signer, + jobs: new SqliteJobs(db), + waitUntil: (p) => void p.catch((err) => console.error(err)), + outboundFetch: (url, init) => fetch(url, init), + locationFetch: (req) => fetch(req), + locationKey: Buffer.from(crypto.getRandomValues(new Uint8Array(32))).toString('base64url'), +} +const app = createApp(() => ({ + ports, + config: { appUrl: 'http://memory', deployment: 'test' }, + authenticate: async () => ({ userId: 'u1', scope: 'session', collectionIds: null }), +})) +await db.insert(schema.organization).values({ id: 'org1', name: 'Org', slug: 'org' }) +await db.insert(schema.user).values({ id: 'u1', name: 'u1', email: 'u1@example.org' }) +await db.insert(schema.member).values({ organizationId: 'org1', userId: 'u1', role: 'owner' }) +const [collection] = await db + .insert(schema.collections) + .values({ organizationId: 'org1', slug: 'works', name: 'works', privateSalt: newSalt() }) + .returning() +await db.insert(schema.placements).values({ + collectionId: collection!.id, + locationId: schema.PLATFORM_LOCATION_ID, + role: 'primary', + sets: 'public+private', +}) +const path = '/api/collections/org/works' +const request = (p: string, init: { method?: string; body?: string; json?: unknown } = {}) => + app.fetch( + new Request(`http://memory${p}`, { + method: init.method ?? 'GET', + headers: { + 'content-type': init.json !== undefined ? 'application/json' : 'application/x-ndjson', + }, + ...(init.json !== undefined ? { body: JSON.stringify(init.json) } : {}), + ...(init.body !== undefined ? { body: init.body } : {}), + }), + ) + +/** Run queued jobs one at a time, interactive first (as claim orders them), measuring each. */ +async function drain() { + for (;;) { + const [job] = await db + .select() + .from(schema.jobs) + .where(and(eq(schema.jobs.status, 'queued'), lte(schema.jobs.runAt, new Date()))) + .orderBy( + sql`CASE WHEN ${schema.jobs.type} IN (${sql.join( + [...BULK_JOBS].map((t) => sql`${t}`), + sql`, `, + )}) THEN 1 ELSE 0 END`, + asc(schema.jobs.runAt), + ) + .limit(1) + if (!job) return + await db.delete(schema.jobs).where(eq(schema.jobs.id, job.id)) + await measure(job.type, () => runJob(job.payload as JobMessage, ports)) + } +} + +/** One push: open, upload batches up to the server's limits, commit, run every job. */ +async function push(records: Iterable, schemas: Record | undefined) { + const open = await request(`${path}/push`, { method: 'POST', json: schemas ? { schemas } : {} }) + const sid = ((await open.json()) as { session_id: string }).session_id + let lines: string[] = [] + let bytes = 0 + const send = async () => { + const body = lines.join('\n') + lines = [] + bytes = 0 + await measure('upload (request)', async () => { + const r = await request(`${path}/push/${sid}/records`, { method: 'POST', body }) + if (r.status !== 200) throw new Error(`upload: ${r.status} ${await r.text()}`) + }) + // Uploads queue compactions; run them as they come, as the queue would. + await drain() + } + for (const rec of records) { + const line = JSON.stringify(rec) + const n = Buffer.byteLength(line) + 1 + if (lines.length >= MAX_BATCH_LINES || bytes + n > MAX_BATCH_BYTES - 64 * 1024) await send() + lines.push(line) + bytes += n + } + if (lines.length) await send() + const t = Date.now() + const c = await request(`${path}/push/${sid}/commit`, { method: 'POST', json: { async: true } }) + if (c.status !== 202) throw new Error(`commit: ${c.status} ${await c.text()}`) + await drain() + const s = (await (await request(`${path}/push/${sid}`)).json()) as { + status: string + error: unknown + result: { semver: string; recordCount: number; changes: unknown } + } + if (s.status !== 'committed') + throw new Error(`commit ended ${s.status}: ${JSON.stringify(s.error)}`) + const { semver, recordCount, changes } = s.result + console.error(` ${semver}: ${recordCount} records, changes ${JSON.stringify(changes)}`) + return Date.now() - t +} + +const schemas = { 'journal-article': { type: 'object' }, 'book-chapter': { type: 'object' } } +const dois = Array.from( + { length: RECORDS }, + () => `10.${1000 + Math.floor(rand() * 9000)}/${token()}`, +) +let rawBytes = 0 +const first = function* () { + for (const d of dois) { + const w = work(d) + rawBytes += Buffer.byteLength(JSON.stringify(w)) + yield w + } +} +console.error(`push 1: ${RECORDS} records`) +const c1 = await push(first(), schemas) +console.error( + `push 1 committed (${(rawBytes / MB).toFixed(0)} MB of records, commit ${(c1 / 1000).toFixed(0)} s)`, +) +const second = function* () { + for (let i = 0; i < UPDATE; i++) yield work(dois[Math.floor(rand() * dois.length)]!, 1) +} +console.error(`push 2: ${UPDATE} updates spread over the first push's records`) +const c2 = await push(second(), undefined) +console.error(`push 2 committed (commit ${(c2 / 1000).toFixed(0)} s)`) + +const rows = [...stats].sort((a, b) => b[1].peak - a[1].peak) +console.log( + JSON.stringify( + { + records: RECORDS, + recordMB: Math.round(rawBytes / MB), + updates: UPDATE, + budgets: { + unitMB: parallelConfig.unitBytes / MB, + pieceMB: compactConfig.pieceBytes / MB, + commitRuns: compactConfig.commitRuns, + }, + jobs: Object.fromEntries( + rows.map(([k, v]) => [ + k, + { + n: v.n, + peakMB: +(v.peak / MB).toFixed(1), + heapMB: +(v.heap / MB).toFixed(1), + buffersMB: +(v.buffers / MB).toFixed(1), + seconds: Math.round(v.ms / 1000), + }, + ]), + ), + }, + null, + 2, + ), +) +await rm(dir, { recursive: true, force: true }) diff --git a/packages/server/scripts/plan-memory.ts b/packages/server/scripts/plan-memory.ts new file mode 100644 index 00000000..4904f7b3 --- /dev/null +++ b/packages/server/scripts/plan-memory.ts @@ -0,0 +1,86 @@ +/** + * Memory of a commit's planning at full size, without the data: run summaries + * as a 1 TB push of ~6.5 KB records leaves them (32 runs, 50,000 parts of 4 MB, + * a mark every 8,192 entries), parsed from JSON as the planner and each plan + * chunk read them, then split into units and sliced as they do. + * + * node --expose-gc --import tsx scripts/plan-memory.ts [--parts N] [--runs N] + */ +import { compareUtf8 } from '@underlay/protocol' + +import { parallelConfig, planRanges } from '../src/push/parallel.js' +import { + normalizeIndex, + type RunIndex, + type RunPart, + runMarks, + sliceRun, +} from '../src/push/runs.js' + +const args = process.argv.slice(2) +const opt = (name: string, def: number) => { + const i = args.indexOf(`--${name}`) + return i >= 0 ? Number(args[i + 1]) : def +} +const PARTS = opt('parts', 50_000) +const RUNS = opt('runs', 32) +const gc = (globalThis as { gc?: () => void }).gc! +const MB = 1024 * 1024 +const live = () => { + gc() + const m = process.memoryUsage() + return m.heapUsed + m.arrayBuffers +} + +// Keys: DOIs spread evenly, each run covering the whole key space (uploads in random order). +const doi = (n: number) => `10.${1000 + (n % 9000)}/${n.toString(36).padStart(10, '0')}` +const perRun = Math.ceil(PARTS / RUNS) +const texts: string[] = [] +for (let r = 0; r < RUNS; r++) { + const parts: RunPart[] = [] + const keys = Array.from({ length: perRun + 1 }, (_, i) => doi(i * RUNS + r)).sort(compareUtf8) + for (let i = 0; i < perRun; i++) { + parts.push({ + part: i, + first: `journal-article\u0000${keys[i]}`, + last: `journal-article\u0000${keys[i + 1]}`, + count: 3200, + bytes: 20 * MB, + length: 4 * MB, + ...(i % 3 === 0 ? { marks: [`journal-article\u0000${keys[i]}x`] } : {}), + }) + } + texts.push(JSON.stringify({ v: 2, seq: r + 1, tier: 3, parts })) +} +const textMB = texts.reduce((n, t) => n + t.length, 0) / MB + +const base = live() +const runs: RunIndex[] = texts.map((t) => normalizeIndex(JSON.parse(t))) +const parsed = live() - base +const parts = runs.flatMap((ix) => ix.parts) +const marks = runMarks(runs, { type: 'journal-article' }).map((m) => + m.slice(m.indexOf('\u0000') + 1), +) +const ranges = planRanges( + 'journal-article', + marks, + parts, + { nodes: [], leafBytes: 0 }, + parallelConfig.unitBytes, +) +const planned = live() - base +let sliceBytes = 0 +for (const r of ranges.slice(0, 50)) + for (const ix of runs) + sliceBytes += JSON.stringify(sliceRun(ix, { type: 'journal-article', ...r })).length +console.log( + JSON.stringify({ + runs: RUNS, + parts: PARTS, + summaryJsonMB: +textMB.toFixed(1), + parsedMB: +(parsed / MB).toFixed(1), + plannedMB: +(planned / MB).toFixed(1), + units: ranges.filter((r) => !r.gap).length, + sliceKB: +(sliceBytes / 50 / 1024).toFixed(1), + }), +) diff --git a/packages/server/src/api/push.ts b/packages/server/src/api/push.ts index 5b708d79..45856758 100644 --- a/packages/server/src/api/push.ts +++ b/packages/server/src/api/push.ts @@ -9,11 +9,12 @@ * GET /:owner/:slug/push/:sid status (and result after an async commit) * DELETE /:owner/:slug/push/:sid abandon */ -import { hashSchema, parseSemver } from '@underlay/protocol' -import { and, eq, isNull } from 'drizzle-orm' +import { hashSchema, parseSemver, schemaRelaxes } from '@underlay/protocol' +import { and, count, eq, isNotNull, isNull, sum } from 'drizzle-orm' import { type Context, Hono } from 'hono' import type { AppEnv } from '../app.js' +import { cleanupConfig } from '../cleanup/config.js' import * as schema from '../db/schema.js' import { SMALL_UPLOAD_BYTES } from '../files/files.js' import { registerJob } from '../jobs.js' @@ -26,6 +27,7 @@ import { MAX_BATCH_LINES, } from '../push/delta.js' import { finalizeSession } from '../push/finalize.js' +import { unitProgress } from '../push/recover.js' import { createSession, getSession, @@ -123,15 +125,34 @@ async function commitRoute(c: Context, session: SessionRow) { body.async === true || session.recordsReceived > commitConfig.asyncAbove || (await revalidates(ports, session)) > commitConfig.asyncAbove - if ( - // The error of a commit refused for missing files is cleared on the next try. - !(await transition(ports, session.id, 'open', 'committing', { - finalizeStartedAt: new Date(), - error: null, - })) - ) { + const started = retryable(session) + ? // A commit that failed on the server's side goes again, from the session's + // uploads (still in storage) and a fresh plan. + await transition(ports, session.id, 'failed', 'committing', { + finalizeStartedAt: new Date(), + error: null, + commitPlan: null, + commitPhase: null, + assemblyLease: null, + stalls: 0, + progressAt: new Date(), + expiresAt: new Date(Date.now() + SESSION_TTL_MS), + }) + : // The error of a commit refused for missing files is cleared on the next try. + await transition(ports, session.id, 'open', 'committing', { + finalizeStartedAt: new Date(), + error: null, + }) + if (!started) { const now = await getSession(ports, session.id) if (now?.status === 'committed') return c.json(now.result ?? {}, 201) + if (now?.status === 'failed') + return jsonError( + c, + 409, + 'Session failed, and not in a way committing again can fix; push again in a new session', + { cause: now.error ?? null }, + ) return jsonError(c, 409, `Session is ${now?.status ?? 'gone'}`) } if (wantsAsync) { @@ -149,6 +170,23 @@ async function commitRoute(c: Context, session: SessionRow) { return c.json(outcome.body, outcome.status) } +/** + * Can a failed session be committed again? When the failure was the server's + * (500: a commit step kept failing or stopped moving; 503: storage cleanup got + * in the way), and its uploads are still in storage with time to spare before + * cleanup deletes them. A refusal (422, 409: invalid records, a moved base) is + * the same the next time. + */ +function retryable(s: SessionRow): boolean { + const code = s.error?.statusCode + return ( + s.status === 'failed' && + s.cleanedAt === null && + (code === 500 || code === 503) && + s.expiresAt.getTime() > Date.now() - cleanupConfig.sessionGraceMs + 60 * 60 * 1000 + ) +} + /** * Records of the base the commit has to revalidate: those of every type whose * schema the session changes. A small push that changes a large type's schema @@ -175,13 +213,22 @@ async function revalidates(ports: Ports, session: SessionRow): Promise { let n = 0 for (const [slug, body] of Object.entries(inputs.schemas)) { const before = was.get(slug) - if (before && before !== hashSchema(body)) n += base.typeCounts[slug] ?? 0 + if (!before || before === hashSchema(body)) continue + // A change that can't reject a record isn't revalidated. + const repo = await ports.stores.forCollection(session.collectionId) + if (schemaRelaxes(await repo.schema(before), body)) continue + n += base.typeCounts[slug] ?? 0 } return n } -function statusBody(s: SessionRow) { - return { +/** + * A session's status. While it takes uploads or commits, `progress` says where + * it is: its runs (and compactions running), then the commit's phase + * (compacting, planning, units, assembling, validating) and its units done. + */ +async function statusBody(ports: Ports, s: SessionRow) { + const body = { session_id: s.id, status: s.status, records_received: s.recordsReceived, @@ -191,6 +238,33 @@ function statusBody(s: SessionRow) { result: s.result ?? null, error: s.error ?? null, } + if (s.status !== 'open' && s.status !== 'committing') return body + const r = schema.pushRuns + const [runs] = await ports.db + .select({ n: count(), bytes: sum(r.bytes) }) + .from(r) + .where(eq(r.sessionId, s.id)) + const [compacting] = await ports.db + .select({ n: count() }) + .from(r) + .where(and(eq(r.sessionId, s.id), isNotNull(r.mergingInto))) + const units = s.commitPlan ? await unitProgress(ports, s.commitPlan) : null + const phase = + s.status === 'open' + ? null + : (s.commitPhase ?? + (!units ? 'committing' : units.done < units.total ? 'units' : 'assembling')) + return { + ...body, + progress: { + runs: runs?.n ?? 0, + run_bytes: Number(runs?.bytes ?? 0), + runs_compacting: compacting?.n ?? 0, + phase, + units, + progress_at: s.progressAt, + }, + } } async function abandon(c: Context, session: SessionRow) { @@ -331,7 +405,7 @@ export function pushRoutes() { if (access instanceof Response) return access const session = await ownSession(c, access) if (session instanceof Response) return session - return c.json(statusBody(session)) + return c.json(await statusBody(c.var.ports, session)) }) app.delete('/:owner/:slug/push/:sid', async (c) => { diff --git a/packages/server/src/api/versions.ts b/packages/server/src/api/versions.ts index 5e5c1569..0937732f 100644 --- a/packages/server/src/api/versions.ts +++ b/packages/server/src/api/versions.ts @@ -39,6 +39,7 @@ import { countAfter, findVersion, getRecord, + recordBodies, loadView, typeRecords, type VersionRow, @@ -569,47 +570,52 @@ export function versionRoutes() { : await previousVersion(c.var.ports.db, view.version) if (fromParam && !from) return jsonError(c, 404, `Version ${fromParam} not found`) const fromView = from ? await loadView(view.repo, from, view.owner, view.withheld) : null - const added: object[] = [] - const updated: object[] = [] + // Added and updated records as their stored lines: a stored record is + // `{"id":…,"type":…,"data":…}`, which is the page's item as it is, so it's + // passed through as text rather than parsed and written out again. + const added: string[] = [] + const updated: string[] = [] const removed: { id: string; type: string }[] = [] let n = 0 let next: string | null = null let last: { type: string; key: string } | null = null - const body = async (side: VersionView, d: { type: string; key: string }) => { - const t = side.types.find((x) => x.slug === d.type)! - const r = await getRecord(side, t, d.key) - return { id: d.key, type: d.type, data: (JSON.parse(r!.body!) as { data: unknown }).data } - } + // A page is capped by its records' bytes as well as their count: 5,000 + // records of several KB each don't fit a Worker as one JSON page. const page: TypedDiff[] = [] + let bytes = 0 for await (const d of diffAll(fromView, view, cursor)) { - if (n === limit) { + const size = d.after?.size ?? 0 + if (n === limit || (n > 0 && bytes + size > diffLimits.pageBytes)) { next = encodeCursor(last!.type, last!.key) break } n++ last = d + bytes += size page.push(d) } - // A page's bodies are read BODY_READS at a time, not one after another. - const bodies = await mapConcurrent(page, BODY_READS, (d) => - d.after ? body(view, d) : Promise.resolve(null), + // Bodies: each leaf the page touches is read once, streamed. + const found = await recordBodies( + view, + page.filter((d) => d.after).map((d) => ({ type: d.type, key: d.key, set: d.afterSet })), ) - page.forEach((d, i) => { - if (!d.before) added.push(bodies[i]!) + const body = (d: TypedDiff) => { + const b = found.get(`${d.type}\u0000${d.key}`) + if (b === undefined) throw new Error(`Record ${d.type}/${d.key} missing from its version`) + return b + } + for (const d of page) { + if (!d.before) added.push(body(d)) else if (!d.after) removed.push({ id: d.key, type: d.type }) - else updated.push(bodies[i]!) - }) + else updated.push(body(d)) + } const schemaChanged = !fromView || JSON.stringify(fromView.types.map((t) => [t.slug, t.schemaHash])) !== JSON.stringify(view.types.map((t) => [t.slug, t.schemaHash])) // File counts re-diff both file trees: on the first page only, not every page. const fileDelta = cursor ? { added: 0, removed: 0 } : await fileCounts(fromView, view) - return c.json({ - from: from?.semver ?? null, - to: view.version.semver, - added, - updated, + const rest = JSON.stringify({ removed, pagination: { limit, hasMore: next !== null, nextCursor: next }, meta: { @@ -620,6 +626,12 @@ export function versionRoutes() { filesRemoved: fileDelta.removed, }, }) + const head = JSON.stringify({ from: from?.semver ?? null, to: view.version.semver }) + return c.body( + `${head.slice(0, -1)},"added":[${added.join(',')}],"updated":[${updated.join(',')}],${rest.slice(1)}`, + 200, + { 'content-type': 'application/json' }, + ) }) app.get('/:owner/:slug/versions/:n/files', async (c) => { @@ -714,26 +726,8 @@ async function lookupShared( /** Versions a record history looks back over. */ const MAX_HISTORY = 500 -/** Concurrent body reads per diff page. */ -const BODY_READS = 16 - -/** `f` over `items`, at most `limit` at a time, results in order. */ -async function mapConcurrent( - items: T[], - limit: number, - f: (item: T) => Promise, -): Promise { - const out = new Array(items.length) - let next = 0 - const worker = async () => { - while (next < items.length) { - const i = next++ - out[i] = await f(items[i]!) - } - } - await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)) - return out -} +/** Record bytes per diff page, at most (past its first record). */ +export const diffLimits = { pageBytes: 4 * 1024 * 1024 } /** File hashes in the sets this view may read (deduplicated, sorted). */ async function visibleFiles(view: VersionView, max: number): Promise { diff --git a/packages/server/src/cleanup/config.ts b/packages/server/src/cleanup/config.ts index 71416f29..021e4725 100644 --- a/packages/server/src/cleanup/config.ts +++ b/packages/server/src/cleanup/config.ts @@ -48,7 +48,10 @@ export const cleanupConfig = { windowMs: 30_000, windowObjects: 400, deleteConcurrency: 16, - /** A push committing for less than this holds up a deletion window (it would fail otherwise). */ + /** + * A push that started committing, or made progress, less than this ago holds up + * a deletion window (it would fail otherwise). + */ committingHoldMs: 6 * HOUR, /** How long a sweep waits for pushes to finish before it tries again. */ waitSeconds: 120, diff --git a/packages/server/src/cleanup/sweep.ts b/packages/server/src/cleanup/sweep.ts index 8525ede4..67b38395 100644 --- a/packages/server/src/cleanup/sweep.ts +++ b/packages/server/src/cleanup/sweep.ts @@ -289,17 +289,23 @@ function counted(stats: schema.CleanupStats, c: Candidate) { sample(stats, c.kind, c.kind === 'collections' ? `collections/${c.owner}/` : c.key) } -/** Pushes committing now (recently started), which a deletion window would fail. */ -async function committing(ports: Ports): Promise { +/** + * Pushes committing now, which a deletion window would fail: started recently, + * or still making progress (a parallel commit of a large push can run for many + * hours; its jobs record progress, and the stalled-commit sweep fails one that + * stops, so a commit that has moved lately is alive). + */ +export async function committing(ports: Ports): Promise { + const since = new Date(Date.now() - cleanupConfig.committingHoldMs) const [row] = await ports.db .select({ id: schema.pushSessions.id }) .from(schema.pushSessions) .where( and( eq(schema.pushSessions.status, 'committing'), - gt( - schema.pushSessions.finalizeStartedAt, - new Date(Date.now() - cleanupConfig.committingHoldMs), + or( + gt(schema.pushSessions.finalizeStartedAt, since), + gt(schema.pushSessions.progressAt, since), ), ), ) diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 00631a1b..7d169741 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -600,6 +600,15 @@ export const pushSessions = sqliteTable( finalizeStartedAt: ts('finalize_started_at'), /** When storage cleanup deleted the session's objects (sessions//) and run rows. */ cleanedAt: ts('cleaned_at'), + /** + * Where a committing session is: 'compacting' while its runs are merged down + * to what one commit reads (push/compact.ts), then null. + */ + commitPhase: text('commit_phase'), + /** Last step a commit's jobs finished; the sweep nudges a session that stops moving. */ + progressAt: ts('progress_at'), + /** Nudges since the last progress; past a few the commit fails (push/recover.ts). */ + stalls: integer('stalls').notNull().default(0), /** A parallel commit's plan (push/parallel.ts): set once, when its units are queued. */ commitPlan: text('commit_plan'), /** Held by the `commit.assemble` job while it runs (one assembler at a time). */ @@ -625,6 +634,8 @@ export const pushRuns = sqliteTable( kind: text('kind', { enum: ['records', 'deletes'] }).notNull(), objectKey: text('object_key').notNull(), count: integer('count').notNull(), + /** Raw NDJSON size, about: compaction and commits are sized by it. */ + bytes: integer('bytes').notNull().default(0), /** Run key range, `type\u0000id`, for planning commit units. */ firstKey: text('first_key').notNull(), lastKey: text('last_key').notNull(), @@ -632,6 +643,8 @@ export const pushRuns = sqliteTable( tier: integer('tier').notNull().default(0), /** Claimed by the compaction writing run `mergingInto`; null while free. */ mergingInto: integer('merging_into'), + /** When the compaction claimed it: a claim that never finishes is queued again. */ + claimedAt: ts('claimed_at'), createdAt: createdAt(), }, (t) => [ @@ -671,6 +684,8 @@ export const commitUnits = sqliteTable( outputKey: text('output_key'), /** When the unit was put on the queue; null while it waits for room (push/parallel.ts). */ queuedAt: ts('queued_at'), + /** Times the unit was put on the queue; past a few the commit fails (push/recover.ts). */ + attempts: integer('attempts').notNull().default(0), createdAt: createdAt(), }, (t) => [ diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index f20b255d..a87ecb5f 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -37,6 +37,7 @@ import { recheckLocations } from './locations/locations.js' import { laggingPlacements, queueMirrors } from './locations/mirror.js' import type { Ports } from './ports.js' import { expireSessions } from './push/finalize.js' +import { recoverCommits } from './push/recover.js' import { queueInfoBackfill } from './versions/collection-info.js' import { appendVersionLog } from './versions/commit.js' import { enqueueDeliveries, purgeOldDeliveries } from './webhooks/webhooks.js' @@ -53,6 +54,8 @@ registerJob('version.published', async (job, ports) => { registerJob('maintenance.sweep', async (_job, ports) => { await expireSessions(ports) + // Commits and compactions that stopped moving: queued again, or failed with a reason. + await recoverCommits(ports) await purgeOldDeliveries(ports) // Weekly counter reconciliation, a few collections at a time. await reconcileDue(ports) diff --git a/packages/server/src/jobs.ts b/packages/server/src/jobs.ts index d80660ee..8f6cf6d7 100644 --- a/packages/server/src/jobs.ts +++ b/packages/server/src/jobs.ts @@ -22,9 +22,13 @@ export type JobHandler = (job: JobMessage, ports: Ports) => Promise /** Job types that go to the bulk queue. Everything else is interactive. */ export const BULK_JOBS: ReadonlySet = new Set([ 'push.commit', // async commits: 100k records and up + 'commit.plan', 'commit.unit', 'commit.assemble', + 'commit.validate', + 'commit.publish', 'push.compact', + 'push.compactPiece', 'refs.index', 'refs.indexUnit', 'refs.compact', diff --git a/packages/server/src/push/compact.ts b/packages/server/src/push/compact.ts index 55e0cfe4..58150f46 100644 --- a/packages/server/src/push/compact.ts +++ b/packages/server/src/push/compact.ts @@ -1,12 +1,24 @@ /** - * Run compaction during upload. Whenever MERGE_FAN_IN runs of one tier - * accumulate in a session, a `push.compact` job merges them into one run of the - * next tier, up to MAX_TIER. A session therefore holds O(log) runs per tier plus - * capped top-tier runs, and a commit unit can read its key range from all of - * them at once. + * Run compaction: the merge phase of the session's external sort. Two triggers: + * + * - During upload, whenever MERGE_FAN_IN runs of one tier accumulate, they + * merge into one run of the next tier (a size-tiered LSM, no top tier). A + * session of any size therefore holds at most MERGE_FAN_IN − 1 runs per + * tier, O(log) tiers. + * - At commit (compactForCommit), the session's runs are merged down to at + * most `commitRuns`, smallest first, so every commit job reads a bounded + * number of runs. The session waits in phase 'compacting' and the last + * compaction to finish queues the commit again. + * + * A compaction is sized by bytes, not runs: one over `pieceBytes` is split into + * key-range pieces (cut at the inputs' part boundaries), each a `push.compactPiece` + * job writing its own parts of the output run; the piece that finds every piece + * written joins their part lists into the run's index. Every step is idempotent: + * a piece already written is skipped, the join writes the same index, and the + * row swap is a conditional batch. * * A compaction claims its inputs by setting `merging_into` to the seq of the run - * it will write, in one statement, so two jobs never merge the same run. When + * it will write (with `claimed_at`, so the sweep can queue a lost one again). When * the merged run is written, one batch records it and drops the input rows. A * commit reads the run list in one query, so it sees either the inputs or the * merged run, never both; input objects stay until the session's objects expire, @@ -14,77 +26,310 @@ * * Precedence is per entry (runs.ts), so any group of runs can be merged. */ -import { and, asc, count, eq, inArray, isNull } from 'drizzle-orm' +import { compareUtf8 } from '@underlay/protocol' +import { and, asc, count, eq, inArray, isNull, or, sql } from 'drizzle-orm' import * as schema from '../db/schema.js' import { registerJob } from '../jobs.js' import type { Ports } from '../ports.js' -import { compactRuns, MAX_TIER, MERGE_FAN_IN, readRunIndex } from './runs.js' -import { getSession, nextRunSeq } from './session.js' +import { + compactRuns, + type KeyRange, + MERGE_FAN_IN, + readRunIndex, + runBytes, + runCount, + type RunIndex, + type RunPart, + sliceRun, + writeRunIndex, +} from './runs.js' +import { getSession, type SessionRow } from './session.js' + +export const compactConfig = { + /** Raw bytes one compaction job merges, about; a larger compaction is split into pieces. */ + pieceBytes: 512 * 1024 * 1024, + /** Runs one commit reads, at most: a session with more is compacted first. */ + commitRuns: 32, +} + +/** Part numbers per piece: piece n numbers its parts from n × this. */ +const PIECE_PARTS = 1_000_000 -type RunKind = 'records' | 'deletes' +const runDir = (sessionId: string, seq: number) => `sessions/${sessionId}/runs/${seq}` +const planKey = (sessionId: string, seq: number) => `${runDir(sessionId, seq)}/compaction.json` +const pieceKey = (sessionId: string, seq: number, piece: number) => + `${runDir(sessionId, seq)}/piece-${piece}.json` -/** Runs merged together: a session's records and deletes are one stream. */ -const GROUPS: Record = { - records: ['records', 'deletes'], - deletes: ['records', 'deletes'], +interface CompactionPlan { + inputs: number[] + tier: number + /** Piece i covers run keys (cuts[i − 1], cuts[i]]; the last piece runs to the end. */ + cuts: string[] } -/** After a run of `kind` and `tier` is recorded: claim a full group and queue its compaction. */ -export async function maybeCompact( - ports: Ports, - sessionId: string, - kind: RunKind, - tier: number, -): Promise { - if (tier >= MAX_TIER) return +/** Compactions run while the session takes uploads, or while its commit waits for them. */ +const active = (s: SessionRow | null) => + s?.status === 'open' || (s?.status === 'committing' && s.commitPhase === 'compacting') + +/** A run number for a compaction's output, while compaction may run. */ +async function compactionSeq(ports: Ports, sessionId: string): Promise { + const s = schema.pushSessions + const [row] = await ports.db + .update(s) + .set({ runs: sql`${s.runs} + 1` }) + .where( + and( + eq(s.id, sessionId), + or(eq(s.status, 'open'), and(eq(s.status, 'committing'), eq(s.commitPhase, 'compacting'))), + ), + ) + .returning({ runs: s.runs }) + return row?.runs ?? null +} + +/** Claim these runs for one compaction and queue it; false if any was already claimed. */ +async function claim(ports: Ports, sessionId: string, seqs: number[]): Promise { const runs = schema.pushRuns - const free = and( - eq(runs.sessionId, sessionId), - eq(runs.tier, tier), - inArray(runs.kind, GROUPS[kind]), - isNull(runs.mergingInto), - ) - const [row] = await ports.db.select({ n: count() }).from(runs).where(free) - if ((row?.n ?? 0) < MERGE_FAN_IN) return - const out = await nextRunSeq(ports, sessionId) - if (out === null) return - const oldest = ports.db - .select({ seq: runs.seq }) - .from(runs) - .where(free) - .orderBy(asc(runs.seq)) - .limit(MERGE_FAN_IN) + const out = await compactionSeq(ports, sessionId) + if (out === null) return false const claimed = await ports.db .update(runs) - .set({ mergingInto: out }) - .where(and(eq(runs.sessionId, sessionId), inArray(runs.seq, oldest))) + .set({ mergingInto: out, claimedAt: new Date() }) + .where(and(eq(runs.sessionId, sessionId), inArray(runs.seq, seqs), isNull(runs.mergingInto))) .returning({ seq: runs.seq }) - if (claimed.length < MERGE_FAN_IN) { - // Another upload claimed some of them first; leave the rest for later. + if (claimed.length < seqs.length) { + // Another compaction claimed some of them first; leave the rest for later. await ports.db .update(runs) - .set({ mergingInto: null }) + .set({ mergingInto: null, claimedAt: null }) .where(and(eq(runs.sessionId, sessionId), eq(runs.mergingInto, out))) - return + return false } await ports.jobs.enqueue({ type: 'push.compact', sessionId, seq: out }) + return true } -/** Write the compacted run `seq` from the runs claimed for it. Idempotent. */ -export async function compactClaimed(ports: Ports, sessionId: string, seq: number): Promise { +/** After a run of `tier` is recorded: claim a full tier and queue its compaction. */ +export async function maybeCompact(ports: Ports, sessionId: string, tier: number): Promise { const runs = schema.pushRuns - const claimed = and(eq(runs.sessionId, sessionId), eq(runs.mergingInto, seq)) - const inputs = await ports.db.select().from(runs).where(claimed).orderBy(asc(runs.seq)) + const free = and(eq(runs.sessionId, sessionId), eq(runs.tier, tier), isNull(runs.mergingInto)) + const [row] = await ports.db.select({ n: count() }).from(runs).where(free) + if ((row?.n ?? 0) < MERGE_FAN_IN) return + const oldest = await ports.db + .select({ seq: runs.seq }) + .from(runs) + .where(free) + .orderBy(asc(runs.seq)) + .limit(MERGE_FAN_IN) + await claim( + ports, + sessionId, + oldest.map((r) => r.seq), + ) +} + +/** + * Before a commit plans: are the session's runs few enough for one commit to + * read? If not, the session goes to phase 'compacting', its smallest free runs + * are claimed in groups until the count will fit, and 'waiting' is returned; + * each finished compaction queues the commit again. On 'ready' the phase is + * cleared, and runs still being compacted are read as they are. + */ +export async function compactForCommit( + ports: Ports, + session: SessionRow, +): Promise<'ready' | 'waiting'> { + const rows = await ports.db + .select() + .from(schema.pushRuns) + .where(eq(schema.pushRuns.sessionId, session.id)) + if (rows.length <= compactConfig.commitRuns) { + if (session.commitPhase !== null) await setPhase(ports, session.id, null) + return 'ready' + } + if (session.commitPhase !== 'compacting') await setPhase(ports, session.id, 'compacting') + const inFlight = new Set(rows.filter((r) => r.mergingInto !== null).map((r) => r.mergingInto)) + const free = rows.filter((r) => r.mergingInto === null).sort((a, b) => a.bytes - b.bytes) + // What the run list comes to once every claimed compaction is done. + let n = free.length + inFlight.size + while (n > compactConfig.commitRuns && free.length >= 2) { + const k = Math.min(MERGE_FAN_IN, n - compactConfig.commitRuns + 1, free.length) + const group = free.splice(0, k) + if ( + await claim( + ports, + session.id, + group.map((r) => r.seq), + ) + ) + n -= k - 1 + } + return 'waiting' +} + +async function setPhase(ports: Ports, sessionId: string, phase: string | null) { + await ports.db + .update(schema.pushSessions) + .set({ commitPhase: phase, progressAt: new Date(), stalls: 0 }) + .where(eq(schema.pushSessions.id, sessionId)) +} + +/** + * Key-range cuts that split a compaction into pieces of about `pieceBytes`, at + * the inputs' part boundaries. A part straddling a cut is read by both pieces. + */ +export function pieceCuts(indexes: RunIndex[], pieceBytes: number): string[] { + const parts: RunPart[] = indexes.flatMap((ix) => ix.parts) + parts.sort((a, b) => compareUtf8(a.last, b.last)) + const cuts: string[] = [] + let acc = 0 + for (let i = 0; i < parts.length - 1; i++) { + acc += parts[i]!.bytes + const key = parts[i]!.last + if (acc >= pieceBytes && key !== cuts[cuts.length - 1] && key !== parts[i + 1]!.last) { + cuts.push(key) + acc = 0 + } + } + return cuts +} + +const pieceRange = (cuts: string[], piece: number): KeyRange => ({ + after: piece === 0 ? null : cuts[piece - 1]!, + through: piece === cuts.length ? null : cuts[piece]!, +}) + +async function claimedInputs(ports: Ports, sessionId: string, seq: number) { + const runs = schema.pushRuns + return ports.db + .select() + .from(runs) + .where(and(eq(runs.sessionId, sessionId), eq(runs.mergingInto, seq))) + .orderBy(asc(runs.seq)) +} + +/** + * The compaction is still moving: date its claim afresh. The sweep (recover.ts) + * queues a compaction again when its claim is older than the stall window, so + * without this a compaction of many pieces, hours long, would have every + * unfinished piece queued again each sweep from the window on. + */ +async function refreshClaim(ports: Ports, sessionId: string, seq: number) { + const runs = schema.pushRuns + await ports.db + .update(runs) + .set({ claimedAt: new Date() }) + .where(and(eq(runs.sessionId, sessionId), eq(runs.mergingInto, seq))) +} + +/** Write the compacted run `seq` from the runs claimed for it, or queue its pieces. Idempotent. */ +export async function compactClaimed(ports: Ports, sessionId: string, seq: number): Promise { + const inputs = await claimedInputs(ports, sessionId, seq) if (inputs.length === 0) return - // A session that stopped taking uploads commits from the inputs as they are. - const session = await getSession(ports, sessionId) - if (session?.status !== 'open') return + if (!active(await getSession(ports, sessionId))) return const store = ports.stores.internal const indexes = await Promise.all(inputs.map((r) => readRunIndex(store, sessionId, r.seq))) - const tier = inputs[0]!.tier + 1 - const index = await compactRuns(store, sessionId, indexes, seq, tier) - const kind = inputs.every((r) => r.kind === inputs[0]!.kind) ? inputs[0]!.kind : 'records' + const tier = Math.max(...inputs.map((r) => r.tier)) + 1 + const total = indexes.reduce((n, ix) => n + runBytes(ix), 0) + if (total <= compactConfig.pieceBytes) { + await refreshClaim(ports, sessionId, seq) + const index = await compactRuns(store, sessionId, indexes, seq, tier) + await finishCompaction(ports, sessionId, seq, index) + return + } + let plan: CompactionPlan + const existing = await store.get(planKey(sessionId, seq)) + if (existing) plan = JSON.parse(await existing.text()) as CompactionPlan + else { + plan = { + inputs: inputs.map((r) => r.seq), + tier, + cuts: pieceCuts(indexes, compactConfig.pieceBytes), + } + await store.put(planKey(sessionId, seq), JSON.stringify(plan), { + contentType: 'application/json', + }) + } + const done = await piecesDone(ports, sessionId, seq) + const todo = Array.from({ length: plan.cuts.length + 1 }, (_, piece) => piece).filter( + (piece) => !done.has(piece), + ) + if (todo.length === 0) await joinPieces(ports, sessionId, seq, plan) + else + await ports.jobs.enqueueBatch( + todo.map((piece) => ({ type: 'push.compactPiece', sessionId, seq, piece })), + ) +} + +async function piecesDone(ports: Ports, sessionId: string, seq: number): Promise> { + const done = new Set() + const prefix = `${runDir(sessionId, seq)}/piece-` + let cursor: string | undefined + do { + const page = await ports.stores.internal.list(prefix, cursor) + for (const key of page.keys) done.add(Number(key.slice(prefix.length, -'.json'.length))) + cursor = page.cursor + } while (cursor) + return done +} + +/** One key-range piece of a split compaction; the last piece to finish joins them. */ +export async function compactPiece( + ports: Ports, + sessionId: string, + seq: number, + piece: number, +): Promise { + const store = ports.stores.internal + const planObj = await store.get(planKey(sessionId, seq)) + if (!planObj) return + const plan = JSON.parse(await planObj.text()) as CompactionPlan + if ((await claimedInputs(ports, sessionId, seq)).length === 0) return // joined already + if (!active(await getSession(ports, sessionId))) return + if (!(await store.head(pieceKey(sessionId, seq, piece)))) { + const range = pieceRange(plan.cuts, piece) + const slices = await Promise.all( + plan.inputs.map(async (s) => sliceRun(await readRunIndex(store, sessionId, s), range)), + ) + const out = await compactRuns( + store, + sessionId, + slices.filter((ix) => ix.parts.length > 0), + seq, + plan.tier, + { range, partBase: piece * PIECE_PARTS }, + ) + await store.put(pieceKey(sessionId, seq, piece), JSON.stringify(out.parts), { + contentType: 'application/json', + }) + await refreshClaim(ports, sessionId, seq) + await touch(ports, sessionId) + } + const done = await piecesDone(ports, sessionId, seq) + if (done.size === plan.cuts.length + 1) await joinPieces(ports, sessionId, seq, plan) +} + +async function joinPieces(ports: Ports, sessionId: string, seq: number, plan: CompactionPlan) { + const store = ports.stores.internal + const parts: RunPart[] = [] + for (let piece = 0; piece <= plan.cuts.length; piece++) { + const obj = await store.get(pieceKey(sessionId, seq, piece)) + if (!obj) throw new Error(`Compaction piece ${piece} of run ${seq} is missing`) + parts.push(...(JSON.parse(await obj.text()) as RunPart[])) + } + const index: RunIndex = { seq, tier: plan.tier, parts } + await writeRunIndex(store, sessionId, index) + await finishCompaction(ports, sessionId, seq, index) +} + +/** Record the merged run in place of its inputs, then carry on: the next tier, or the commit. */ +async function finishCompaction(ports: Ports, sessionId: string, seq: number, index: RunIndex) { + const runs = schema.pushRuns + const inputs = await claimedInputs(ports, sessionId, seq) + if (inputs.length === 0) return + const tier = index.tier ?? 0 + const kind = inputs.every((r) => r.kind === 'deletes') ? 'deletes' : 'records' await ports.db.batch([ ports.db .insert(runs) @@ -92,18 +337,36 @@ export async function compactClaimed(ports: Ports, sessionId: string, seq: numbe sessionId, seq, kind, - objectKey: `sessions/${sessionId}/runs/${seq}`, - count: index.blocks.reduce((n, b) => n + b.count, 0), - firstKey: index.blocks[0]?.first ?? '', - lastKey: index.blocks[index.blocks.length - 1]?.last ?? '', + objectKey: runDir(sessionId, seq), + count: runCount(index), + bytes: runBytes(index), + firstKey: index.parts[0]?.first ?? '', + lastKey: index.parts[index.parts.length - 1]?.last ?? '', tier, }) .onConflictDoNothing(), - ports.db.delete(runs).where(claimed), + ports.db.delete(runs).where(and(eq(runs.sessionId, sessionId), eq(runs.mergingInto, seq))), ]) - await maybeCompact(ports, sessionId, kind, tier) + const session = await getSession(ports, sessionId) + if (session?.status === 'open') await maybeCompact(ports, sessionId, tier) + else if (session?.status === 'committing' && session.commitPhase === 'compacting') { + await touch(ports, sessionId) + await ports.jobs.enqueue({ type: 'push.commit', sessionId }) + } +} + +/** A commit's job finished a step: the sweep leaves a moving session alone. */ +export async function touch(ports: Ports, sessionId: string) { + await ports.db + .update(schema.pushSessions) + .set({ progressAt: new Date(), stalls: 0 }) + .where(and(eq(schema.pushSessions.id, sessionId), eq(schema.pushSessions.status, 'committing'))) } registerJob('push.compact', async (job, ports) => { await compactClaimed(ports, String(job.sessionId), Number(job.seq)) }) + +registerJob('push.compactPiece', async (job, ports) => { + await compactPiece(ports, String(job.sessionId), Number(job.seq), Number(job.piece)) +}) diff --git a/packages/server/src/push/delta.ts b/packages/server/src/push/delta.ts index 50522e7f..ae7107be 100644 --- a/packages/server/src/push/delta.ts +++ b/packages/server/src/push/delta.ts @@ -30,7 +30,7 @@ import type { Ports } from '../ports.js' import { type BaseVersion, commitVersion, type TypeInput } from '../versions/commit.js' import { deltaChanges, isPrivateSchema, stageOutOfLine } from './changes.js' import type { SessionCommitResult } from './outcome.js' -import { planParallel } from './parallel.js' +import { planParallel, SerialTooLargeError } from './parallel.js' import { type RunEntry, type RunIndex, writeRun } from './runs.js' import { loadInputs, @@ -44,7 +44,8 @@ import { transition, } from './session.js' -export const MAX_BATCH_BYTES = 16 * 1024 * 1024 +/** 8 MB: an upload request holds its batch several times over (text, records, run lines). */ +export const MAX_BATCH_BYTES = 8 * 1024 * 1024 export const MAX_BATCH_LINES = 10_000 const MAX_REPORTED = 100 @@ -345,8 +346,13 @@ export async function commitDeltaSession( const root = base ? await repo.root(base.hash) : null const priv = root?.private ? await repo.privateSet(root.private) : null const sets = { pub: root?.public ?? emptySet(), priv: priv ?? emptySet() } - if (await planParallel(ports, session, { inputs, runs, base, repo, fence, ...sets })) { - return { status: 'parallel' } + try { + if (await planParallel(ports, session, { inputs, runs, base, repo, fence, ...sets })) { + return { status: 'parallel' } + } + } catch (err) { + if (err instanceof SerialTooLargeError) return { status: 'schema_refused', error: err.message } + throw err } return commitVersion(ports, { collectionId: session.collectionId, diff --git a/packages/server/src/push/finalize.ts b/packages/server/src/push/finalize.ts index 61149a52..d1a7c373 100644 --- a/packages/server/src/push/finalize.ts +++ b/packages/server/src/push/finalize.ts @@ -12,6 +12,7 @@ import { and, eq, lt } from 'drizzle-orm' import { fenced } from '../cleanup/fence.js' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' +import { compactForCommit } from './compact.js' import { commitDeltaSession } from './delta.js' import { commitOutcome, @@ -36,6 +37,9 @@ export async function finalizeSession(ports: Ports, sessionId: string): Promise< await resumeParallel(ports, session) return pendingOutcome(sessionId) } + // Too many runs for one commit to read: compact them first. The last + // compaction queues this job again. + if ((await compactForCommit(ports, session)) === 'waiting') return pendingOutcome(sessionId) // The change streams are rebuilt from the session's runs on each attempt. const outcome = await commitOutcome(ports, sessionId, () => fenced(ports.db, (fence) => commitDeltaSession(ports, session, fence)), diff --git a/packages/server/src/push/outcome.ts b/packages/server/src/push/outcome.ts index 2c2adeb0..6fc72aaf 100644 --- a/packages/server/src/push/outcome.ts +++ b/packages/server/src/push/outcome.ts @@ -162,6 +162,8 @@ export async function settleSession( error: outcome.body as never, finalizeStartedAt: null, commitPlan: null, + commitPhase: null, + stalls: 0, assemblyLease: null, expiresAt: new Date(Date.now() + SESSION_TTL_MS), }) diff --git a/packages/server/src/push/parallel.ts b/packages/server/src/push/parallel.ts index 5161ee9c..5247fe27 100644 --- a/packages/server/src/push/parallel.ts +++ b/packages/server/src/push/parallel.ts @@ -1,31 +1,42 @@ /** - * Parallel commit (edge-redesign-build.md, "Parallel commit"). One commit job - * handles a few million changed records in a Worker's CPU budget. A delta - * session bigger than that splits each (set, type) merge into key-range units - * that run as separate jobs, then one assembly job builds the interior levels - * and publishes. The trees come out byte-identical to the serial merge. + * Parallel commit (edge-redesign-build.md, "Parallel commit"). A delta session + * of any size commits as one version: each (set, type) merge splits into + * key-range units that run as separate jobs, then one assembly job builds the + * interior levels and publishes. The trees come out byte-identical to the + * serial merge. * - * plan (in the push.commit job) split candidates per (set, type): the - * runs' natural-boundary marks plus natural last keys of the base - * tree's upper level. Intervals no run block touches become gaps, - * left to assembly; the rest are grouped into units of about - * `unitEntries` run entries. Each unit gets a slice object listing - * the run blocks overlapping its range, so it reads nothing else. + * plan (in the push.commit job, once compaction has left at most + * `compactConfig.commitRuns` runs) split candidates per (set, + * type): the runs' natural-boundary marks plus natural last keys of + * the base tree's nodes at a level chosen by the tree's size. + * Intervals no run touches become gaps, left to assembly; the rest + * are grouped into units of about `unitBytes` of work: the changes' + * raw bytes plus the base leaves they will rewrite. The plan (base, + * trees, every unit's range) is one object; nothing here reads more + * than the runs' summaries and the base's upper levels. + * commit.plan one chunk of the plan's units: writes each unit's slice + * (the run parts overlapping its range) and its row, under + * an id derived from the plan, so a retry rewrites the same + * rows. D1 runs at most 1,000 queries an invocation, so rows + * are written in chunks; the last chunk ends phase 'planning'. * commit.unit mergeTree in range mode: the range's leaves, change * counts and file reference deltas, to an output object. - * commit.assemble one at a time (a lease on the session). If a unit's - * `through` was deleted, merge it with the following ranges - * up to the first that survived, queue the new unit, and - * stop. Otherwise assembleTree per (set, type), then + * commit.assemble one at a time (a lease on the session), never while the + * plan is still being written. If a unit's `through` was + * deleted, merge it with the following ranges up to the + * first that survived, queue the new unit, and stop. + * Otherwise assembleTree per (set, type), then * commitVersion with the prebuilt trees. * * Only commits where every type is plain go parallel: no type changes privacy * or schema, and none is removed. Those need O(type size) work that stays in - * the serial commit for now. + * the serial commit for now, so the serial commit refuses one over + * `serialMaxBytes` with an error that says to split the schema change off. */ import { assembleTree, bodyOfRecord, + compileSchema, boundaryBytes, compareUtf8, dropRecordBody, @@ -43,10 +54,13 @@ import { type Segment, type SetName, type SetObject, + schemaRelaxes, + sha256Hex, + tracksTypes, trailingZeros, type TreeSummary, } from '@underlay/protocol' -import { and, asc, count, eq, gte, inArray, isNotNull, isNull, lt, or } from 'drizzle-orm' +import { and, asc, count, eq, gte, inArray, isNotNull, isNull, lt, or, sql } from 'drizzle-orm' import { chunks } from '../db/chunks.js' import * as schema from '../db/schema.js' @@ -54,8 +68,17 @@ import { registerJob } from '../jobs.js' import type { Ports } from '../ports.js' import { type BaseVersion, commitVersion } from '../versions/commit.js' import { deltaChanges, isPrivateSchema } from './changes.js' +import { touch } from './compact.js' import { commitOutcome, settleSession } from './outcome.js' -import { blocksInRange, type RunBlock, type RunIndex, runMarks } from './runs.js' +import { + normalizeIndex, + partsInRange, + readRunIndex, + type RunIndex, + runMarks, + type RunPart, + sliceRun, +} from './runs.js' import { getSession, loadInputs, @@ -65,24 +88,42 @@ import { } from './session.js' export const parallelConfig = { - /** Run entries above which a delta commit goes parallel. */ - above: 2_000_000, - /** Run entries per unit, about. */ - unitEntries: 1_000_000, - /** Level whose nodes' natural last keys are split candidates (2: about one per 4M records). */ - baseLevel: 2, + /** Estimated work (raw bytes) above which a delta commit goes parallel. */ + aboveBytes: 256 * 1024 * 1024, + /** Work per unit, about: changes' raw bytes plus the base leaves they rewrite. */ + unitBytes: 256 * 1024 * 1024, + /** Base nodes per tree the planner may read split keys from: it picks the lowest level under this. */ + baseNodes: 50_000, + /** A fixed base level (tests); null picks one by `baseNodes`. */ + baseLevel: null as number | null, + /** Units per `commit.plan` chunk (D1: about 1,000 queries an invocation). */ + chunkUnits: 1_000, + /** A commit the serial path can't take over this (a schema or privacy change) is refused. */ + serialMaxBytes: 2 * 1024 * 1024 * 1024, } /** How long an assembler holds the session before another may take over. */ const LEASE_MS = 15 * 60 * 1000 /** Body bytes a leaf holds before they go to a compressed member (each compresses with its raw text in memory). */ const SPILL_BYTES = 1024 * 1024 +/** A unit's repository writes in flight: leaf bodies (compressed) and their raw bytes. */ +const unitSink = { concurrency: 8, pendingBytes: 8 * 1024 * 1024 } const builderOpts = { payloadBytes: recordPayloadBytes, dropPayload: dropRecordBody, spillBytes: SPILL_BYTES, } +/** One unit's range, as planned. */ +interface UnitSpec { + set: SetName + type: string + ord: number + after: string | null + through: string | null + gap: boolean +} + interface CommitPlan { id: string /** @@ -92,8 +133,27 @@ interface CommitPlan { */ fence?: number base: BaseVersion | null - /** Per type: its base trees, which the units merge into. */ - types: Record + /** + * Per type: its base trees (`public`, `private`: which sets the base has, + * for the change streams), the trees its units merge into (`bases`: the + * moved tree for a privacy change; plans before moves: the base trees), a + * privacy move, and whether its new trees are revalidated in jobs. + */ + types: Record< + string, + { + privateType: boolean + public: TreeSummary + private: TreeSummary + bases?: { public: TreeSummary; private: TreeSummary } + move?: 'toPrivate' | 'toPublic' + validate?: boolean + } + > + /** The runs the units read (their summaries are sliced per unit). Plans before chunks: absent. */ + runs?: number[] + units?: UnitSpec[] + chunks?: number } interface UnitOutput { @@ -117,26 +177,50 @@ async function getJson(ports: Ports, key: string): Promise { return JSON.parse(await obj.text()) as T } +/** Run `fn` over items, at most `n` at a time. */ +async function pool(items: T[], n: number, fn: (item: T) => Promise): Promise { + let i = 0 + await Promise.all( + Array.from({ length: Math.min(n, items.length) }, async () => { + while (i < items.length) await fn(items[i++]!) + }), + ) +} + const emptyTree: TreeSummary = { root: null, count: 0, bytes: 0 } const summaryOf = (t: { root: string | null; count: number; bytes: number } | undefined) => t ? { root: t.root, count: t.count, bytes: t.bytes } : emptyTree const isNatural = (key: string) => trailingZeros(boundaryBytes(key)) >= LEAF_BOUNDARY_BITS -/** Natural last keys of the base tree's nodes at `level`, in order. */ -async function baseCandidates(repo: Repo, root: string | null, level: number): Promise { +/** Mean entries per leaf, and per interior child (protocol constants' means). */ +const LEAF_MEAN = 1024 +const FANOUT_MEAN = 64 + +/** Raw bytes of the base leaf a change lands in, about: a leaf's mean entries at the tree's mean size. */ +const leafBytesOf = (t: TreeSummary) => (t.count > 0 ? (t.bytes / t.count) * LEAF_MEAN : 0) + +/** The base level whose nodes the planner reads: the lowest with at most `baseNodes` of them. */ +function baseLevelFor(count: number): number { + if (parallelConfig.baseLevel !== null) return parallelConfig.baseLevel + let level = 1 + while (count / (LEAF_MEAN * FANOUT_MEAN ** (level - 1)) > parallelConfig.baseNodes) level++ + return level +} + +/** The base tree's nodes at `level` (or its root, when that's lower), in order. */ +async function baseNodes(repo: Repo, root: string | null, level: number): Promise { if (root === null) return [] const source = new RepoSource(recordTree, repo) const top = await rootDesc(source, root) - if (top.level <= level) return [] - const out: string[] = [] + if (top.level <= level) return [top] + const out: NodeDesc[] = [] const visit = async (desc: NodeDesc): Promise => { const node = await source.node(desc.hash) if (node.kind !== 'node') return for (const c of node.children) { - if (c.level === level) { - if (isNatural(c.lastKey)) out.push(c.lastKey) - } else await visit(c) + if (c.level === level) out.push(c) + else await visit(c) } } await visit(top) @@ -151,56 +235,81 @@ interface Range { /** * Group the intervals between candidates into units and gaps. An interval no - * block of the type overlaps holds no change, so it's a gap; its `through` is - * then an unchanged base key (a run mark lies in a block, so its interval is + * run part of the type overlaps holds no change, so it's a gap; its `through` + * is then an unchanged base key (a run mark lies in a part, so its interval is * touched), which is what assembly needs. + * + * An interval's work is its changes' raw bytes (a part's spread evenly over the + * intervals it spans), plus, when touched, the base bytes it may rewrite: at + * most its base nodes' bytes (spread the same way), and at most a leaf for each + * change. */ -function planRanges( +export function planRanges( slug: string, candidates: string[], - blocks: RunBlock[], - unitEntries: number, + parts: RunPart[], + base: { nodes: NodeDesc[]; leafBytes: number }, + unitBytes: number, ): Range[] { const n = candidates.length + 1 // Interval i is (candidates[i − 1], candidates[i]]; a key belongs to the first // interval whose upper candidate is ≥ it. - const intervalOf = (id: string | null) => { - if (id === null) return n - 1 + const firstAtLeast = (id: string, strict: boolean) => { let a = 0 let z = candidates.length while (a < z) { const m = (a + z) >> 1 - if (compareUtf8(candidates[m]!, id) >= 0) z = m + const c = compareUtf8(candidates[m]!, id) + if (strict ? c > 0 : c >= 0) z = m else a = m + 1 } return a } + const intervalOf = (id: string | null) => (id === null ? n - 1 : firstAtLeast(id, false)) const idOf = (runKey: string, edge: 'first' | 'last') => { const sep = runKey.indexOf('\u0000') if (runKey.slice(0, sep) === slug) return runKey.slice(sep + 1) - // A block reaching into a neighbouring type spans this type's whole edge. + // A part reaching into a neighbouring type spans this type's whole edge. return edge === 'first' ? '' : null } - // Difference arrays: how many blocks touch each interval, and its estimated - // entries (a block's count spread evenly over the intervals it spans). - const diff = new Int32Array(n + 1) - const rate = new Float64Array(n + 1) - for (const b of blocks) { - const i0 = intervalOf(idOf(b.first, 'first')) - const i1 = intervalOf(idOf(b.last, 'last')) - diff[i0]!++ - diff[i1 + 1]!-- - rate[i0]! += b.count / (i1 - i0 + 1) - rate[i1 + 1]! -= b.count / (i1 - i0 + 1) + // Difference arrays: parts touching each interval, and per-interval rates of + // change bytes, change entries and base bytes. + const touch = new Int32Array(n + 1) + const bytes = new Float64Array(n + 1) + const entries = new Float64Array(n + 1) + const baseBytes = new Float64Array(n + 1) + const spread = (arr: Float64Array, i0: number, i1: number, v: number) => { + arr[i0]! += v / (i1 - i0 + 1) + arr[i1 + 1]! -= v / (i1 - i0 + 1) + } + for (const p of parts) { + const i0 = intervalOf(idOf(p.first, 'first')) + const i1 = intervalOf(idOf(p.last, 'last')) + touch[i0]!++ + touch[i1 + 1]!-- + spread(bytes, i0, i1, p.bytes) + spread(entries, i0, i1, p.count) + } + let low: string | null = null + for (const node of base.nodes) { + // The node holds base keys in (low, lastKey]. + const i0 = low === null ? 0 : Math.min(n - 1, firstAtLeast(low, true)) + const i1 = intervalOf(node.lastKey) + if (i1 >= i0) spread(baseBytes, i0, i1, node.bytes) + low = node.lastKey } const out: Range[] = [] let unit: Range | null = null let acc = 0 let touched = 0 - let estimate = 0 + let b = 0 + let e = 0 + let bb = 0 for (let i = 0; i < n; i++) { - touched += diff[i]! - estimate += rate[i]! + touched += touch[i]! + b += bytes[i]! + e += entries[i]! + bb += baseBytes[i]! const after = i === 0 ? null : candidates[i - 1]! const through = i === n - 1 ? null : candidates[i]! if (touched === 0) { @@ -211,7 +320,8 @@ function planRanges( else out.push({ after, through, gap: true }) continue } - if (unit && acc > 0 && acc + estimate > unitEntries) { + const work = Math.max(0, b) + Math.max(0, Math.min(bb, e * base.leafBytes)) + if (unit && acc > 0 && acc + work > unitBytes) { out.push(unit) unit = null } @@ -220,16 +330,108 @@ function planRanges( acc = 0 } unit.through = through - acc += estimate + acc += work } if (unit) out.push(unit) return out } +/** Estimated work of a commit: per type, change bytes plus the base leaves the changes may rewrite. */ +function estimatedWork(runs: RunIndex[], trees: Map): number { + let total = 0 + for (const [slug, sets] of trees) { + const parts = runs.flatMap((ix) => partsInRange(ix, { type: slug })) + const changeBytes = parts.reduce((n, p) => n + p.bytes, 0) + const changes = parts.reduce((n, p) => n + p.count, 0) + total += changeBytes + for (const t of sets) total += Math.min(t.bytes, changes * leafBytesOf(t)) + } + return total +} + +/** How a type's trees get from the base to the new version (decided per type by the planner). */ +interface TypeMode { + /** The type's whole tree moves to the other set (a privacy change), as the serial build moves it. */ + move?: 'toPrivate' | 'toPublic' + /** Its schema changed in a way that can reject records: the new trees are revalidated in jobs. */ + validate: boolean + /** Only the serial build handles it; the bytes it would read or rewrite there. */ + serialOnly?: number +} + +/** + * Decide each type's mode. A privacy change goes parallel when the serial build + * would move the type's tree as it is and the base counts file references per + * type (so moving them is O(files)): public → private with nothing in the + * private set yet, and private → public with no private upsert of the type in + * this push. Anything else that changes privacy, and a type removed from a base + * without per-type counts, stays serial at the cost of the type. + */ +async function typeModes( + repo: Repo, + inputs: SessionInputs, + runs: RunIndex[], + base: BaseVersion | null, + pub: SetObject, + priv: SetObject, +) { + const tracked = { + public: await tracksTypes(repo, base?.publicRefsRoot ?? null), + private: await tracksTypes(repo, base?.privateRefsRoot ?? null), + } + const hashes = schemaHashes(inputs.schemas) + const modes: Record = {} + for (const [slug, s] of Object.entries(inputs.schemas)) { + const pubBase = pub.types[slug] + const privBase = priv.types[slug] + const prior = pubBase ?? privBase + const bytes = (pubBase?.bytes ?? 0) + (privBase?.bytes ?? 0) + const changed = !!prior && prior.schema !== hashes[slug] + const mode: TypeMode = { + validate: changed && !schemaRelaxes(await repo.schema(prior.schema), s), + } + const pubRoot = pubBase?.root ?? null + const privRoot = privBase?.root ?? null + if (isPrivateSchema(s) && pubRoot) { + if (!privRoot && tracked.public) mode.move = 'toPrivate' + else mode.serialOnly = bytes + } else if (!isPrivateSchema(s) && !pubBase && privBase) { + const privateUpserts = runs.some((ix) => partsInRange(ix, { type: slug }).some((p) => p.p)) + if (tracked.private && !privateUpserts) mode.move = 'toPublic' + else mode.serialOnly = bytes + } + modes[slug] = mode + } + // Removed types release their file references: O(files) with per-type counts, else O(type). + let removedSerial = 0 + for (const [set, types] of [ + ['public', pub.types], + ['private', priv.types], + ] as const) { + for (const [slug, t] of Object.entries(types)) { + if (!inputs.schemas[slug] && !tracked[set]) removedSerial += t.bytes + } + } + return { modes, removedSerial } +} + +/** The trees a type's units merge into: its base trees, or the moved tree. */ +function mergeBases(mode: TypeMode, pubBase: TreeSummary, privBase: TreeSummary) { + if (mode.move === 'toPrivate') return { public: emptyTree, private: pubBase } + if (mode.move === 'toPublic') return { public: privBase, private: emptyTree } + return { public: pubBase, private: privBase } +} + /** * Plan a parallel commit for a delta session, or return false when it should - * commit serially (small, or not every type is plain). On true the units are - * queued and the assembly job settles the session. + * commit serially (small, or a change only the serial build handles). On true + * the plan's chunks are queued and the assembly job settles the session. + * Throws SerialTooLargeError for a commit only the serial build can take that + * is too big for it. + * + * Work counts the changes, the base leaves they rewrite, the records a changed + * schema revalidates, and whatever only the serial build can do, so a small + * push that revalidates a large type goes parallel too. */ export async function planParallel( ports: Ports, @@ -245,78 +447,72 @@ export async function planParallel( }, ): Promise { const { inputs, runs, pub, priv } = ctx - const total = runs.reduce((n, r) => n + r.blocks.reduce((m, b) => m + b.count, 0), 0) - if (total <= parallelConfig.above) return false - const hashes = schemaHashes(inputs.schemas) - for (const [slug, s] of Object.entries(inputs.schemas)) { - const pubBase = pub.types[slug] - const privBase = priv.types[slug] - const prior = pubBase ?? privBase - if (prior && prior.schema !== hashes[slug]) return false - if (isPrivateSchema(s) ? !!pubBase : !pubBase && !!privBase) return false + const trees = new Map() + for (const slug of Object.keys(inputs.schemas)) + trees.set(slug, [summaryOf(pub.types[slug]), summaryOf(priv.types[slug])]) + const { modes, removedSerial } = await typeModes(ctx.repo, inputs, runs, ctx.base, pub, priv) + let serialOnly = removedSerial + let revalidate = 0 + for (const [slug, m] of Object.entries(modes)) { + serialOnly += m.serialOnly ?? 0 + if (m.validate) for (const t of trees.get(slug)!) revalidate += t.bytes } - for (const slug of [...Object.keys(pub.types), ...Object.keys(priv.types)]) { - if (!inputs.schemas[slug]) return false + const anySerialOnly = + removedSerial > 0 || Object.values(modes).some((m) => m.serialOnly !== undefined) + const work = estimatedWork(runs, trees) + revalidate + serialOnly + if (work <= parallelConfig.aboveBytes) return false + if (anySerialOnly) { + if (work > parallelConfig.serialMaxBytes) throw new SerialTooLargeError() + return false } const planId = crypto.randomUUID() const dir = planDir(session.id, planId) - const plan: CommitPlan = { id: planId, base: ctx.base, fence: ctx.fence, types: {} } - const rows: (typeof schema.commitUnits.$inferInsert)[] = [] + const plan: CommitPlan = { + id: planId, + base: ctx.base, + fence: ctx.fence, + types: {}, + runs: runs.map((r) => r.seq), + units: [], + } for (const [slug, s] of Object.entries(inputs.schemas)) { const privateType = isPrivateSchema(s) + const mode = modes[slug]! + const original = { public: summaryOf(pub.types[slug]), private: summaryOf(priv.types[slug]) } plan.types[slug] = { privateType, - public: summaryOf(pub.types[slug]), - private: summaryOf(priv.types[slug]), + ...original, + bases: mergeBases(mode, original.public, original.private), + ...(mode.move ? { move: mode.move } : {}), + ...(mode.validate ? { validate: true } : {}), } - const typeRuns = runs.map((ix) => ({ ix, blocks: blocksInRange(ix, { type: slug }) })) - const blocks = typeRuns.flatMap((r) => r.blocks) - if (blocks.length === 0) continue - const anyPrivate = blocks.some((b) => b.p) + const parts = runs.flatMap((ix) => partsInRange(ix, { type: slug })) + if (parts.length === 0) continue + const anyPrivate = parts.some((p) => p.p) const marks = runMarks(runs, { type: slug }).map((m) => m.slice(m.indexOf('\u0000') + 1)) for (const set of (privateType ? ['private'] : ['public', 'private']) as SetName[]) { - const baseRoot = plan.types[slug]![set].root + const tree = plan.types[slug]!.bases![set] // Nothing can reach a private tree that doesn't exist without a private upsert. - if (set === 'private' && !privateType && !baseRoot && !anyPrivate) continue - const fromBase = await baseCandidates(ctx.repo, baseRoot, parallelConfig.baseLevel) + if (set === 'private' && !privateType && !tree.root && !anyPrivate) continue + const nodes = await baseNodes(ctx.repo, tree.root, baseLevelFor(tree.count)) + const fromBase = nodes.map((d) => d.lastKey) const candidates = [...new Set([...marks, ...fromBase])].filter(isNatural).sort(compareUtf8) - const ranges = planRanges(slug, candidates, blocks, parallelConfig.unitEntries) - for (const [ord, r] of ranges.entries()) { - const id = crypto.randomUUID() - let slicesKey: string | null = null - if (!r.gap) { - slicesKey = `${dir}/${id}.slices.json` - const range = { type: slug, after: r.after, through: r.through } - const slices = typeRuns - .map(({ ix }) => ({ seq: ix.seq, tier: ix.tier, blocks: blocksInRange(ix, range) })) - .filter((x) => x.blocks.length > 0) - await putJson(ports, slicesKey, { runs: slices }) - } - rows.push({ - id, - sessionId: session.id, - planId, - set, - type: slug, - ord, - after: r.after, - through: r.through, - gap: r.gap, - status: r.gap ? 'done' : 'pending', - slicesKey, - }) - } + const ranges = planRanges( + slug, + candidates, + parts, + { nodes, leafBytes: leafBytesOf(tree) }, + parallelConfig.unitBytes, + ) + for (const [ord, r] of ranges.entries()) plan.units!.push({ set, type: slug, ord, ...r }) } } + plan.chunks = Math.max(1, Math.ceil(plan.units!.length / parallelConfig.chunkUnits)) await putJson(ports, `${dir}/plan.json`, plan) - // D1 binds at most 100 parameters per statement: 12 columns, 8 rows. - for (let i = 0; i < rows.length; i += 8) { - await ports.db.insert(schema.commitUnits).values(rows.slice(i, i + 8)) - } const won = await ports.db .update(schema.pushSessions) - .set({ commitPlan: planId }) + .set({ commitPlan: planId, commitPhase: 'planning', progressAt: new Date(), stalls: 0 }) .where( and( eq(schema.pushSessions.id, session.id), @@ -325,22 +521,129 @@ export async function planParallel( ), ) .returning({ id: schema.pushSessions.id }) - if (won.length === 0) { - // Another attempt planned first; its plan stands. - await ports.db.delete(schema.commitUnits).where(eq(schema.commitUnits.planId, planId)) - return true - } - await queueWork(ports, session.id, planId) + // Another attempt planned first; its plan stands. + if (won.length === 0) return true + await queueChunks(ports, session.id, planId, plan.chunks) return true } +/** A commit only the serial build can take, too large for it. */ +export class SerialTooLargeError extends Error { + constructor() { + super( + 'This push changes a type’s privacy in a way that needs the whole type rewritten (records already in both sets, or private records pushed as the type turns public), or removes a type from a collection that predates per-type file counts, and is too large to commit that way. Split the change off: push the privacy change or the removal on its own, with no records.', + ) + this.name = 'SerialTooLargeError' + } +} + +const chunkDone = (sessionId: string, planId: string, chunk: number) => + `${planDir(sessionId, planId)}/chunk-${chunk}.done` + +/** Queue the plan's chunks that aren't written yet. */ +async function queueChunks(ports: Ports, sessionId: string, planId: string, chunks: number) { + const todo: number[] = [] + for (let c = 0; c < chunks; c++) { + if (!(await ports.stores.internal.head(chunkDone(sessionId, planId, c)))) todo.push(c) + } + await ports.jobs.enqueueBatch( + todo.map((chunk) => ({ type: 'commit.plan', sessionId, planId, chunk })), + ) +} + +/** A unit's id, from its place in the plan: a chunk written twice writes the same rows. */ +const unitId = (planId: string, u: UnitSpec) => + sha256Hex(`${planId}\u0000${u.set}\u0000${u.type}\u0000${u.ord}`).slice(0, 32) + +/** Write one chunk of a plan's units: slices and rows. The last chunk ends phase 'planning'. */ +export async function writePlanChunk( + ports: Ports, + sessionId: string, + planId: string, + chunk: number, +): Promise { + const session = await getSession(ports, sessionId) + if (session?.status !== 'committing' || session.commitPlan !== planId) return + const dir = planDir(sessionId, planId) + const plan = await getJson(ports, `${dir}/plan.json`) + const store = ports.stores.internal + if (!(await store.head(chunkDone(sessionId, planId, chunk)))) { + const runs = await Promise.all(plan.runs!.map((seq) => readRunIndex(store, sessionId, seq))) + const specs = plan.units!.slice( + chunk * parallelConfig.chunkUnits, + (chunk + 1) * parallelConfig.chunkUnits, + ) + const rows: (typeof schema.commitUnits.$inferInsert)[] = specs.map((u) => { + const id = unitId(planId, u) + return { + id, + sessionId, + planId, + set: u.set, + type: u.type, + ord: u.ord, + after: u.after, + through: u.through, + gap: u.gap, + status: u.gap ? 'done' : 'pending', + slicesKey: u.gap ? null : `${dir}/${id}.slices.json`, + } + }) + await pool(rows, 16, async (row) => { + if (row.gap) return + const range = { type: row.type, after: row.after ?? null, through: row.through ?? null } + const slices = runs.map((ix) => sliceRun(ix, range)).filter((x) => x.parts.length > 0) + await putJson(ports, row.slicesKey!, { runs: slices }) + }) + // D1 binds at most 100 parameters per statement: 12 a row, 8 rows. + const inserts = [] + for (let i = 0; i < rows.length; i += 8) + inserts.push( + ports.db + .insert(schema.commitUnits) + .values(rows.slice(i, i + 8)) + .onConflictDoNothing(), + ) + for (let i = 0; i < inserts.length; i += 50) { + const part = inserts.slice(i, i + 50) + await ports.db.batch(part as [(typeof part)[0], ...typeof part]) + } + await store.put(chunkDone(sessionId, planId, chunk), '') + await touch(ports, sessionId) + } + // Units start as soon as their rows exist; assembly waits for the whole plan. + let done = 0 + for (let c = 0; c < plan.chunks!; c++) { + if (await store.head(chunkDone(sessionId, planId, c))) done++ + } + if (done === plan.chunks) { + await ports.db + .update(schema.pushSessions) + .set({ commitPhase: null }) + .where( + and( + eq(schema.pushSessions.id, sessionId), + eq(schema.pushSessions.commitPlan, planId), + eq(schema.pushSessions.commitPhase, 'planning'), + ), + ) + await queueWork(ports, sessionId, planId) + } else await topUpUnits(ports, session.userId) +} + /** * Commit units on the queue at once, per session and per account * (v2-scale-review.md S3): a billion-record push plans about a thousand units, * and they wait their turn instead of filling the bulk queue. A unit queued - * longer than `staleMs` ago and still pending counts as lost and goes again. + * longer than `staleMs` ago and still pending counts as lost and goes again; + * one queued `maxAttempts` times fails its commit (recover.ts). */ -export const unitLimits = { perSession: 32, perAccount: 64, staleMs: 30 * 60 * 1000 } +export const unitLimits = { + perSession: 32, + perAccount: 64, + staleMs: 30 * 60 * 1000, + maxAttempts: 4, +} /** Queue a plan's pending units (within the windows), or the assembly when none is left. Idempotent. */ async function queueWork( @@ -367,7 +670,7 @@ async function queueWork( * Put a user's waiting units on the queue, oldest session first, as far as the * windows allow. Runs after planning and after every unit, so the windows refill * as work finishes. A unit is claimed by setting `queuedAt`, so two top-ups - * never queue it twice. + * never queue it twice; each claim counts an attempt. */ export async function topUpUnits(ports: Ports, userId: string): Promise { const { db } = ports @@ -406,7 +709,11 @@ export async function topUpUnits(ports: Ports, userId: string): Promise { unitLimits.perAccount - account, ) if (room <= 0) continue - const waiting = or(isNull(u.queuedAt), lt(u.queuedAt, staleBefore)) + const waiting = and( + or(isNull(u.queuedAt), lt(u.queuedAt, staleBefore)), + // A unit out of attempts stays put; the sweep fails its commit. + lt(u.attempts, unitLimits.maxAttempts), + ) const next = await db .select({ id: u.id }) .from(u) @@ -418,7 +725,7 @@ export async function topUpUnits(ports: Ports, userId: string): Promise { for (const part of chunks(next.map((x) => x.id))) { const rows = await db .update(u) - .set({ queuedAt: new Date() }) + .set({ queuedAt: new Date(), attempts: sql`${u.attempts} + 1` }) .where(and(inArray(u.id, part), eq(u.status, 'pending'), waiting)) .returning({ id: u.id }) claimed.push(...rows.map((r) => r.id)) @@ -428,9 +735,32 @@ export async function topUpUnits(ports: Ports, userId: string): Promise { } } -/** A push.commit job found a plan already in place (a retry): make sure its work is queued. */ +/** A push.commit job found a plan already in place (a retry, or the sweep): make sure its work is queued. */ export async function resumeParallel(ports: Ports, session: SessionRow): Promise { - if (session.commitPlan) await queueWork(ports, session.id, session.commitPlan) + if (!session.commitPlan) return + if (session.commitPhase === 'planning') { + const plan = await getJson( + ports, + `${planDir(session.id, session.commitPlan)}/plan.json`, + ) + await queueChunks(ports, session.id, session.commitPlan, plan.chunks ?? 1) + return + } + if (session.commitPhase === 'validating') { + const { tasks } = await getJson<{ tasks: ValidationTask[] }>( + ports, + `${validationDir(session.id, session.commitPlan)}.json`, + ) + await queueValidation(ports, session.id, session.commitPlan, tasks.length) + return + } + await queueWork(ports, session.id, session.commitPlan) +} + +/** A unit's slices, in the current run index form (plans before pages listed blocks). */ +async function unitRuns(ports: Ports, slicesKey: string): Promise { + const { runs } = await getJson<{ runs: Parameters[0][] }>(ports, slicesKey) + return runs.map(normalizeIndex) } /** Run one unit. Idempotent: only a pending unit of the session's current plan runs. */ @@ -443,11 +773,12 @@ export async function runCommitUnit(ports: Ports, unitId: string): Promise const session = await getSession(ports, unit.sessionId) if (session?.status !== 'committing' || session.commitPlan !== unit.planId) return const plan = await getJson(ports, `${planDir(session.id, unit.planId)}/plan.json`) - const { runs } = await getJson<{ runs: RunIndex[] }>(ports, unit.slicesKey!) + const runs = await unitRuns(ports, unit.slicesKey!) const t = plan.types[unit.type]! const set = unit.set as SetName const repo = await ports.stores.forCollection(session.collectionId) - const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + // Writes in flight hold compressed leaves; halved from the default so two units fit an isolate. + const sink = new RepoSink(repo, { bodyOf: bodyOfRecord, ...unitSink }) const range = { after: unit.after, through: unit.through } const changes = deltaChanges( ports.stores.internal, @@ -462,7 +793,9 @@ export async function runCommitUnit(ports: Ports, unitId: string): Promise ) const leaves: NodeDesc[] = [] const refs = new FileRefDelta() - const merged = await mergeTree(new RepoSource(recordTree, repo), sink, t[set].root, changes, { + // A privacy change merges into the moved tree (`bases`). + const into = (t.bases ?? t)[set].root + const merged = await mergeTree(new RepoSource(recordTree, repo), sink, into, changes, { ...builderOpts, range, leafOutput: (d) => leaves.push(d), @@ -483,6 +816,7 @@ export async function runCommitUnit(ports: Ports, unitId: string): Promise .update(schema.commitUnits) .set({ status: survived ? 'done' : 'failed', outputKey }) .where(and(eq(schema.commitUnits.id, unit.id), eq(schema.commitUnits.status, 'pending'))) + await touch(ports, session.id) // The last unit to finish queues the assembly (more than one may; it holds a lease). const [left] = await ports.db .select({ id: schema.commitUnits.id }) @@ -520,26 +854,22 @@ async function mergeFailed(ports: Ports, sessionId: string, planId: string, rows let j = i while (j < group.length - 1 && group[j]!.status === 'failed') j++ const members = group.slice(i, j + 1) - const blocks = new Map< - number, - { seq: number; tier?: number; blocks: Map } - >() + const merged = new Map }>() for (const m of members) { if (!m.slicesKey) continue - const { runs } = await getJson<{ runs: RunIndex[] }>(ports, m.slicesKey) - for (const r of runs) { - const entry = blocks.get(r.seq) ?? { seq: r.seq, tier: r.tier ?? 0, blocks: new Map() } - for (const b of r.blocks) entry.blocks.set(`${b.part}:${b.offset}`, b) - blocks.set(r.seq, entry) + for (const r of await unitRuns(ports, m.slicesKey)) { + const entry = merged.get(r.seq) ?? { seq: r.seq, tier: r.tier ?? 0, parts: new Map() } + for (const p of r.parts) entry.parts.set(p.part, p) + merged.set(r.seq, entry) } } const id = crypto.randomUUID() const slicesKey = `${planDir(sessionId, planId)}/${id}.slices.json` await putJson(ports, slicesKey, { - runs: [...blocks.values()].map((r) => ({ + runs: [...merged.values()].map((r) => ({ seq: r.seq, tier: r.tier, - blocks: [...r.blocks.values()].sort((a, b) => a.part - b.part || a.offset - b.offset), + parts: [...r.parts.values()].sort((a, b) => compareUtf8(a.first, b.first)), })), }) const first = members[0]! @@ -576,6 +906,9 @@ async function mergeFailed(ports: Ports, sessionId: string, planId: string, rows export async function assembleParallel(ports: Ports, sessionId: string): Promise { const session = await getSession(ports, sessionId) if (session?.status !== 'committing' || !session.commitPlan) return + // The last plan chunk queues the assembly once every unit's row is written; + // once assembled, a commit that revalidates is in its validation jobs. + if (session.commitPhase === 'planning' || session.commitPhase === 'validating') return const planId = session.commitPlan const now = Date.now() const lease = await ports.db @@ -624,12 +957,14 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise const source = new RepoSource(recordTree, repo) const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) const built: Record = {} - for (const [slug, t] of Object.entries(plan.types)) - built[slug] = { public: t.public, private: t.private } + for (const [slug, t] of Object.entries(plan.types)) built[slug] = { ...(t.bases ?? t) } const refs = new FileRefDelta() const stats = { added: 0, removed: 0, updated: 0 } for (const group of trees(rows)) { const { set, type } = group[0]! as { set: SetName; type: string } + // A type made public counts every record of its new public tree as added + // (the build does that), not its public changes. + const countStats = !(plan.types[type]!.move === 'toPublic' && set === 'public') // Each segment's output is read once, when assembly reaches it. const segments: Segment[] = group .filter((u) => !u.gap) @@ -638,54 +973,318 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise through: u.through, leaves: async () => { const out = await getJson(ports, u.outputKey!) - stats.added += out.stats.added - stats.removed += out.stats.removed - stats.updated += out.stats.updated + if (countStats) { + stats.added += out.stats.added + stats.removed += out.stats.removed + stats.updated += out.stats.updated + } // A unit covers one type, so its deltas are that type's too. for (const [h, n] of out.refs) refs.add(set, type, h, n) return out.leaves }, })) - const result = await assembleTree(source, sink, plan.types[type]![set].root, segments) + const result = await assembleTree(source, sink, built[type]![set].root, segments) built[type]![set] = summaryOf( result.root ? { root: result.root.hash, count: result.root.count, bytes: result.root.bytes } : undefined, ) + await touch(ports, sessionId) } await sink.flush() - const inputs = await loadInputs(ports, sessionId) - const hashes = schemaHashes(inputs.schemas) - const outcome = await commitOutcome(ports, sessionId, () => - commitVersion(ports, { - collectionId: session.collectionId, - // Plans written before the fence existed fail if any window has opened since. - fence: plan.fence ?? 0, - base: plan.base, - types: Object.entries(inputs.schemas).map(([slug, s]) => ({ - slug, - schema: s, - schemaHash: hashes[slug]!, - public: null, - private: null, - })), - metadata: inputs.metadata, - declaredFiles: inputs.files, - message: session.message, - pushedBy: session.userId, - appId: session.appId, - actorId: session.actorId, - pushSessionId: session.id, - prebuilt: { trees: built, refs, stats }, - }), - ) - await settleSession(ports, sessionId, outcome) + const assembled: Assembled = { built, refs: refsOut(refs), stats } + if (Object.values(plan.types).some((t) => t.validate)) { + // A changed schema: the new trees are revalidated in jobs, then published. + await putJson(ports, `${planDir(sessionId, planId)}/assembled.json`, assembled) + await startValidation(ports, session, plan, built) + await release() + return + } + await publish(ports, session, plan, assembled) } catch (err) { await release() throw err } } +/** What assembly built, kept for the publish job when a schema is revalidated first. */ +interface Assembled { + built: Record + /** File reference deltas: [set, type, file hash, delta]. */ + refs: [SetName, string, string, number][] + stats: { added: number; removed: number; updated: number } +} + +const refsOut = (refs: FileRefDelta): Assembled['refs'] => + (['public', 'private'] as const).flatMap((set) => + [...refs.typed[set]].map(([k, n]) => { + const slash = k.lastIndexOf('/') + return [set, k.slice(0, slash), k.slice(slash + 1), n] as [SetName, string, string, number] + }), + ) + +function refsIn(rows: Assembled['refs']): FileRefDelta { + const refs = new FileRefDelta() + for (const [set, type, h, n] of rows) refs.add(set, type, h, n) + return refs +} + +/** Commit the assembled trees as the new version and settle the session. */ +async function publish(ports: Ports, session: SessionRow, plan: CommitPlan, a: Assembled) { + const inputs = await loadInputs(ports, session.id) + const hashes = schemaHashes(inputs.schemas) + const moves: Record = {} + for (const [slug, t] of Object.entries(plan.types)) if (t.move) moves[slug] = t.move + const outcome = await commitOutcome(ports, session.id, () => + commitVersion(ports, { + collectionId: session.collectionId, + // Plans written before the fence existed fail if any window has opened since. + fence: plan.fence ?? 0, + base: plan.base, + types: Object.entries(inputs.schemas).map(([slug, s]) => ({ + slug, + schema: s, + schemaHash: hashes[slug]!, + public: null, + private: null, + })), + metadata: inputs.metadata, + declaredFiles: inputs.files, + message: session.message, + pushedBy: session.userId, + appId: session.appId, + actorId: session.actorId, + pushSessionId: session.id, + prebuilt: { trees: a.built, refs: refsIn(a.refs), stats: a.stats, moves }, + }), + ) + await settleSession(ports, session.id, outcome) +} + +// --- Revalidating a changed schema ------------------------------------------------------ + +export const validateConfig = { + /** Record bytes one validation job reads, about. */ + taskBytes: 256 * 1024 * 1024, +} + +/** One validation job: one key range `(after, through]` of one new tree. */ +interface ValidationTask { + type: string + set: SetName + root: string + after: string | null + through: string | null +} + +interface ValidationResult { + total: number + errors: { recordId: string; type: string; errors: string[] }[] +} + +const MAX_REPORTED = 100 +const validationDir = (sessionId: string, planId: string) => + `${planDir(sessionId, planId)}/validate` + +/** + * Split the new trees of every type to revalidate into key ranges of about + * `taskBytes` (at node boundaries of the level the planner reads), and queue a + * job for each. The session goes to phase 'validating'; the job that finds + * every range done queues the publish. + */ +async function startValidation( + ports: Ports, + session: SessionRow, + plan: CommitPlan, + built: Assembled['built'], +) { + const repo = await ports.stores.forCollection(session.collectionId) + const tasks: ValidationTask[] = [] + for (const [type, t] of Object.entries(plan.types)) { + if (!t.validate) continue + for (const set of ['public', 'private'] as SetName[]) { + const tree = built[type]![set] + if (!tree.root) continue + const nodes = await baseNodes(repo, tree.root, baseLevelFor(tree.count)) + let after: string | null = null + let acc = 0 + for (const [i, n] of nodes.entries()) { + acc += n.bytes + const last = i === nodes.length - 1 + if (acc >= validateConfig.taskBytes || last) { + tasks.push({ type, set, root: tree.root, after, through: last ? null : n.lastKey }) + after = n.lastKey + acc = 0 + } + } + } + } + await putJson(ports, `${validationDir(session.id, plan.id)}.json`, { tasks }) + await ports.db + .update(schema.pushSessions) + .set({ commitPhase: 'validating', progressAt: new Date(), stalls: 0 }) + .where(and(eq(schema.pushSessions.id, session.id), eq(schema.pushSessions.commitPlan, plan.id))) + await queueValidation(ports, session.id, plan.id, tasks.length) +} + +/** Validation results written so far. */ +async function validationsDone(ports: Ports, sessionId: string, planId: string) { + const prefix = `${validationDir(sessionId, planId)}/` + const done = new Set() + let cursor: string | undefined + do { + const page = await ports.stores.internal.list(prefix, cursor) + for (const key of page.keys) done.add(Number(key.slice(prefix.length, -'.json'.length))) + cursor = page.cursor + } while (cursor) + return done +} + +/** Queue the validation jobs not done yet, or the publish when all are. */ +async function queueValidation(ports: Ports, sessionId: string, planId: string, count: number) { + const done = await validationsDone(ports, sessionId, planId) + const todo = Array.from({ length: count }, (_, i) => i).filter((i) => !done.has(i)) + if (todo.length === 0) { + await ports.jobs.enqueue({ type: 'commit.publish', sessionId }) + return + } + await ports.jobs.enqueueBatch( + todo.map((task) => ({ type: 'commit.validate', sessionId, planId, task })), + ) +} + +/** A tree's entries with bodies in `(after, through]`, leaf by leaf, each streamed. */ +async function* entriesInRange( + source: RepoSource, + root: string, + after: string | null, + through: string | null, +): AsyncGenerator { + const inRange = (k: string) => + (after === null || compareUtf8(k, after) > 0) && + (through === null || compareUtf8(k, through) <= 0) + async function* visit(desc: NodeDesc, low: string | null): AsyncGenerator { + if (after !== null && compareUtf8(desc.lastKey, after) <= 0) return + if (through !== null && low !== null && compareUtf8(low, through) >= 0) return + if (desc.level === 0) { + for await (const e of source.leafStream(desc.hash)) if (inRange(e.key)) yield e + return + } + const node = await source.node(desc.hash) + if (node.kind !== 'node') return + for (const [i, c] of node.children.entries()) + yield* visit(c, i === 0 ? low : node.children[i - 1]!.lastKey) + } + yield* visit(await rootDesc(source, root), null) +} + +/** One validation job. Idempotent: a range already checked is skipped. */ +export async function runValidation( + ports: Ports, + sessionId: string, + planId: string, + task: number, +): Promise { + const session = await getSession(ports, sessionId) + if ( + session?.status !== 'committing' || + session.commitPlan !== planId || + session.commitPhase !== 'validating' + ) + return + const { tasks } = await getJson<{ tasks: ValidationTask[] }>( + ports, + `${validationDir(sessionId, planId)}.json`, + ) + const key = `${validationDir(sessionId, planId)}/${task}.json` + if (!(await ports.stores.internal.head(key))) { + const t = tasks[task]! + const inputs = await loadInputs(ports, sessionId) + const validate = compileSchema(inputs.schemas[t.type]!) + const repo = await ports.stores.forCollection(session.collectionId) + const source = new RepoSource(recordTree, repo) + const result: ValidationResult = { total: 0, errors: [] } + for await (const e of entriesInRange(source, t.root, t.after, t.through)) { + const errs = validate((JSON.parse(e.body!) as { data: unknown }).data) + if (errs.length === 0) continue + result.total++ + if (result.errors.length < MAX_REPORTED) + result.errors.push({ recordId: e.key, type: t.type, errors: errs }) + } + await putJson(ports, key, result) + await touch(ports, sessionId) + } + const done = await validationsDone(ports, sessionId, planId) + if (done.size === tasks.length) await ports.jobs.enqueue({ type: 'commit.publish', sessionId }) +} + +/** After validation: publish the assembled version, or fail with the records that don't fit. */ +export async function publishValidated(ports: Ports, sessionId: string): Promise { + const session = await getSession(ports, sessionId) + if ( + session?.status !== 'committing' || + !session.commitPlan || + session.commitPhase !== 'validating' + ) + return + const planId = session.commitPlan + const dir = validationDir(sessionId, planId) + const { tasks } = await getJson<{ tasks: ValidationTask[] }>(ports, `${dir}.json`) + if ((await validationsDone(ports, sessionId, planId)).size < tasks.length) return + const now = Date.now() + const lease = await ports.db + .update(schema.pushSessions) + .set({ assemblyLease: new Date(now) }) + .where( + and( + eq(schema.pushSessions.id, sessionId), + eq(schema.pushSessions.commitPlan, planId), + or( + isNull(schema.pushSessions.assemblyLease), + lt(schema.pushSessions.assemblyLease, new Date(now - LEASE_MS)), + ), + ), + ) + .returning({ id: schema.pushSessions.id }) + if (lease.length === 0) return + try { + let total = 0 + const errors: ValidationResult['errors'] = [] + for (let i = 0; i < tasks.length; i++) { + const r = await getJson(ports, `${dir}/${i}.json`) + total += r.total + errors.push(...r.errors.slice(0, MAX_REPORTED - errors.length)) + } + if (total > 0) { + const outcome = await commitOutcome(ports, sessionId, async () => ({ + status: 'invalid' as const, + errors, + total, + })) + await settleSession(ports, sessionId, outcome) + return + } + const plan = await getJson(ports, `${planDir(sessionId, planId)}/plan.json`) + const assembled = await getJson( + ports, + `${planDir(sessionId, planId)}/assembled.json`, + ) + await publish(ports, session, plan, assembled) + } finally { + await ports.db + .update(schema.pushSessions) + .set({ assemblyLease: null }) + .where(eq(schema.pushSessions.id, sessionId)) + } +} + +registerJob('commit.plan', async (job, ports) => + writePlanChunk(ports, String(job.sessionId), String(job.planId), Number(job.chunk)), +) registerJob('commit.unit', async (job, ports) => runCommitUnit(ports, String(job.unitId))) registerJob('commit.assemble', async (job, ports) => assembleParallel(ports, String(job.sessionId))) +registerJob('commit.validate', async (job, ports) => + runValidation(ports, String(job.sessionId), String(job.planId), Number(job.task)), +) +registerJob('commit.publish', async (job, ports) => publishValidated(ports, String(job.sessionId))) diff --git a/packages/server/src/push/recover.ts b/packages/server/src/push/recover.ts new file mode 100644 index 00000000..c4716d7a --- /dev/null +++ b/packages/server/src/push/recover.ts @@ -0,0 +1,152 @@ +/** + * Recovering commits that stop moving, from the maintenance sweep (every 10 + * minutes). Every step of a commit is a job that may be lost: a queue message + * past its retries, a Worker that runs out of memory on every attempt. Each + * step records progress on the session (`progress_at`), so a session that + * hasn't moved for `stallMs` is nudged: its next step is queued again, and + * every step is idempotent. A commit that can't finish fails with an error + * that says why, rather than holding one of the user's session slots forever. + * + * - compaction claims older than `stallMs` are queued again (open sessions + * too: a lost compaction leaves its runs claimed); + * - a commit unit queued `unitLimits.maxAttempts` times fails its commit, + * naming the unit's type and range; + * - a commit waiting for room in the user's unit windows isn't stalled; + * - any other committing session without progress is nudged (push.commit + * resumes it from wherever it is), and after `maxStalls` nudges in a row + * without progress it fails. + */ +import { and, count, eq, isNotNull, isNull, lt, or } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { topUpUnits, unitLimits } from './parallel.js' +import { transition } from './session.js' + +export const recoverConfig = { + /** A commit step, or a compaction claim, with no progress for this long is stale. */ + stallMs: 30 * 60 * 1000, + /** Nudges in a row without progress before the commit fails. */ + maxStalls: 3, +} + +async function fail(ports: Ports, sessionId: string, error: string) { + await transition(ports, sessionId, 'committing', 'failed', { + error: { statusCode: 500, error }, + commitPhase: null, + }) + console.error(`[push] commit of session ${sessionId} failed: ${error}`) +} + +export async function recoverCommits(ports: Ports): Promise { + const { db } = ports + const staleBefore = new Date(Date.now() - recoverConfig.stallMs) + + // Compaction claims that never finished, in sessions still compacting: queue them again. + const runs = schema.pushRuns + const s = schema.pushSessions + const lost = await db + .selectDistinct({ sessionId: runs.sessionId, seq: runs.mergingInto }) + .from(runs) + .innerJoin(s, eq(s.id, runs.sessionId)) + .where( + and( + isNotNull(runs.mergingInto), + lt(runs.claimedAt, staleBefore), + or(eq(s.status, 'open'), and(eq(s.status, 'committing'), eq(s.commitPhase, 'compacting'))), + ), + ) + .limit(100) + for (const c of lost) { + await db + .update(runs) + .set({ claimedAt: new Date() }) + .where(and(eq(runs.sessionId, c.sessionId), eq(runs.mergingInto, c.seq!))) + await ports.jobs.enqueue({ type: 'push.compact', sessionId: c.sessionId, seq: c.seq }) + } + + const stuck = await db + .select() + .from(s) + .where( + and( + eq(s.status, 'committing'), + or( + lt(s.progressAt, staleBefore), + and(isNull(s.progressAt), lt(s.finalizeStartedAt, staleBefore)), + ), + ), + ) + .limit(100) + for (const session of stuck) { + if (session.commitPlan) { + const u = schema.commitUnits + const pending = await db + .select() + .from(u) + .where(and(eq(u.planId, session.commitPlan), eq(u.status, 'pending'))) + const unitStale = new Date(Date.now() - unitLimits.staleMs) + const spent = pending.find( + (x) => x.attempts >= unitLimits.maxAttempts && x.queuedAt && x.queuedAt < unitStale, + ) + if (spent) { + const range = `(${spent.after ?? 'start'}, ${spent.through ?? 'end'}]` + await fail( + ports, + session.id, + `A commit unit failed ${spent.attempts} times (type ${spent.type}, ${spent.set} set, keys ${range}). Push again; if it repeats, report this session.`, + ) + continue + } + // Units still waiting for room in the user's windows: alive, just queued. + if (pending.some((x) => x.queuedAt === null)) { + await db.update(s).set({ progressAt: new Date(), stalls: 0 }).where(eq(s.id, session.id)) + await topUpUnits(ports, session.userId) + continue + } + } + if (session.stalls >= recoverConfig.maxStalls) { + const where = + session.commitPhase === 'compacting' + ? 'compacting its uploads' + : session.commitPhase === 'planning' + ? 'planning' + : session.commitPhase === 'validating' + ? 'revalidating records against the new schema' + : session.commitPlan + ? 'committing in parallel' + : 'committing' + await fail( + ports, + session.id, + `The commit stopped making progress while ${where}. Push again; if it repeats, report this session.`, + ) + continue + } + await db + .update(s) + .set({ progressAt: new Date(), stalls: session.stalls + 1 }) + .where(and(eq(s.id, session.id), eq(s.status, 'committing'))) + // push.commit picks up from wherever the session is: compaction, the + // plan's chunks, the units (stale ones go again) or the assembly. + await ports.jobs.enqueue({ type: 'push.commit', sessionId: session.id }) + } +} + +/** A parallel commit's units: how many are done of how many (the status route). */ +export async function unitProgress(ports: Ports, planId: string) { + const u = schema.commitUnits + const rows = await ports.db + .select({ status: u.status, gap: u.gap, n: count() }) + .from(u) + .where(eq(u.planId, planId)) + .groupBy(u.status, u.gap) + let done = 0 + let total = 0 + for (const r of rows) { + if (r.status === 'superseded' || r.gap) continue + total += r.n + if (r.status === 'done') done += r.n + } + return { done, total } +} diff --git a/packages/server/src/push/runs.ts b/packages/server/src/push/runs.ts index 98ace16b..ab0677dd 100644 --- a/packages/server/src/push/runs.ts +++ b/packages/server/src/push/runs.ts @@ -7,25 +7,27 @@ * into part objects of a few MB, in the platform's internal area: * * sessions//runs//.ndjson.gz - * sessions//runs//index.json {seq, tier, blocks: [...]} + * sessions//runs//.blocks.json the part's blocks + * sessions//runs//index.json {v: 2, seq, tier, parts: [...]} * - * The index gives each block's first and last key, entry count, byte range in - * its part, and its natural-boundary marks (below). A reader fetches only the - * blocks overlapping a key range, by range GET, so a commit unit reads its - * slice of every run without reading the rest. + * The index is a summary: per part, its first and last key, entry count, raw + * and compressed size, and its natural-boundary marks (below). Each part's + * blocks (first and last key, entry count, byte range in the part) are a page + * of their own, read only by a cursor that reaches the part. So a planner reads + * O(parts) for any size of push, and a reader fetches only the blocks + * overlapping its key range, by range GET. Indexes written before pages + * (`{seq, tier, blocks}`) still read: their blocks become inline pages. * * Precedence: an entry's precedence is the seq of the upload it came from * (`q`, or the run's seq for an upload run). On equal keys the highest * precedence wins: later uploads replace earlier ones. Because precedence is - * carried per entry, any group of runs can be compacted into one. + * carried per entry, any group of runs can be compacted into one, and a + * compaction can be split by key range into pieces that run as separate jobs + * (compact.ts): the pieces' parts, in key order, are the merged run. * - * Compaction happens during upload (compact.ts): when MERGE_FAN_IN runs of one - * tier accumulate they're merged into one run of the next tier, up to - * MAX_TIER. Upload batches hold at most 10,000 entries, so a top-tier run holds - * at most about 2.5M, and a compaction always fits in one job. - * - * Memory: a reader holds one block's lines and a bounded prefetch of compressed - * bytes, so a k-way merge over hundreds of runs stays within a Worker. + * Memory: a reader holds one part's block page, one block's lines and a + * bounded prefetch of compressed bytes, so a k-way merge stays within a Worker + * for any run size. The number of runs a merge reads is bounded by compaction. */ import { boundaryBytes, @@ -65,22 +67,41 @@ export interface RunBlock { part: number offset: number length: number + /** Split-key candidates (indexes written before pages kept them per block). */ + marks?: string[] + p?: true +} + +export interface RunPart { + /** Part object number: a name, not a position (a compaction piece numbers its own). */ + part: number + /** First and last run keys. */ + first: string + last: string + count: number + /** Raw NDJSON size, about (string length), for byte budgets. */ + bytes: number + /** Size of the part object (compressed). */ + length: number /** - * Natural-boundary marks: run keys of the block's upserts whose id has at + * Natural-boundary marks: run keys of the part's upserts whose id has at * least MARK_BITS trailing zeros in its boundary hash. They're the split-key * candidates for parallel commit units, about one per 8k entries, so the * planner reads no data. */ marks?: string[] - /** Some upsert in the block was pushed as private. */ + /** Some upsert in the part was pushed as private. */ p?: true + /** The part's blocks, inline: indexes written before pages. Otherwise `.blocks.json`. */ + blocks?: RunBlock[] } export interface RunIndex { seq: number - /** 0 for an upload, n for a run compacted from runs of tier n − 1. */ + /** 0 for an upload, n for a run compacted from runs of tier ≤ n − 1. */ tier?: number - blocks: RunBlock[] + /** In key order; disjoint. A planner's slice of a run lists only the parts it needs. */ + parts: RunPart[] } /** A slice of one type: ids in `(after, through]`; null is unbounded. */ @@ -90,21 +111,29 @@ export interface RunRange { through?: string | null } +/** A range of run keys `(after, through]` across types (a compaction piece); null is unbounded. */ +export interface KeyRange { + after: string | null + through: string | null +} + export const BLOCK_BYTES = 64 * 1024 -export const PART_BYTES = 4 * 1024 * 1024 /** Runs per compaction. */ export const MERGE_FAN_IN = 16 -/** Runs of this tier are never compacted again. */ -export const MAX_TIER = 2 /** 2^13 = 8,192: about one mark per eight leaves. */ export const MARK_BITS = 13 -/** - * Compressed bytes a merge holds, split across its runs. Each cursor holds its - * current span and the next one, so a span gets half of a run's share. With - * Crossref-sized records, 8 MB a span left a two-run commit unit holding 16 MB - * of spans alone (2026-10-15). - */ -const PREFETCH_BUDGET = 4 * 1024 * 1024 + +export const runConfig = { + /** Compressed bytes per part object. A writer holds one open part and one being stored. */ + partBytes: 4 * 1024 * 1024, + /** + * Compressed bytes a merge holds, split across its runs. Each cursor holds its + * current span and the next one, so a span gets half of a run's share. With + * records of several KB, 8 MB a span left a two-run commit unit holding 16 MB + * of spans alone (2026-10-15). + */ + prefetchBudget: 4 * 1024 * 1024, +} /** Sort key: type, then id, by UTF-8 bytes. A NUL separator keeps types apart. */ export const runKey = (e: { t: string; k: string }) => `${e.t}\u0000${e.k}` @@ -114,13 +143,21 @@ export const compareRunKeys = (a: RunEntry, b: RunEntry) => const runPrefix = (sessionId: string, seq: number) => `sessions/${sessionId}/runs/${seq}` const partKey = (sessionId: string, seq: number, part: number) => `${runPrefix(sessionId, seq)}/${part}.ndjson.gz` +const pageKey = (sessionId: string, seq: number, part: number) => + `${runPrefix(sessionId, seq)}/${part}.blocks.json` +export const indexKey = (sessionId: string, seq: number) => + `${runPrefix(sessionId, seq)}/index.json` /** Is this id a split-key candidate? */ export const isMark = (id: string) => trailingZeros(boundaryBytes(id)) >= MARK_BITS +/** Entries in a run, and its raw size. */ +export const runCount = (ix: RunIndex) => ix.parts.reduce((n, p) => n + p.count, 0) +export const runBytes = (ix: RunIndex) => ix.parts.reduce((n, p) => n + p.bytes, 0) + /** Writes one run block by block. Entries must arrive sorted. */ export class RunWriter { - readonly #blocks: RunBlock[] = [] + readonly #parts: RunPart[] = [] #lines: string[] = [] #marks: string[] = [] #private = false @@ -130,17 +167,29 @@ export class RunWriter { // Blocks are compressed in order on one chain, then appended to the open part. #chain: Promise = Promise.resolve() #queued = 0 - #part = 0 + #part: number #partChunks: Uint8Array[] = [] #partBytes = 0 + #partBlocks: RunBlock[] = [] + #partMarks: string[] = [] + #partPrivate = false + #partRaw = 0 readonly #puts: Promise[] = [] + /** + * `partBase` numbers this writer's parts (a compaction piece writes into a + * run other pieces write too); `writeIndex: false` leaves the index to whoever + * joins the pieces. + */ constructor( readonly store: Store, readonly sessionId: string, readonly seq: number, readonly tier = 0, - ) {} + readonly opts: { partBase?: number; writeIndex?: boolean } = {}, + ) { + this.#part = opts.partBase ?? 0 + } add(e: RunEntry): void { const key = runKey(e) @@ -153,20 +202,17 @@ export class RunWriter { this.#lines.push(line) if (!e.x && isMark(e.k)) this.#marks.push(key) if (e.p) this.#private = true - this.#bytes += line.length + this.#bytes += line.length + 1 if (this.#bytes >= BLOCK_BYTES) this.#flushBlock() } #flushBlock() { if (this.#lines.length === 0) return const text = this.#lines.join('\n') + '\n' - const block = { - first: this.#first!, - last: this.#last!, - count: this.#lines.length, - ...(this.#marks.length > 0 ? { marks: this.#marks } : {}), - ...(this.#private ? { p: true as const } : {}), - } + const block = { first: this.#first!, last: this.#last!, count: this.#lines.length } + const marks = this.#marks + const priv = this.#private + const raw = this.#bytes this.#lines = [] this.#marks = [] this.#private = false @@ -175,11 +221,19 @@ export class RunWriter { this.#queued++ this.#chain = this.#chain.then(async () => { const gz = await gzip(text) - this.#blocks.push({ ...block, part: this.#part, offset: this.#partBytes, length: gz.length }) + this.#partBlocks.push({ + ...block, + part: this.#part, + offset: this.#partBytes, + length: gz.length, + }) + this.#partMarks.push(...marks) + if (priv) this.#partPrivate = true + this.#partRaw += raw this.#partChunks.push(gz) this.#partBytes += gz.length this.#queued-- - if (this.#partBytes >= PART_BYTES) this.#flushPart() + if (this.#partBytes >= runConfig.partBytes) this.#flushPart() }) } @@ -191,17 +245,44 @@ export class RunWriter { body.set(c, at) at += c.length } - const key = partKey(this.sessionId, this.seq, this.#part) - this.#puts.push(this.store.put(key, body, { contentType: 'application/gzip' })) + const blocks = this.#partBlocks + const part = this.#part + this.#parts.push({ + part, + first: blocks[0]!.first, + last: blocks[blocks.length - 1]!.last, + count: blocks.reduce((n, b) => n + b.count, 0), + bytes: this.#partRaw, + length: this.#partBytes, + ...(this.#partMarks.length > 0 ? { marks: this.#partMarks } : {}), + ...(this.#partPrivate ? { p: true as const } : {}), + }) + const store = this.store + this.#puts.push( + Promise.all([ + store.put(partKey(this.sessionId, this.seq, part), body, { + contentType: 'application/gzip', + }), + store.put(pageKey(this.sessionId, this.seq, part), JSON.stringify(blocks), { + contentType: 'application/json', + }), + ]).then(() => {}), + ) this.#part++ this.#partChunks = [] this.#partBytes = 0 + this.#partBlocks = [] + this.#partMarks = [] + this.#partPrivate = false + this.#partRaw = 0 } /** Bounded memory: callers writing large runs await this between entries. */ async drain(): Promise { if (this.#queued >= 4) await this.#chain - if (this.#puts.length >= 2) await Promise.all(this.#puts.splice(0)) + // One part being stored while the next fills: when a second part is + // flushed, wait for the first's put. Two parts in memory at most. + if (this.#puts.length >= 2) await this.#puts.shift() } async finish(): Promise { @@ -209,18 +290,18 @@ export class RunWriter { await this.#chain this.#flushPart() await Promise.all(this.#puts.splice(0)) - const index: RunIndex = { seq: this.seq, tier: this.tier, blocks: this.#blocks } - await this.store.put( - `${runPrefix(this.sessionId, this.seq)}/index.json`, - JSON.stringify(index), - { - contentType: 'application/json', - }, - ) + const index: RunIndex = { seq: this.seq, tier: this.tier, parts: this.#parts } + if (this.opts.writeIndex !== false) await writeRunIndex(this.store, this.sessionId, index) return index } } +export async function writeRunIndex(store: Store, sessionId: string, index: RunIndex) { + await store.put(indexKey(sessionId, index.seq), JSON.stringify({ v: 2, ...index }), { + contentType: 'application/json', + }) +} + /** Write an in-memory batch (one upload request) as a run. */ export async function writeRun(store: Store, sessionId: string, seq: number, entries: RunEntry[]) { entries.sort(compareRunKeys) @@ -234,71 +315,132 @@ export async function writeRun(store: Store, sessionId: string, seq: number, ent return w.finish() } +/** + * A run index (or a slice of one) as stored, in the current form. An index from + * before pages lists blocks: each part becomes a part with its blocks inline, + * its raw size estimated from the compressed one. + */ +export function normalizeIndex(raw: { + seq: number + tier?: number + parts?: RunPart[] + blocks?: RunBlock[] +}): RunIndex { + if (raw.parts) return { seq: raw.seq, tier: raw.tier ?? 0, parts: raw.parts } + const parts: RunPart[] = [] + for (const b of raw.blocks ?? []) { + let p = parts[parts.length - 1] + if (!p || p.part !== b.part) { + p = { part: b.part, first: b.first, last: b.last, count: 0, bytes: 0, length: 0, blocks: [] } + parts.push(p) + } + p.last = b.last + p.count += b.count + p.length += b.length + p.bytes += b.length * 4 + p.blocks!.push(b) + if (b.marks) p.marks = [...(p.marks ?? []), ...b.marks] + if (b.p) p.p = true + } + return { seq: raw.seq, tier: raw.tier ?? 0, parts } +} + export async function readRunIndex( store: Store, sessionId: string, seq: number, ): Promise { - const obj = await store.get(`${runPrefix(sessionId, seq)}/index.json`) + const obj = await store.get(indexKey(sessionId, seq)) if (!obj) throw new Error(`Run ${seq} of session ${sessionId} is missing`) - return JSON.parse(await obj.text()) as RunIndex + return normalizeIndex(JSON.parse(await obj.text())) } -const normalize = (range?: RunRange | string): RunRange | undefined => - typeof range === 'string' ? { type: range } : range +// --- Ranges ------------------------------------------------------------------------- + +/** Run-key bounds: keys k with lo < k, and k ≤ hi (k < hi when `hiOpen`). Null is unbounded. */ +interface Bounds { + lo: string | null + hi: string | null + hiOpen: boolean +} + +function boundsOf(range: RunRange | KeyRange | string): Bounds { + if (typeof range === 'string') range = { type: range } + if (!('type' in range)) return { lo: range.after, hi: range.through, hiOpen: false } + // Every key of the type sorts after `type\0` and before `type\u0001`. + return { + lo: `${range.type}\u0000${range.after ?? ''}`, + hi: range.through == null ? `${range.type}\u0001` : `${range.type}\u0000${range.through}`, + hiOpen: range.through == null, + } +} + +const inBounds = (b: Bounds, key: string) => + (b.lo === null || compareUtf8(key, b.lo) > 0) && + (b.hi === null || (b.hiOpen ? compareUtf8(key, b.hi) < 0 : compareUtf8(key, b.hi) <= 0)) /** Is the entry inside the range? */ -export function inRunRange(range: RunRange, e: { t: string; k: string }): boolean { - return ( - e.t === range.type && - (range.after == null || compareUtf8(e.k, range.after) > 0) && - (range.through == null || compareUtf8(e.k, range.through) <= 0) - ) +export function inRunRange(range: RunRange | KeyRange, e: { t: string; k: string }): boolean { + return inBounds(boundsOf(range), runKey(e)) } /** - * The blocks of a run that can hold entries in the range: a contiguous span, - * found by binary search (blocks are sorted and disjoint). + * The items (parts, or a part's blocks) that can hold keys in the bounds: a + * contiguous span, found by binary search (items are sorted and disjoint). */ -export function blocksInRange(index: RunIndex, range?: RunRange): RunBlock[] { - if (!range) return index.blocks - // Every key of the type sorts after `type\0` and before `type\u0001`. - const lo = `${range.type}\u0000${range.after ?? ''}` - const hi = range.through == null ? `${range.type}\u0001` : `${range.type}\u0000${range.through}` - const hiInclusive = range.through != null - const blocks = index.blocks - // The first block whose last key is past `lo`, and the first block starting past `hi`. - const search = (pred: (b: RunBlock) => boolean) => { +function overlapping(items: T[], b: Bounds): T[] { + const search = (pred: (x: T) => boolean) => { let a = 0 - let z = blocks.length + let z = items.length while (a < z) { const m = (a + z) >> 1 - if (pred(blocks[m]!)) z = m + if (pred(items[m]!)) z = m else a = m + 1 } return a } - const start = search((b) => compareUtf8(b.last, lo) > 0) - const end = search((b) => - hiInclusive ? compareUtf8(b.first, hi) > 0 : compareUtf8(b.first, hi) >= 0, - ) - return blocks.slice(start, Math.max(start, end)) + const start = b.lo === null ? 0 : search((x) => compareUtf8(x.last, b.lo!) > 0) + const end = + b.hi === null + ? items.length + : search((x) => + b.hiOpen ? compareUtf8(x.first, b.hi!) >= 0 : compareUtf8(x.first, b.hi!) > 0, + ) + return items.slice(start, Math.max(start, end)) +} + +/** The parts of a run that can hold entries in the range. */ +export function partsInRange(index: RunIndex, range?: RunRange | KeyRange | string): RunPart[] { + return range === undefined ? index.parts : overlapping(index.parts, boundsOf(range)) } +/** A run cut down to the parts a range needs (a commit unit's or a compaction piece's slice). */ +export const sliceRun = (index: RunIndex, range: RunRange | KeyRange): RunIndex => ({ + seq: index.seq, + tier: index.tier ?? 0, + parts: partsInRange(index, range), +}) + +// --- Reading --------------------------------------------------------------------------- + /** - * Reads one run's entries in order, optionally limited to a range. Neighbouring - * blocks of a part are fetched together, up to `spanBytes` compressed, and the - * next span is prefetched while the current one is consumed. Only the current - * block is decompressed, and lines are parsed one at a time. + * Reads one run's entries in order, optionally limited to a range. Part by + * part: the part's block page is read, then neighbouring blocks are fetched + * together, up to `spanBytes` compressed, the next span prefetched while the + * current one is consumed. Only the current block is decompressed, and lines + * are parsed one at a time. */ export class RunCursor { - readonly #spans: RunBlock[][] = [] + readonly #parts: RunPart[] + #part = 0 + #page: Promise | null = null + #spans: RunBlock[][] = [] #span = 0 #fetch: Promise | null = null #blocks: Uint8Array[] = [] #lines: string[] = [] #line = 0 - readonly #range: RunRange | undefined + readonly #bounds: Bounds | undefined /** The current entry, or undefined once the run is exhausted. */ head: RunEntry | undefined @@ -306,28 +448,46 @@ export class RunCursor { readonly store: Store, readonly sessionId: string, readonly index: RunIndex, - range?: RunRange | string, - spanBytes = 1024 * 1024, + range?: RunRange | KeyRange | string, + readonly spanBytes = 1024 * 1024, ) { - this.#range = normalize(range) + this.#bounds = range === undefined ? undefined : boundsOf(range) + this.#parts = this.#bounds ? overlapping(index.parts, this.#bounds) : index.parts + } + + #loadPage(i: number): Promise { + const part = this.#parts[i]! + if (part.blocks) return Promise.resolve(part.blocks) + const key = pageKey(this.sessionId, this.index.seq, part.part) + const p = this.store.get(key).then(async (obj) => { + if (!obj) throw new Error(`Run page ${key} is missing`) + return JSON.parse(await obj.text()) as RunBlock[] + }) + p.catch(() => {}) + return p + } + + /** Spans of the part's blocks in range. */ + #plan(blocks: RunBlock[]) { + const spans: RunBlock[][] = [] let cur: RunBlock[] = [] let bytes = 0 - for (const b of blocksInRange(index, this.#range)) { + for (const b of this.#bounds ? overlapping(blocks, this.#bounds) : blocks) { const prev = cur[cur.length - 1] - const contiguous = prev && prev.part === b.part && prev.offset + prev.length === b.offset - if (prev && (!contiguous || bytes + b.length > spanBytes)) { - this.#spans.push(cur) + const contiguous = prev && prev.offset + prev.length === b.offset + if (prev && (!contiguous || bytes + b.length > this.spanBytes)) { + spans.push(cur) cur = [] bytes = 0 } cur.push(b) bytes += b.length } - if (cur.length > 0) this.#spans.push(cur) + if (cur.length > 0) spans.push(cur) + return spans } - #load(i: number): Promise { - const span = this.#spans[i]! + #load(span: RunBlock[]): Promise { const start = span[0]!.offset const length = span.reduce((n, b) => n + b.length, 0) const key = partKey(this.sessionId, this.index.seq, span[0]!.part) @@ -349,19 +509,33 @@ export class RunCursor { async next(): Promise { for (;;) { while (this.#line < this.#lines.length) { - const e = JSON.parse(this.#lines[this.#line++]!) as RunEntry - if (!this.#range || inRunRange(this.#range, e)) return (this.head = e) + const line = this.#lines[this.#line++]! + const e = JSON.parse(line) as RunEntry + if (!this.#bounds || inBounds(this.#bounds, runKey(e))) return (this.head = e) } + this.#lines = [] if (this.#blocks.length > 0) { this.#lines = splitLines(await gunzipText(this.#blocks.shift()!)) this.#line = 0 continue } - if (this.#span >= this.#spans.length) return (this.head = undefined) - const current = this.#fetch ?? this.#load(this.#span) - this.#span++ - this.#fetch = this.#span < this.#spans.length ? this.#load(this.#span) : null - this.#blocks = await current + if (this.#span < this.#spans.length) { + const current = this.#fetch ?? this.#load(this.#spans[this.#span]!) + this.#span++ + this.#fetch = this.#span < this.#spans.length ? this.#load(this.#spans[this.#span]!) : null + // The next part's page is read while the last span of this one is used. + if (!this.#fetch && this.#part < this.#parts.length) + this.#page ??= this.#loadPage(this.#part) + this.#blocks = await current + continue + } + if (this.#part >= this.#parts.length) return (this.head = undefined) + const page = this.#page ?? this.#loadPage(this.#part) + this.#page = null + this.#part++ + this.#spans = this.#plan(await page) + this.#span = 0 + this.#fetch = null } } } @@ -371,7 +545,7 @@ export async function* readRun( store: Store, sessionId: string, index: RunIndex, - range?: RunRange | string, + range?: RunRange | KeyRange | string, ): AsyncGenerator { const c = new RunCursor(store, sessionId, index, range) for (let e = await c.next(); e; e = await c.next()) yield e @@ -430,11 +604,11 @@ export async function* mergeRuns( store: Store, sessionId: string, indexes: RunIndex[], - range?: RunRange | string, + range?: RunRange | KeyRange | string, ): AsyncGenerator { const spanBytes = Math.max( BLOCK_BYTES, - Math.floor(PREFETCH_BUDGET / (2 * Math.max(1, indexes.length))), + Math.floor(runConfig.prefetchBudget / (2 * Math.max(1, indexes.length))), ) const heap = new CursorHeap() await Promise.all( @@ -457,16 +631,27 @@ export async function* mergeRuns( } } -/** Merge runs into one new run of the given tier (a compaction). */ +/** + * Merge runs into one new run of the given tier (a compaction), or, with + * `range`, one key-range piece of it: its parts numbered from `partBase`, no + * index written (compact.ts joins the pieces). + */ export async function compactRuns( store: Store, sessionId: string, indexes: RunIndex[], seq: number, tier: number, + piece?: { range: KeyRange; partBase: number }, ): Promise { - const w = new RunWriter(store, sessionId, seq, tier) - for await (const e of mergeRuns(store, sessionId, indexes)) { + const w = new RunWriter( + store, + sessionId, + seq, + tier, + piece ? { partBase: piece.partBase, writeIndex: false } : {}, + ) + for await (const e of mergeRuns(store, sessionId, indexes, piece?.range)) { w.add(e) await w.drain() } @@ -476,12 +661,10 @@ export async function compactRuns( /** The marks of runs, within a range, sorted and unique: split-key candidates. */ export function runMarks(indexes: RunIndex[], range?: RunRange): string[] { const out = new Set() + const b = range ? boundsOf(range) : undefined for (const ix of indexes) { - for (const b of blocksInRange(ix, range)) { - for (const m of b.marks ?? []) { - const sep = m.indexOf('\u0000') - if (!range || inRunRange(range, { t: m.slice(0, sep), k: m.slice(sep + 1) })) out.add(m) - } + for (const p of b ? overlapping(ix.parts, b) : ix.parts) { + for (const m of p.marks ?? []) if (!b || inBounds(b, m)) out.add(m) } } return [...out].sort(compareUtf8) diff --git a/packages/server/src/push/session.ts b/packages/server/src/push/session.ts index 60b7e3df..118ac9f1 100644 --- a/packages/server/src/push/session.ts +++ b/packages/server/src/push/session.ts @@ -11,10 +11,14 @@ import { and, asc, count, eq, gt, or, sql } from 'drizzle-orm' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' import { maybeCompact } from './compact.js' -import { readRunIndex, type RunIndex } from './runs.js' +import { readRunIndex, runBytes, runCount, type RunIndex } from './runs.js' -/** Idle timeout: pushed back by every upload. Runs live in storage, so this can be generous. */ -export const SESSION_TTL_MS = 60 * 60 * 1000 +/** + * Idle timeout: pushed back by every upload. Runs live in storage, so this can + * be generous: a client loading a corpus into one session over days can stop + * and pick the same session up again (GET …/push/:sid says where it is). + */ +export const SESSION_TTL_MS = 12 * 60 * 60 * 1000 /** Sessions one user may have open or committing at once (v2-scale-review.md S3). */ export const limits = { openSessions: 20 } @@ -110,7 +114,7 @@ export async function recordRun( index: RunIndex, counters: { records?: number }, ): Promise { - const count = index.blocks.reduce((n, b) => n + b.count, 0) + const count = runCount(index) await ports.db.batch([ ports.db.insert(schema.pushRuns).values({ sessionId, @@ -118,8 +122,9 @@ export async function recordRun( kind, objectKey: `sessions/${sessionId}/runs/${index.seq}`, count, - firstKey: index.blocks[0]?.first ?? '', - lastKey: index.blocks[index.blocks.length - 1]?.last ?? '', + bytes: runBytes(index), + firstKey: index.parts[0]?.first ?? '', + lastKey: index.parts[index.parts.length - 1]?.last ?? '', tier: index.tier ?? 0, }), ports.db @@ -129,7 +134,7 @@ export async function recordRun( }) .where(eq(schema.pushSessions.id, sessionId)), ]) - await maybeCompact(ports, sessionId, kind, index.tier ?? 0) + await maybeCompact(ports, sessionId, index.tier ?? 0) } export async function sessionRuns( diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index f5f45052..8851496e 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -32,6 +32,8 @@ export interface PlatformStorage { repoPrefix: string /** Key prefix for platform-internal objects. */ internalPrefix: string + /** The isolate's memory cache for repository objects (default 32 MB; the bulk Worker's is smaller). */ + memoryBytes?: number } /** Primary placements change rarely; remember them briefly per isolate. */ @@ -59,7 +61,7 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S cache, scope: `loc:${locationId}`, trusted: true, - lru: sharedLru(), + lru: sharedLru(platform.memoryBytes), }) repos.set(locationId, repo) return repo diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts index 00f1a55a..fa2851ec 100644 --- a/packages/server/src/versions/commit.ts +++ b/packages/server/src/versions/commit.ts @@ -82,14 +82,16 @@ export interface CommitInput { pushSessionId?: string | null /** * Trees already merged by a parallel commit (push/parallel.ts), per type, with - * the file reference deltas and change counts of the units that built them. - * The types' change sources are then ignored. Only for commits where no type - * changes privacy or schema and none is removed. + * the file reference deltas and change counts of the units that built them, + * and the types that moved their tree to the other set (protocol BuildInput). + * The types' change sources are then ignored, and the parallel commit has + * revalidated any changed schema itself. */ prebuilt?: { trees: Record refs: FileRefDelta stats: { added: number; removed: number; updated: number } + moves?: Record } /** Validate a record's data against its type's schema; errors or null. Used when a schema changes. */ validate?: (schema: Record, data: unknown) => string[] | null diff --git a/packages/server/src/versions/view.ts b/packages/server/src/versions/view.ts index 3ead44f9..000749d2 100644 --- a/packages/server/src/versions/view.ts +++ b/packages/server/src/versions/view.ts @@ -229,8 +229,57 @@ async function bodyOf(source: RepoSource, root: string, key: string let hash = root for (;;) { const node = await source.node(hash) - if (node.kind === 'leaf') - return (await source.leafEntries(hash)).find((e) => e.key === key)!.body! + if (node.kind === 'leaf') { + // Streamed, stopping at the record: a leaf of large records is never held whole. + for await (const e of source.leafStream(hash)) if (e.key === key) return e.body! + throw new Error(`Record ${key} is not in leaf ${hash}`) + } hash = node.children.find((c) => compareUtf8(key, c.lastKey) <= 0)!.hash } } + +/** + * Bodies of records by (type, id, set), each leaf read once and streamed: a page + * of records from one leaf decompresses it once, not once a record. Keys are + * `type\u0000id`; a record not in the view is left out. `concurrency` leaves at + * a time. + */ +export async function recordBodies( + view: VersionView, + wanted: { type: string; key: string; set: 'public' | 'private' }[], + concurrency = 4, +): Promise> { + const source = new RepoSource(recordTree, view.repo) + const byLeaf = new Map>() + for (const w of wanted) { + const t = view.types.find((x) => x.slug === w.type) + let hash = (w.set === 'public' ? t?.public : t?.private)?.root ?? null + while (hash) { + const node = await source.node(hash) + if (node.kind === 'leaf') break + hash = node.children.find((c) => compareUtf8(w.key, c.lastKey) <= 0)?.hash ?? null + } + if (!hash) continue + const keys = byLeaf.get(hash) ?? new Set() + keys.add(`${w.type}\u0000${w.key}`) + byLeaf.set(hash, keys) + } + const out = new Map() + const leaves = [...byLeaf] + let next = 0 + const worker = async () => { + while (next < leaves.length) { + const [hash, keys] = leaves[next++]! + const type = [...keys][0]!.split('\u0000')[0]! + let left = keys.size + for await (const e of source.leafStream(hash)) { + const k = `${type}\u0000${e.key}` + if (!keys.has(k) || view.withheld.has(e.hash)) continue + out.set(k, e.body!) + if (--left === 0) break + } + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, leaves.length) }, worker)) + return out +} diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index 4583f56e..309a509f 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -27,6 +27,8 @@ import type { JobMessage, Ports, PublicAssets } from './ports.js' import { createStores } from './stores.js' export interface Env { + /** 'bulk' in the bulk job Worker (wrangler.bulk.jsonc). */ + ROLE?: string DB: D1Database JOBS: Queue /** Bulk jobs (jobs.ts BULK_JOBS); without it, everything goes to JOBS. */ @@ -133,6 +135,8 @@ function makePorts(env: Env, ctx: ExecutionContext, req?: Request): Ports { files: s3, repoPrefix: env.REPO_PREFIX ?? 'repo', internalPrefix: env.INTERNAL_PREFIX ?? 'internal', + // The bulk Worker's isolates run jobs, whose reads rarely repeat. + ...(env.ROLE === 'bulk' ? { memoryBytes: 8 * 1024 * 1024 } : {}), }), cache, signer: () => (signer ??= ed25519Signer(env.SIGNING_KEY)), diff --git a/packages/server/test/memory.test.ts b/packages/server/test/memory.test.ts new file mode 100644 index 00000000..43e3980a --- /dev/null +++ b/packages/server/test/memory.test.ts @@ -0,0 +1,54 @@ +/** + * Every push job's memory stays bounded on large records: scripts/memory.ts + * pushes about 40 MB of records of several KB each (and updates spread over them) + * through uploads, compaction in pieces, a parallel commit and indexing, and + * reports each job's peak live memory. A job that held its records (rather than + * streaming them) would show tens of MB of JS heap here. + * + * Only the JS heap is checked: array buffers (compressed parts and spans) are + * bounded by the size constants, and Node releases them lazily after a GC, so + * their readings vary from run to run. scripts/memory.ts with the defaults gives + * the full picture at production sizes. + */ +import { execFile } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +import { expect, it } from 'vitest' + +const run = promisify(execFile) + +it('keeps every push job’s heap bounded on large records', async () => { + const cwd = fileURLToPath(new URL('..', import.meta.url)) + const { stdout } = await run( + process.execPath, + [ + '--expose-gc', + '--import', + 'tsx', + 'scripts/memory.ts', + '--records', + '4000', + '--update', + '1000', + '--piece-mb', + '8', + '--unit-mb', + '8', + '--sample', + '5', + ], + { cwd, maxBuffer: 16 * 1024 * 1024 }, + ) + const report = JSON.parse(stdout) as { + recordMB: number + jobs: Record + } + expect(report.recordMB).toBeGreaterThan(35) + for (const job of ['push.compactPiece', 'commit.unit', 'commit.assemble', 'refs.index']) + expect(report.jobs[job]?.n ?? 0, job).toBeGreaterThan(0) + for (const [job, r] of Object.entries(report.jobs)) { + // An upload holds its batch (8 MB) as text and records; nothing else holds records. + expect(r.heapMB, job).toBeLessThan(job.startsWith('upload') ? 32 : 16) + } +}, 180_000) diff --git a/packages/server/test/paging.test.ts b/packages/server/test/paging.test.ts index d1801ab8..01c11698 100644 --- a/packages/server/test/paging.test.ts +++ b/packages/server/test/paging.test.ts @@ -8,6 +8,7 @@ import { } from '@underlay/protocol' import { afterAll, describe, expect, it } from 'vitest' +import { diffLimits } from '../src/api/versions.js' import * as schema from '../src/db/schema.js' import { commitVersion, type TypeInput } from '../src/versions/commit.js' import { cleanup, harness } from './harness.js' @@ -89,6 +90,9 @@ describe('diff and manifest paging', () => { removed: p.removed.map((x) => x.id), }) expect(ids(one).added).toHaveLength(200) + // Items are the stored records passed through: exactly id, type and data. + expect(one.added[0]).toStrictEqual({ id: id(3000), type: 'Doc', data: { n: 0 } }) + expect(one.updated[0]).toStrictEqual({ id: id(0), type: 'Doc', data: { n: 1 } }) expect(ids(one).updated).toHaveLength(520) expect(ids(one).removed).toHaveLength(100) const paged = { added: [] as string[], updated: [] as string[], removed: [] as string[] } @@ -108,6 +112,29 @@ describe('diff and manifest paging', () => { expect(pages).toBe(Math.ceil(820 / 97)) expect(paged).toEqual(ids(one)) + // Pages are capped by their records' bytes too, past the first record. + const pageBytes = diffLimits.pageBytes + diffLimits.pageBytes = 2000 + try { + const small = { added: [] as string[], updated: [] as string[], removed: [] as string[] } + let pagesByBytes = 0 + do { + const p = (await get( + `${base}/diff?from=${v.semver}&limit=5000${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, + )) as DiffPage + const x = ids(p) + small.added.push(...x.added) + small.updated.push(...x.updated) + small.removed.push(...x.removed) + cursor = p.pagination.nextCursor + pagesByBytes++ + } while (cursor) + expect(pagesByBytes).toBeGreaterThan(10) + expect(small).toEqual(ids(one)) + } finally { + diffLimits.pageBytes = pageBytes + } + type Entry = { id: string hash: string diff --git a/packages/server/test/parallel.test.ts b/packages/server/test/parallel.test.ts index e433beb2..e3695e49 100644 --- a/packages/server/test/parallel.test.ts +++ b/packages/server/test/parallel.test.ts @@ -79,8 +79,8 @@ async function push( Object.assign( parallelConfig, parallel - ? { above: 0, unitEntries: 400, baseLevel: 0, ...(parallel === true ? {} : parallel) } - : { above: Infinity }, + ? { aboveBytes: 0, unitBytes: 64_000, baseLevel: 0, ...(parallel === true ? {} : parallel) } + : { aboveBytes: Infinity }, ) const commit = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) expect(commit.status).toBe(parallel ? 202 : 201) @@ -124,7 +124,7 @@ describe('parallel commit', () => { const path = '/api/collections/org/win' const m = markIds('w', 12) Object.assign(unitLimits, { perSession: 3, perAccount: 3 }) - Object.assign(parallelConfig, { above: 0, unitEntries: 1 }) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 1 }) const res = await h.request(`${path}/push`, { method: 'POST', user, @@ -146,13 +146,14 @@ describe('parallel commit', () => { (await h.ports.db.select().from(schema.jobs)).filter( (j) => j.type === 'commit.unit' && j.status !== 'failed', ).length - const planned = await h.ports.db.select().from(schema.commitUnits) - expect(planned.filter((u) => !u.gap).length).toBeGreaterThan(3) let most = 0 for (;;) { most = Math.max(most, await queuedUnits()) if ((await drainSqliteJobs(h.ports, { max: 1 })) === 0) break } + // Rows are written by the plan's chunk jobs, so they're counted after. + const planned = await h.ports.db.select().from(schema.commitUnits) + expect(planned.filter((u) => !u.gap).length).toBeGreaterThan(3) expect(most).toBeLessThanOrEqual(3) const status = (await (await h.request(`${path}/push/${sid}`, { user })).json()) as { status: string @@ -211,7 +212,7 @@ describe('parallel commit', () => { ] const deletes = [Array.from({ length: 40 }, (_, i) => id(10_500 + i * 13)), [m[3]!, m[6]!]] // One interval per unit, so every deleted candidate is some unit's `through`. - const pa = await push(h, user, pathA, {}, second, deletes, { unitEntries: 1 }) + const pa = await push(h, user, pathA, {}, second, deletes, { unitBytes: 1 }) const pb = await push(h, user, pathB, {}, second, deletes, false) expect(pa.result.changes).toEqual(pb.result.changes) @@ -258,7 +259,7 @@ describe('parallel commit', () => { user, ndjson: batch.map((k) => rec(k, k)), }) - Object.assign(parallelConfig, { above: 0, unitEntries: 1, baseLevel: 0 }) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 1, baseLevel: 0 }) expect((await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( 202, ) @@ -296,7 +297,7 @@ describe('parallel commit', () => { user, ndjson: [...batch.map((k) => rec(k, k)), rec('withpic', 'P', { pic: 'portrait' })], }) - Object.assign(parallelConfig, { above: 0, unitEntries: 1, baseLevel: 0 }) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 1, baseLevel: 0 }) const commit = async () => { const res = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) expect(res.status).toBe(202) @@ -332,29 +333,6 @@ describe('parallel commit', () => { .where(eq(schema.versions.collectionId, c.id)) expect(versions.map((v) => [v.recordCount, v.fileCount])).toEqual([[batch.length + 2, 1]]) }) - - it('stays serial when a type changes schema', async () => { - const h = await harness() - const user = await h.member() - await h.collection('pc') - const path = '/api/collections/org/pc' - const batch = [Array.from({ length: 50 }, (_, i) => rec(`x${i}`, `n${i}`))] - await push(h, user, path, { schemas: { Author } }, batch, [], false) - const Author2 = { - ...Author, - properties: { name: { type: 'string' }, born: { type: 'integer' } }, - } - Object.assign(parallelConfig, { above: 0 }) - const res = await h.request(`${path}/push`, { - method: 'POST', - user, - json: { schemas: { Author: Author2 } }, - }) - const sid = ((await res.json()) as { session_id: string }).session_id - await h.request(`${path}/push/${sid}/records`, { method: 'POST', user, ndjson: batch[0]! }) - const commit = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) - expect(commit.status).toBe(201) - }) }) // Keep the helper honest: marks are natural leaf boundaries. diff --git a/packages/server/test/push.test.ts b/packages/server/test/push.test.ts index 186abe8c..faaee834 100644 --- a/packages/server/test/push.test.ts +++ b/packages/server/test/push.test.ts @@ -128,9 +128,18 @@ describe('delta push', () => { try { // One record, the same schema: in the request. expect((await push({}, [rec('f', { name: 'f' })])).status).toBe(201) - // One record, but the schema changes for a type of six: a job. + // One record, and a schema change that only drops a required field: nothing + // to revalidate, so in the request. const looser = { ...Author, required: [] } - const res = await push({ schemas: { Author: looser } }, [rec('g', { name: 'g' })]) + expect((await push({ schemas: { Author: looser } }, [rec('g', { name: 'g' })])).status).toBe( + 201, + ) + // One record, but a schema change that can reject records, for a type of seven: a job. + const stricter = { + ...looser, + properties: { name: { type: 'string' }, born: { type: 'string' } }, + } + const res = await push({ schemas: { Author: stricter } }, [rec('h', { name: 'h' })]) expect(res.status).toBe(202) await h.drain() } finally { @@ -182,9 +191,9 @@ describe('delta push', () => { // The node advertises its limits (protocol section 11.4). expect(opened.limits).toEqual({ open_bytes: 8 * 1024 * 1024, - batch_bytes: 16 * 1024 * 1024, + batch_bytes: 8 * 1024 * 1024, batch_lines: 10_000, - session_idle_seconds: 3600, + session_idle_seconds: 43_200, open_sessions: 20, file_bytes: 32 * 1024 * 1024, }) diff --git a/packages/server/test/recover.test.ts b/packages/server/test/recover.test.ts new file mode 100644 index 00000000..27aca1bb --- /dev/null +++ b/packages/server/test/recover.test.ts @@ -0,0 +1,264 @@ +/** + * Commits that stop moving: the maintenance sweep queues their next step again, + * and fails one that can't finish with a reason, instead of leaving it + * committing forever (2026-10-15: a session sat at 20 of 34 units for 80 min). + */ +import { and, eq } from 'drizzle-orm' +import { afterAll, afterEach, describe, expect, it } from 'vitest' + +import { committing } from '../src/cleanup/sweep.js' +import * as schema from '../src/db/schema.js' +import { drainSqliteJobs } from '../src/jobs.js' +import { compactConfig, compactPiece } from '../src/push/compact.js' +import { parallelConfig, unitLimits } from '../src/push/parallel.js' +import { recoverCommits, recoverConfig } from '../src/push/recover.js' +import { isMark, MERGE_FAN_IN } from '../src/push/runs.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) +const defaults = { + parallel: { ...parallelConfig }, + units: { ...unitLimits }, + recover: { ...recoverConfig }, + compact: { ...compactConfig }, +} +afterEach(() => { + Object.assign(parallelConfig, defaults.parallel) + Object.assign(unitLimits, defaults.units) + Object.assign(recoverConfig, defaults.recover) + Object.assign(compactConfig, defaults.compact) +}) + +const Author = { type: 'object', properties: { name: { type: 'string' } }, required: ['name'] } +const rec = (id: string) => ({ id, type: 'Author', data: { name: id } }) + +function ids(n: number): string[] { + const out: string[] = [] + for (let i = 0; out.length < n; i++) { + const id = `a${String(i).padStart(7, '0')}` + if (isMark(id) || i % 50 === 0) out.push(id) + } + return out +} + +async function json(res: Response) { + return (await res.json()) as any +} + +/** Open a session and upload `batches`; its id. */ +async function upload(h: Harness, user: string, path: string, batches: object[][]) { + const sid = ( + await json( + await h.request(`${path}/push`, { method: 'POST', user, json: { schemas: { Author } } }), + ) + ).session_id as string + for (const b of batches) + expect( + (await h.request(`${path}/push/${sid}/records`, { method: 'POST', user, ndjson: b })).status, + ).toBe(200) + return sid +} + +const dropJobs = (h: Harness, type: string) => + h.ports.db.delete(schema.jobs).where(eq(schema.jobs.type, type)) + +/** Run every job but those of `type`, which are lost as they're queued. */ +async function drainLosing(h: Harness, type: string) { + for (;;) { + await dropJobs(h, type) + if ((await drainSqliteJobs(h.ports, { max: 1 })) === 0) return + } +} + +const session = async (h: Harness, sid: string) => + (await h.ports.db.select().from(schema.pushSessions).where(eq(schema.pushSessions.id, sid)))[0]! + +describe('recovering stalled commits', () => { + it('queues lost units again, and the commit finishes', async () => { + const h = await harness() + const user = await h.member() + await h.collection('lost') + const path = '/api/collections/org/lost' + const sid = await upload(h, user, path, [ids(400).map(rec)]) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 1, baseLevel: 0 }) + expect((await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user })).status).toBe( + 202, + ) + // Plan the units, losing every unit job. + await drainLosing(h, 'commit.unit') + expect((await session(h, sid)).status).toBe('committing') + + Object.assign(recoverConfig, { stallMs: -1 }) + Object.assign(unitLimits, { staleMs: -1 }) + await recoverCommits(h.ports) + await h.drain() + expect((await session(h, sid)).status).toBe('committed') + }) + + it('fails a commit whose unit used up its attempts, naming the unit', async () => { + const h = await harness() + const user = await h.member() + await h.collection('spent') + const path = '/api/collections/org/spent' + const sid = await upload(h, user, path, [ids(400).map(rec)]) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 1, baseLevel: 0 }) + await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + await drainLosing(h, 'commit.unit') + // One unit has been tried as often as it may be. + const [unit] = await h.ports.db + .select() + .from(schema.commitUnits) + .where(and(eq(schema.commitUnits.sessionId, sid), eq(schema.commitUnits.status, 'pending'))) + await h.ports.db + .update(schema.commitUnits) + .set({ attempts: unitLimits.maxAttempts, queuedAt: new Date(0) }) + .where(eq(schema.commitUnits.id, unit!.id)) + Object.assign(recoverConfig, { stallMs: -1 }) + await recoverCommits(h.ports) + const s = await session(h, sid) + expect(s.status).toBe('failed') + expect(s.error?.error).toMatch(/commit unit failed 4 times \(type Author/) + const status = await json(await h.request(`${path}/push/${sid}`, { user })) + expect(status.status).toBe('failed') + + // The failure was the server's: the same session commits again, from its uploads. + const again = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + expect(again.status).toBe(202) + await h.drain() + const retried = await json(await h.request(`${path}/push/${sid}`, { user })) + expect(retried).toMatchObject({ status: 'committed', error: null }) + expect(retried.result.recordCount).toBe(400) + }) + + it('refuses to commit again a session that failed for a reason the client must fix', async () => { + const h = await harness() + const user = await h.member() + await h.collection('refused') + const path = '/api/collections/org/refused' + const sid = await upload(h, user, path, [ids(5).map(rec)]) + await h.ports.db + .update(schema.pushSessions) + .set({ status: 'failed', error: { statusCode: 422, error: 'Schema validation failed' } }) + .where(eq(schema.pushSessions.id, sid)) + const res = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(409) + expect((await json(res)).error).toMatch(/new session/) + }) + + it('holds storage cleanup for a long commit while it makes progress', async () => { + const h = await harness() + const user = await h.member() + await h.collection('long') + const path = '/api/collections/org/long' + const sid = await upload(h, user, path, [ids(5).map(rec)]) + const hours = (n: number) => new Date(Date.now() - n * 60 * 60 * 1000) + const set = (v: Partial) => + h.ports.db.update(schema.pushSessions).set(v).where(eq(schema.pushSessions.id, sid)) + await set({ status: 'committing', finalizeStartedAt: hours(30), progressAt: hours(0.2) }) + expect(await committing(h.ports)).toBe(true) + await set({ progressAt: hours(7) }) + expect(await committing(h.ports)).toBe(false) + }) + + it('nudges a lost commit job, and fails one that never moves', async () => { + const h = await harness() + const user = await h.member() + await h.collection('nudge') + const path = '/api/collections/org/nudge' + const sid = await upload(h, user, path, [ids(20).map(rec)]) + const commit = await h.request(`${path}/push/${sid}/commit?async=true`, { + method: 'POST', + user, + }) + expect(commit.status).toBe(202) + await dropJobs(h, 'push.commit') + Object.assign(recoverConfig, { stallMs: -1 }) + // Nudged up to maxStalls times: each queues push.commit again; lose those too. + for (let i = 0; i < recoverConfig.maxStalls; i++) { + await recoverCommits(h.ports) + expect((await session(h, sid)).stalls).toBe(i + 1) + await dropJobs(h, 'push.commit') + } + await recoverCommits(h.ports) + const s = await session(h, sid) + expect(s.status).toBe('failed') + expect(s.error?.error).toMatch(/stopped making progress while committing/) + + // A second session: one nudge, and the job runs this time. + const sid2 = await upload(h, user, path, [ids(20).map(rec)]) + await h.request(`${path}/push/${sid2}/commit?async=true`, { method: 'POST', user }) + await dropJobs(h, 'push.commit') + await recoverCommits(h.ports) + await h.drain() + expect((await session(h, sid2)).status).toBe('committed') + }) + + it('queues a lost compaction again', async () => { + const h = await harness() + const user = await h.member() + await h.collection('compact') + const path = '/api/collections/org/compact' + const all = ids(MERGE_FAN_IN * 5) + const batches = Array.from({ length: MERGE_FAN_IN }, (_, i) => + all.slice(i * 5, i * 5 + 5).map(rec), + ) + const sid = await upload(h, user, path, batches) + // The 16th upload claimed a compaction; lose its job. + await dropJobs(h, 'push.compact') + const claimed = () => + h.ports.db.select().from(schema.pushRuns).where(eq(schema.pushRuns.sessionId, sid)) + expect((await claimed()).every((r) => r.mergingInto !== null)).toBe(true) + Object.assign(recoverConfig, { stallMs: -1 }) + await recoverCommits(h.ports) + await h.drain() + const runs = await claimed() + expect(runs.map((r) => [r.tier, r.mergingInto])).toEqual([[1, null]]) + // And the session still commits. + Object.assign(compactConfig, { commitRuns: 1 }) + const res = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(201) + }) + + it('leaves a long compaction alone while its pieces keep finishing', async () => { + const h = await harness() + const user = await h.member() + await h.collection('pieces') + const path = '/api/collections/org/pieces' + // Every compaction splits into pieces, one per input part. + Object.assign(compactConfig, { pieceBytes: 1 }) + const all = ids(MERGE_FAN_IN * 5) + const batches = Array.from({ length: MERGE_FAN_IN }, (_, i) => + all.slice(i * 5, i * 5 + 5).map(rec), + ) + const sid = await upload(h, user, path, batches) + const jobs = (type: string) => + h.ports.db.select().from(schema.jobs).where(eq(schema.jobs.type, type)) + // Run the compaction job alone: it plans the pieces and queues them. + while ((await jobs('push.compactPiece')).length === 0) + expect(await drainSqliteJobs(h.ports, { max: 1 })).toBe(1) + const runs = () => + h.ports.db.select().from(schema.pushRuns).where(eq(schema.pushRuns.sessionId, sid)) + const seq = (await runs())[0]!.mergingInto! + // The claim is older than the stall window, but a piece has just finished. + const old = new Date(Date.now() - 2 * recoverConfig.stallMs) + await h.ports.db + .update(schema.pushRuns) + .set({ claimedAt: old }) + .where(eq(schema.pushRuns.sessionId, sid)) + await compactPiece(h.ports, sid, seq, 0) + expect((await runs()).every((r) => r.claimedAt! > old)).toBe(true) + await dropJobs(h, 'push.compact') + await recoverCommits(h.ports) + expect(await jobs('push.compact')).toEqual([]) + // Once nothing has finished within the window, it is queued again, and completes. + await h.ports.db + .update(schema.pushRuns) + .set({ claimedAt: old }) + .where(eq(schema.pushRuns.sessionId, sid)) + await dropJobs(h, 'push.compactPiece') + await recoverCommits(h.ports) + expect((await jobs('push.compact')).length).toBe(1) + await h.drain() + expect((await runs()).map((r) => [r.tier, r.mergingInto])).toEqual([[1, null]]) + }) +}) diff --git a/packages/server/test/runs.test.ts b/packages/server/test/runs.test.ts index ed85dbee..dfcbffb3 100644 --- a/packages/server/test/runs.test.ts +++ b/packages/server/test/runs.test.ts @@ -8,6 +8,7 @@ import { isMark, mergeRuns, readRun, + runConfig, type RunEntry, type RunIndex, runMarks, @@ -48,9 +49,16 @@ describe('sorted runs', () => { k: `id${i}`, b: Array.from({ length: 16 }, (_, j) => sha256Hex(`${i}.${j}`)).join(''), }) - const ix = await writeRun(store, 's', 1, entries) - expect(ix.blocks.length).toBeGreaterThan(50) - expect(new Set(ix.blocks.map((b) => b.part)).size).toBeGreaterThan(1) + const partBytes = runConfig.partBytes + runConfig.partBytes = 1024 * 1024 + const ix = await writeRun(store, 's', 1, entries).finally(() => { + runConfig.partBytes = partBytes + }) + expect(ix.parts.length).toBeGreaterThan(3) + expect(ix.parts.reduce((n, p) => n + p.count, 0)).toBe(12_000) + // The summary carries no blocks; each part's are a page of their own. + expect(ix.parts.every((p) => p.blocks === undefined)).toBe(true) + expect(await store.get('sessions/s/runs/1/0.blocks.json')).not.toBeNull() const all = await collect(readRun(store, 's', ix)) expect(all.length).toBe(12_000) expect(all.slice(0, 6000).every((e) => e.t === 'A')).toBe(true) @@ -70,7 +78,7 @@ describe('sorted runs', () => { const ids = Array.from({ length: 40_000 }, (_, i) => `r${String(i).padStart(6, '0')}`) const entries = ids.map((k) => ({ t: 'T', k, h: sha256Hex(k) })) const ix = await writeRun(store, 's', 1, entries) - const total = ix.blocks.reduce((n, b) => n + b.length, 0) + const total = ix.parts.reduce((n, p) => n + p.length, 0) const range = { type: 'T', after: ids[20_000]!, through: ids[20_999]! } store.gets = 0 store.bytes = 0 diff --git a/packages/server/test/schema-parallel.test.ts b/packages/server/test/schema-parallel.test.ts new file mode 100644 index 00000000..5ecb36cb --- /dev/null +++ b/packages/server/test/schema-parallel.test.ts @@ -0,0 +1,316 @@ +/** + * Schema changes, privacy changes and type removals on the parallel path: each + * commits the same version, with the same counts and file sets, as the serial + * build, and a changed schema is revalidated in jobs. A change that can't reject + * a record (schemaRelaxes) isn't revalidated at all. + */ +import { createHash } from 'node:crypto' + +import { eq } from 'drizzle-orm' +import { afterAll, afterEach, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { parallelConfig } from '../src/push/parallel.js' +import { isMark } from '../src/push/runs.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) +const defaults = { ...parallelConfig } +afterEach(() => Object.assign(parallelConfig, defaults)) + +const sha = (s: string) => createHash('sha256').update(s).digest('hex') +const Doc = { + type: 'object', + properties: { name: { type: 'string' }, pic: { type: 'object' } }, + required: ['name'], +} +const Note = { type: 'object', properties: { text: { type: 'string' } } } + +/** Ids, some of them split-key candidates, so units split. */ +function ids(prefix: string, n: number): string[] { + const out: string[] = [] + for (let i = 0; out.length < n; i++) { + const id = `${prefix}${String(i).padStart(6, '0')}` + if (isMark(id) || i % 37 === 0) out.push(id) + } + return out +} +const doc = (id: string, name: string, extra: Record = {}) => { + const { pic, ...rest } = extra + return { + id, + type: 'Doc', + data: { name, ...(pic ? { pic: { $file: `sha256:${sha(String(pic))}` } } : {}) }, + ...rest, + } +} + +async function json(res: Response) { + return (await res.json()) as any +} + +type Mode = 'parallel' | 'serial' + +/** One push, committed in `mode`; the session's final status. */ +async function push( + h: Harness, + user: string, + path: string, + open: Record, + records: object[], + mode: Mode, + deletes: { type: string; id: string }[] = [], +) { + Object.assign( + parallelConfig, + mode === 'parallel' + ? { aboveBytes: 0, unitBytes: 4000, baseLevel: 0 } + : { aboveBytes: Infinity }, + ) + const sid = (await json(await h.request(`${path}/push`, { method: 'POST', user, json: open }))) + .session_id as string + if (records.length) + expect( + (await h.request(`${path}/push/${sid}/records`, { method: 'POST', user, ndjson: records })) + .status, + ).toBe(200) + if (deletes.length) + await h.request(`${path}/push/${sid}/deletes`, { method: 'POST', user, ndjson: deletes }) + await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + await h.drain() + const status = await json(await h.request(`${path}/push/${sid}`, { user })) + const [row] = await h.ports.db + .select() + .from(schema.pushSessions) + .where(eq(schema.pushSessions.id, sid)) + return { sid, status, parallel: row!.commitPlan !== null } +} + +/** The head version's sets (salt-free parts) and counts. */ +async function head(h: Harness, collectionId: string) { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, collectionId)) + const [v] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, c!.headVersionId!)) + const repo = await h.ports.stores.forCollection(collectionId) + const root = await repo.root(v!.hash) + const priv = root.private ? await repo.privateSet(root.private) : null + return { + public: root.public, + private: priv ? { types: priv.types, files: priv.files } : null, + version: { recordCount: v!.recordCount, fileCount: v!.fileCount, changes: v!.changes }, + } +} + +/** Keys the parallel commit wrote under a session's plans. */ +async function planKeys(h: Harness, sid: string) { + const out: string[] = [] + let cursor: string | undefined + do { + const page = await h.ports.stores.internal.list(`sessions/${sid}/commit/`, cursor) + out.push(...page.keys) + cursor = page.cursor + } while (cursor) + return out +} + +async function planOf(h: Harness, sid: string) { + const key = (await planKeys(h, sid)).find((k) => k.endsWith('/plan.json'))! + return JSON.parse(await (await h.ports.stores.internal.get(key))!.text()) as { + types: Record + } +} + +/** Two collections with the same base: `a` gets the parallel commit, `b` the serial one. */ +async function pair(name: string, base: { schemas: object; records: object[] }) { + const h = await harness() + const user = await h.member() + const a = await h.collection(`${name}-a`) + const b = await h.collection(`${name}-b`) + const paths = { a: `/api/collections/org/${name}-a`, b: `/api/collections/org/${name}-b` } + for (const path of [paths.a, paths.b]) { + for (const f of ['f1', 'f2']) + await h.request(`${path}/files/${sha(f)}`, { method: 'PUT', user, body: f }) + const r = await push(h, user, path, { schemas: base.schemas }, base.records, 'serial') + expect(r.status.status).toBe('committed') + } + return { h, user, a, b, paths } +} + +const baseDocs = () => + ids('d', 300).map((id, i) => doc(id, `name ${i}`, i % 50 === 0 ? { pic: 'f1' } : {})) + +describe('schema and privacy changes in a parallel commit', () => { + it('revalidates a changed schema in jobs, and builds what the serial commit builds', async () => { + const { h, user, a, b, paths } = await pair('sch', { + schemas: { Doc }, + records: baseDocs(), + }) + const Doc2 = { + ...Doc, + properties: { ...Doc.properties, name: { type: 'string', maxLength: 20 } }, + } + const changes = [ + ...ids('d', 300) + .slice(0, 40) + .map((id) => doc(id, 'renamed', { pic: 'f2' })), + ...ids('e', 30).map((id) => doc(id, 'new')), + ] + const deletes = ids('d', 300) + .slice(200, 205) + .map((id) => ({ type: 'Doc', id })) + const pa = await push( + h, + user, + paths.a, + { schemas: { Doc: Doc2 } }, + changes, + 'parallel', + deletes, + ) + const pb = await push(h, user, paths.b, { schemas: { Doc: Doc2 } }, changes, 'serial', deletes) + expect(pa.status.status).toBe('committed') + expect(pb.status.status).toBe('committed') + expect(pa.parallel).toBe(true) + expect((await planOf(h, pa.sid)).types.Doc!.validate).toBe(true) + expect((await planKeys(h, pa.sid)).some((k) => k.includes('/validate/'))).toBe(true) + expect(await head(h, a.id)).toEqual(await head(h, b.id)) + }, 30_000) + + it('fails a commit whose records don’t fit the new schema, as the serial commit does', async () => { + const { h, user, paths } = await pair('bad', { schemas: { Doc }, records: baseDocs() }) + const Doc2 = { + ...Doc, + properties: { ...Doc.properties, name: { type: 'string', maxLength: 6 } }, + } + const one = [doc('zz', 'ok')] + const pa = await push(h, user, paths.a, { schemas: { Doc: Doc2 } }, one, 'parallel') + const pb = await push(h, user, paths.b, { schemas: { Doc: Doc2 } }, one, 'serial') + for (const p of [pa, pb]) { + expect(p.status.status).toBe('failed') + expect(p.status.error.error).toBe('Schema validation failed') + } + expect(pa.status.error.totalErrors).toBe(pb.status.error.totalErrors) + expect(pa.status.error.totalErrors).toBeGreaterThan(200) + expect(pa.status.error.validationErrors.length).toBe(100) + }, 30_000) + + it('skips revalidation for a change that only relaxes the schema', async () => { + const { h, user, a, b, paths } = await pair('rel', { schemas: { Doc }, records: baseDocs() }) + const looser = { ...Doc, required: [], description: 'Docs' } + const changes = ids('d', 300) + .slice(0, 60) + .map((id) => doc(id, 'x')) + const pa = await push(h, user, paths.a, { schemas: { Doc: looser } }, changes, 'parallel') + const pb = await push(h, user, paths.b, { schemas: { Doc: looser } }, changes, 'serial') + expect(pa.status.status).toBe('committed') + expect(pa.parallel).toBe(true) + expect((await planOf(h, pa.sid)).types.Doc!.validate).toBeUndefined() + expect((await planKeys(h, pa.sid)).some((k) => k.includes('/validate'))).toBe(false) + expect(pb.status.status).toBe('committed') + expect(await head(h, a.id)).toEqual(await head(h, b.id)) + }, 30_000) + + it('makes a type private by moving its tree, as the serial commit does', async () => { + const { h, user, a, b, paths } = await pair('priv', { schemas: { Doc }, records: baseDocs() }) + const PrivateDoc = { ...Doc, private: true } + const changes = ids('d', 300) + .slice(100, 160) + .map((id) => doc(id, 'moved', { pic: 'f2' })) + const pa = await push(h, user, paths.a, { schemas: { Doc: PrivateDoc } }, changes, 'parallel') + const pb = await push(h, user, paths.b, { schemas: { Doc: PrivateDoc } }, changes, 'serial') + expect(pa.status.status).toBe('committed') + expect(pa.parallel).toBe(true) + expect((await planOf(h, pa.sid)).types.Doc!.move).toBe('toPrivate') + expect(pb.status.status).toBe('committed') + const ha = await head(h, a.id) + expect(ha).toEqual(await head(h, b.id)) + expect(ha.public.types.Doc).toBeUndefined() + expect(ha.private?.types.Doc?.count).toBe(300) + }, 30_000) + + it('makes a private type public by moving its tree, as the serial commit does', async () => { + const PrivateDoc = { ...Doc, private: true } + const { h, user, a, b, paths } = await pair('pub', { + schemas: { Doc: PrivateDoc }, + records: baseDocs(), + }) + const changes = ids('e', 40).map((id) => doc(id, 'public', { pic: 'f2' })) + const pa = await push(h, user, paths.a, { schemas: { Doc } }, changes, 'parallel') + const pb = await push(h, user, paths.b, { schemas: { Doc } }, changes, 'serial') + expect(pa.status.status).toBe('committed') + expect(pa.parallel).toBe(true) + expect((await planOf(h, pa.sid)).types.Doc!.move).toBe('toPublic') + expect(pb.status.status).toBe('committed') + expect(pa.status.result.changes).toEqual(pb.status.result.changes) + const ha = await head(h, a.id) + expect(ha).toEqual(await head(h, b.id)) + expect(ha.public.types.Doc?.count).toBe(340) + }, 30_000) + + it('keeps a type turning public with private records in the push serial', async () => { + const PrivateDoc = { ...Doc, private: true } + const { h, user, paths } = await pair('mixed', { + schemas: { Doc: PrivateDoc }, + records: baseDocs().slice(0, 40), + }) + const changes = [doc('k1', 'secret', { private: true }), doc('k2', 'open')] + const pa = await push(h, user, paths.a, { schemas: { Doc } }, changes, 'parallel') + expect(pa.status.status).toBe('committed') + expect(pa.parallel).toBe(false) + }) + + it('removes a type, as the serial commit does', async () => { + const records = [ + ...baseDocs(), + ...ids('n', 50).map((id) => ({ id, type: 'Note', data: { text: id } })), + ] + const { h, user, a, b, paths } = await pair('rm', { schemas: { Doc, Note }, records }) + const changes = ids('d', 300) + .slice(0, 30) + .map((id) => doc(id, 'kept')) + const pa = await push(h, user, paths.a, { schemas: { Doc } }, changes, 'parallel') + const pb = await push(h, user, paths.b, { schemas: { Doc } }, changes, 'serial') + expect(pa.status.status).toBe('committed') + expect(pa.parallel).toBe(true) + expect(pb.status.status).toBe('committed') + const ha = await head(h, a.id) + expect(ha).toEqual(await head(h, b.id)) + expect(ha.public.types.Note).toBeUndefined() + }, 30_000) + + it('counts revalidation as work: a small push that revalidates a large type goes parallel', async () => { + const { h, user, paths } = await pair('small', { schemas: { Doc }, records: baseDocs() }) + const Doc2 = { + ...Doc, + properties: { ...Doc.properties, name: { type: 'string', maxLength: 50 } }, + } + // Above the one record's bytes, below the type's. + Object.assign(parallelConfig, { aboveBytes: 2000 }) + const sid = ( + await json( + await h.request(`${paths.a}/push`, { + method: 'POST', + user, + json: { schemas: { Doc: Doc2 } }, + }), + ) + ).session_id as string + await h.request(`${paths.a}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: [doc('q', 'q')], + }) + Object.assign(parallelConfig, { unitBytes: 4000, baseLevel: 0 }) + const res = await h.request(`${paths.a}/push/${sid}/commit`, { method: 'POST', user }) + expect(res.status).toBe(202) + await h.drain() + const status = await json(await h.request(`${paths.a}/push/${sid}`, { user })) + expect(status.status).toBe('committed') + expect((await planOf(h, sid)).types.Doc!.validate).toBe(true) + }) +}) diff --git a/packages/server/test/stream-push.test.ts b/packages/server/test/stream-push.test.ts new file mode 100644 index 00000000..e1565e63 --- /dev/null +++ b/packages/server/test/stream-push.test.ts @@ -0,0 +1,235 @@ +/** + * One version from any number of uploads: paged run indexes, compaction split + * into key-range pieces, compaction down to a bounded run count before a + * commit, plans written in chunks, and the serial commit's size refusal. + */ +import { compareUtf8, memoryStore, sha256Hex } from '@underlay/protocol' +import { eq } from 'drizzle-orm' +import { afterAll, afterEach, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { compactConfig, pieceCuts } from '../src/push/compact.js' +import { parallelConfig } from '../src/push/parallel.js' +import { + compactRuns, + isMark, + mergeRuns, + normalizeIndex, + readRun, + readRunIndex, + runConfig, + type RunEntry, + type RunIndex, + writeRun, +} from '../src/push/runs.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) +const defaults = { + parallel: { ...parallelConfig }, + compact: { ...compactConfig }, + runs: { ...runConfig }, +} +afterEach(() => { + Object.assign(parallelConfig, defaults.parallel) + Object.assign(compactConfig, defaults.compact) + Object.assign(runConfig, defaults.runs) +}) + +async function collect(it: AsyncIterable): Promise { + const out: T[] = [] + for await (const x of it) out.push(x) + return out +} + +const Author = { type: 'object', properties: { name: { type: 'string' } }, required: ['name'] } +const rec = (id: string, name: string) => ({ id, type: 'Author', data: { name } }) +const body = (i: number) => Array.from({ length: 8 }, (_, j) => sha256Hex(`${i}.${j}`)).join('') + +describe('run pages', () => { + it('reads a range across many parts, each part’s blocks from its own page', async () => { + runConfig.partBytes = 64 * 1024 + const store = memoryStore() + const entries: RunEntry[] = Array.from({ length: 6000 }, (_, i) => ({ + t: 'T', + k: `r${String(i).padStart(5, '0')}`, + b: body(i), + })) + const ix = await writeRun(store, 's', 1, entries) + expect(ix.parts.length).toBeGreaterThan(10) + const range = { type: 'T', after: 'r01234', through: 'r04321' } + const got = await collect(readRun(store, 's', ix, range)) + expect(got.map((e) => e.k)).toEqual(entries.slice(1235, 4322).map((e) => e.k)) + // The index read back is the summary the writer returned. + expect(await readRunIndex(store, 's', 1)).toEqual(ix) + }) + + it('reads an index written before pages (blocks inline)', async () => { + const store = memoryStore() + const entries: RunEntry[] = Array.from({ length: 3000 }, (_, i) => ({ + t: 'T', + k: `k${String(i).padStart(4, '0')}`, + b: body(i), + })) + const ix = await writeRun(store, 's', 1, entries) + // Rebuild the old form: every block, with its part, in one list. + const blocks = [] + for (const p of ix.parts) { + const page = await store.get(`sessions/s/runs/1/${p.part}.blocks.json`) + blocks.push(...(JSON.parse(await page!.text()) as object[])) + } + await store.put('sessions/s/runs/1/index.json', JSON.stringify({ seq: 1, tier: 0, blocks })) + const old = await readRunIndex(store, 's', 1) + expect(old.parts.every((p) => p.blocks && p.blocks.length > 0)).toBe(true) + expect(old.parts.reduce((n, p) => n + p.count, 0)).toBe(3000) + const got = await collect( + readRun(store, 's', old, { type: 'T', after: 'k0100', through: 'k0200' }), + ) + expect(got.length).toBe(100) + // Slices of old plans normalize the same way. + expect(normalizeIndex({ seq: 1, tier: 0, blocks: [] as never[] }).parts).toEqual([]) + }) + + it('a compaction split into key-range pieces is the same run as one done whole', async () => { + runConfig.partBytes = 32 * 1024 + const store = memoryStore() + const runs: RunIndex[] = [] + for (let seq = 1; seq <= 6; seq++) { + const entries = Array.from({ length: 700 }, (_, i) => { + const n = (seq * 7919 + i * 104_729) % 3000 + return { t: n % 3 ? 'A' : 'B', k: `k${n}`, b: body(n * 10 + seq), s: seq } + }) + runs.push(await writeRun(store, 's', seq, entries)) + } + const whole = await collect(readRun(store, 's', await compactRuns(store, 's', runs, 10, 1))) + const cuts = pieceCuts(runs, 64 * 1024) + expect(cuts.length).toBeGreaterThan(3) + expect([...cuts].sort(compareUtf8)).toEqual(cuts) + const parts = [] + for (let piece = 0; piece <= cuts.length; piece++) { + const range = { after: piece ? cuts[piece - 1]! : null, through: cuts[piece] ?? null } + const out = await compactRuns(store, 's', runs, 11, 1, { range, partBase: piece * 1000 }) + parts.push(...out.parts) + } + const joined: RunIndex = { seq: 11, tier: 1, parts } + expect(await collect(readRun(store, 's', joined))).toEqual(whole) + expect(await collect(mergeRuns(store, 's', [joined]))).toEqual( + await collect(mergeRuns(store, 's', runs)), + ) + }) +}) + +async function json(res: Response) { + return (await res.json()) as any +} + +/** Open a session, upload `batches`, commit, drain the jobs; the session's final status. */ +async function push(h: Harness, user: string, base: string, batches: object[][], open = {}) { + const sid = ( + await json( + await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Author }, ...open }, + }), + ) + ).session_id as string + for (const b of batches) { + const r = await h.request(`${base}/push/${sid}/records`, { method: 'POST', user, ndjson: b }) + expect(r.status).toBe(200) + } + const commit = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect([201, 202, 422]).toContain(commit.status) + await h.drain() + return { sid, status: await json(await h.request(`${base}/push/${sid}`, { user })) } +} + +async function headTree(h: Harness, collectionId: string) { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, collectionId)) + const [v] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, c!.headVersionId!)) + const repo = await h.ports.stores.forCollection(collectionId) + return { count: v!.recordCount, root: (await repo.root(v!.hash)).public.types.Author } +} + +describe('one version from many uploads', () => { + it('compacts down to the commit’s run limit, in pieces, then commits in planned chunks', async () => { + const h = await harness() + const user = await h.member() + const a = await h.collection('many') + const b = await h.collection('one') + // 60 uploads, interleaved keys: every upload spans the whole key range. + // Ids are split-key candidates (marks) every so often, so units can split. + const ids: string[] = [] + for (let i = 0; ids.length < 2400; i++) { + const id = `r${String(i).padStart(7, '0')}` + if (isMark(id) || i % 97 === 0) ids.push(id) + } + const batches = Array.from({ length: 60 }, (_, j) => + Array.from({ length: 40 }, (_, i) => rec(ids[i * 60 + j]!, body(i))), + ) + Object.assign(compactConfig, { commitRuns: 4, pieceBytes: 40_000 }) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 50_000, baseLevel: 0, chunkUnits: 2 }) + runConfig.partBytes = 16 * 1024 + const many = await push(h, user, '/api/collections/org/many', batches) + expect(many.status).toMatchObject({ status: 'committed' }) + // Compaction ran in pieces and left at most four runs for the commit. + const runs = await h.ports.db + .select() + .from(schema.pushRuns) + .where(eq(schema.pushRuns.sessionId, many.sid)) + expect(runs.length).toBeLessThanOrEqual(4) + const pieces = await h.ports.stores.internal.list(`sessions/${many.sid}/runs/`) + expect(pieces.keys.some((k) => k.includes('/piece-'))).toBe(true) + // The plan was written in several chunks. + const units = await h.ports.db + .select() + .from(schema.commitUnits) + .where(eq(schema.commitUnits.sessionId, many.sid)) + expect(units.filter((u) => !u.gap).length).toBeGreaterThan(4) + + // The same records in one upload, committed serially: the same tree. + Object.assign(compactConfig, defaults.compact) + Object.assign(parallelConfig, { aboveBytes: Infinity }) + const one = await push(h, user, '/api/collections/org/one', [batches.flat()]) + expect(one.status).toMatchObject({ status: 'committed' }) + expect(await headTree(h, a.id)).toEqual(await headTree(h, b.id)) + expect((await headTree(h, a.id)).count).toBe(2400) + }, 60_000) + + it('says where a session is: its runs, then the commit’s phase and units', async () => { + const h = await harness() + const user = await h.member() + await h.collection('st') + const path = '/api/collections/org/st' + const sid = ( + await json( + await h.request(`${path}/push`, { method: 'POST', user, json: { schemas: { Author } } }), + ) + ).session_id as string + for (let j = 0; j < 6; j++) + await h.request(`${path}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: Array.from({ length: 10 }, (_, i) => rec(`k${j}-${i}`, body(i))), + }) + const status = async () => json(await h.request(`${path}/push/${sid}`, { user })) + expect((await status()).progress).toMatchObject({ runs: 6, phase: null, units: null }) + expect((await status()).progress.run_bytes).toBeGreaterThan(6 * 10 * 512) + Object.assign(compactConfig, { commitRuns: 2 }) + const res = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + // Too many runs for one commit: it compacts first, and answers 202. + expect(res.status).toBe(202) + expect((await status()).progress).toMatchObject({ phase: 'compacting' }) + expect((await status()).progress.runs_compacting).toBeGreaterThan(0) + await h.drain() + const done = await status() + expect(done.status).toBe('committed') + expect(done.progress).toBeUndefined() + }) +}) diff --git a/packages/server/wrangler.bulk.jsonc b/packages/server/wrangler.bulk.jsonc new file mode 100644 index 00000000..63caa0b7 --- /dev/null +++ b/packages/server/wrangler.bulk.jsonc @@ -0,0 +1,117 @@ +// The bulk job Worker: the same code as wrangler.jsonc's Worker, deployed as a +// second Worker that consumes only the bulk queue (jobs.ts BULK_JOBS: commits, +// compactions, indexing, mirrors, cleanup). It serves no routes. +// +// Why a second Worker: Workers memory is 128 MB per isolate, and an isolate +// runs several invocations at once. Bulk jobs are sized to share an isolate +// with each other; they aren't sized to share one with 16 MB upload batches and +// page renders, which is what happened while they ran in the main Worker. Its +// isolates also keep a smaller memory cache (ROLE=bulk, stores.ts), since job +// reads rarely repeat. +// +// wrangler deploy --env staging (main Worker first: it gives up the bulk queue) +// wrangler deploy -c wrangler.bulk.jsonc --env staging → underlay-staging-bulk +// +// A queue has one consumer, so the main Worker's deploy, which no longer lists +// the bulk queue, has to come first; bulk jobs wait on the queue until this +// Worker is up. Secrets (wrangler secret put -c wrangler.bulk.jsonc --env ), +// the same values as the main Worker's: +// R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, SIGNING_KEY, LOCATION_KEY +// +// Keep bindings and vars in step with wrangler.jsonc. +{ + "$schema": "../../node_modules/wrangler/config-schema.json", + "name": "underlay-bulk", + "main": "src/worker.ts", + "compatibility_date": "2026-09-30", + "compatibility_flags": ["nodejs_compat"], + "limits": { "cpu_ms": 300000, "subrequests": 100000 }, + "workers_dev": false, + "observability": { "enabled": true }, + "env": { + "staging": { + "name": "underlay-staging-bulk", + "vars": { + "ROLE": "bulk", + "APP_URL": "https://staging.underlay.org", + "DEPLOYMENT": "staging", + "R2_ENDPOINT": "https://b66a542ad5e0af992703a2ce8d1d8747.r2.cloudflarestorage.com", + "R2_BUCKET": "underlay-staging", + "OIDC_ISSUER_URL": "https://staging-auth.knowledgefutures.org", + "OIDC_ACCOUNT_URL": "https://staging-auth.knowledgefutures.org", + "OIDC_CLIENT_ID": "kf_underlay", + "ASSETS_BASE_URL": "https://assets-staging.underlay.org", + }, + "r2_buckets": [ + { "binding": "BUCKET", "bucket_name": "underlay-staging" }, + { "binding": "PUBLIC_ASSETS", "bucket_name": "underlay-staging-public" }, + ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "underlay-staging", + "database_id": "884ed8bb-14e8-4ea4-bdc4-81d433e5b551", + "migrations_dir": "drizzle", + }, + ], + "queues": { + "producers": [ + { "binding": "JOBS", "queue": "underlay-staging-jobs" }, + { "binding": "JOBS_BULK", "queue": "underlay-staging-bulk" }, + { "binding": "USAGE", "queue": "underlay-staging-usage" }, + ], + "consumers": [ + { + "queue": "underlay-staging-bulk", + "max_batch_size": 1, + "max_concurrency": 20, + "max_retries": 10, + "dead_letter_queue": "underlay-staging-dead", + }, + ], + }, + }, + "prod": { + "name": "underlay-prod-bulk", + "vars": { + "ROLE": "bulk", + "APP_URL": "https://www.underlay.org", + "DEPLOYMENT": "prod", + "R2_ENDPOINT": "https://b66a542ad5e0af992703a2ce8d1d8747.r2.cloudflarestorage.com", + "R2_BUCKET": "underlay-prod", + "OIDC_ISSUER_URL": "https://auth.knowledgefutures.org", + "OIDC_ACCOUNT_URL": "https://account.knowledgefutures.org", + "OIDC_CLIENT_ID": "kf_underlay", + "ASSETS_BASE_URL": "https://assets.underlay.org", + }, + "r2_buckets": [ + { "binding": "BUCKET", "bucket_name": "underlay-prod" }, + { "binding": "PUBLIC_ASSETS", "bucket_name": "underlay-prod-public" }, + ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "underlay-prod", + "database_id": "12e03c06-7ba6-42af-baf5-7ce5a0be0ee8", + "migrations_dir": "drizzle", + }, + ], + "queues": { + "producers": [ + { "binding": "JOBS", "queue": "underlay-prod-jobs" }, + { "binding": "JOBS_BULK", "queue": "underlay-prod-bulk" }, + { "binding": "USAGE", "queue": "underlay-prod-usage" }, + ], + "consumers": [ + { + "queue": "underlay-prod-bulk", + "max_batch_size": 1, + "max_concurrency": 20, + "max_retries": 10, + "dead_letter_queue": "underlay-prod-dead", + }, + ], + }, + }, + }, +} diff --git a/packages/server/wrangler.jsonc b/packages/server/wrangler.jsonc index e8ecb7da..fed74fa5 100644 --- a/packages/server/wrangler.jsonc +++ b/packages/server/wrangler.jsonc @@ -6,6 +6,9 @@ // wrangler deploy --env staging → staging.underlay.org // wrangler deploy --env prod → www.underlay.org // +// Bulk jobs run in a second Worker from the same code, which consumes the bulk +// queue: deploy it after this one (wrangler.bulk.jsonc says how, and why). +// // Secrets (wrangler secret put --env ): // R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY bucket-scoped R2 S3 credentials // SIGNING_KEY Ed25519 seed for version logs (one per deployment) @@ -80,7 +83,7 @@ { "name": "RL_USER", "namespace_id": "1002", "simple": { "limit": 5000, "period": 60 } }, { "name": "RL_PAGE", "namespace_id": "1003", "simple": { "limit": 600, "period": 60 } }, ], - // Interactive jobs, and bulk ones (one per invocation): jobs.ts BULK_JOBS. + // Interactive jobs here; bulk ones (jobs.ts BULK_JOBS) in the bulk Worker. "queues": { "producers": [ { "binding": "JOBS", "queue": "underlay-staging-jobs" }, @@ -95,13 +98,7 @@ "max_retries": 10, "dead_letter_queue": "underlay-staging-dead", }, - { - "queue": "underlay-staging-bulk", - "max_batch_size": 1, - "max_concurrency": 20, - "max_retries": 10, - "dead_letter_queue": "underlay-staging-dead", - }, + // The bulk queue's consumer is the bulk Worker (wrangler.bulk.jsonc). // Usage events (billing/usage.ts): up to 100 requests' events per log object. { "queue": "underlay-staging-usage", @@ -151,7 +148,7 @@ { "name": "RL_USER", "namespace_id": "2002", "simple": { "limit": 5000, "period": 60 } }, { "name": "RL_PAGE", "namespace_id": "2003", "simple": { "limit": 600, "period": 60 } }, ], - // Interactive jobs, and bulk ones (one per invocation): jobs.ts BULK_JOBS. + // Interactive jobs here; bulk ones (jobs.ts BULK_JOBS) in the bulk Worker. "queues": { "producers": [ { "binding": "JOBS", "queue": "underlay-prod-jobs" }, @@ -166,13 +163,7 @@ "max_retries": 10, "dead_letter_queue": "underlay-prod-dead", }, - { - "queue": "underlay-prod-bulk", - "max_batch_size": 1, - "max_concurrency": 20, - "max_retries": 10, - "dead_letter_queue": "underlay-prod-dead", - }, + // The bulk queue's consumer is the bulk Worker (wrangler.bulk.jsonc). // Usage events (billing/usage.ts): up to 100 requests' events per log object. { "queue": "underlay-prod-usage",