From e1a2e749a972766971462974325d25bdb2ce51ac Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 09:24:41 +0100 Subject: [PATCH 01/42] feat(control): add a lock file reclaimed only when its holder is provably gone The lock records the holder's process, start time, boot, PID namespace and container. Another writer reclaims it when the machine booted since, when the holder shared its PID namespace and its process ended, or when the holder's container is no longer running. Anything else is refused with the holder named and the operator exit to clear it. --- .../__tests__/recoverable-lock.test.ts | 139 +++++++++ .../core/src/services/recoverable-lock.ts | 283 ++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 packages/core/src/services/__tests__/recoverable-lock.test.ts create mode 100644 packages/core/src/services/recoverable-lock.ts diff --git a/packages/core/src/services/__tests__/recoverable-lock.test.ts b/packages/core/src/services/__tests__/recoverable-lock.test.ts new file mode 100644 index 000000000..556fbd1e4 --- /dev/null +++ b/packages/core/src/services/__tests__/recoverable-lock.test.ts @@ -0,0 +1,139 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + acquireRecoverableLock, + clearRecoverableLock, + currentLockHolder, + LockBusyError, + type LockHolder, + lockHolderGone, + readLockHolder, +} from "../recoverable-lock"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function lockPath(): string { + const root = mkdtempSync(join(tmpdir(), "omx-lock-")); + roots.push(root); + return join(root, "store.lock"); +} +/** A pid that is certainly not running: a child that has already exited. */ +function exitedPid(): number { + const child = spawnSync(process.execPath, ["-e", ""]); + if (!child.pid) throw new Error("no pid"); + return child.pid; +} +const foreign = (overrides: Partial): LockHolder => ({ + ...currentLockHolder("other writer"), + ...overrides, +}); +const neverAsked = async () => { + throw new Error("Docker must not be asked"); +}; + +describe("recoverable lock", () => { + it("records its holder and removes the file on release", async () => { + const path = lockPath(); + const release = await acquireRecoverableLock(path, { purpose: "test writer" }); + expect(readLockHolder(path)).toMatchObject({ purpose: "test writer", pid: process.pid }); + release(); + expect(existsSync(path)).toBe(false); + }); + + it("refuses a live holder after the wait and names it", async () => { + const path = lockPath(); + const release = await acquireRecoverableLock(path, { purpose: "first writer" }); + const error = await acquireRecoverableLock(path, { purpose: "second", waitMs: 50 }).catch( + (caught: unknown) => caught, + ); + expect(error).toBeInstanceOf(LockBusyError); + expect((error as LockBusyError).holder?.purpose).toBe("first writer"); + expect((error as Error).message).toContain("openmapx control clear --lock store.lock"); + release(); + }); + + it("waits for a holder that releases within the wait", async () => { + const path = lockPath(); + const release = await acquireRecoverableLock(path, { purpose: "first writer" }); + setTimeout(release, 50); + const second = await acquireRecoverableLock(path, { purpose: "second", waitMs: 2_000 }); + expect(readLockHolder(path)?.purpose).toBe("second"); + second(); + }); + + it("reclaims a lock whose holder in this PID namespace has exited", async () => { + const path = lockPath(); + writeFileSync(path, JSON.stringify(foreign({ pid: exitedPid() }))); + const release = await acquireRecoverableLock(path, { + purpose: "next writer", + waitMs: 50, + containerRunning: neverAsked, + }); + expect(readLockHolder(path)?.purpose).toBe("next writer"); + release(); + expect(readdirSync(join(path, ".."))).toEqual([]); + }); + + it("reclaims a lock left before the last boot", async () => { + const self = currentLockHolder("self"); + const stale = foreign({ bootId: "before-reboot", pidNamespace: "elsewhere", pid: 1 }); + expect(await lockHolderGone(stale, { ...self, bootId: "after-reboot" }, neverAsked)).toBe(true); + }); + + it("asks Docker only for a holder in another namespace and container", async () => { + const self = { ...currentLockHolder("self"), bootId: null, containerId: null }; + const inContainer = foreign({ + bootId: null, + pidNamespace: "pid:[1]", + pid: 1, + containerId: "a".repeat(64), + }); + const asked: string[] = []; + expect( + await lockHolderGone(inContainer, self, async (id) => { + asked.push(id); + return false; + }), + ).toBe(true); + expect(await lockHolderGone(inContainer, self, async () => true)).toBe(false); + // Docker could not answer: the holder is not provably gone. + expect(await lockHolderGone(inContainer, self, async () => undefined)).toBe(false); + expect(asked).toEqual(["a".repeat(64)]); + }); + + it("never reclaims a holder that cannot be proved gone", async () => { + const path = lockPath(); + writeFileSync( + path, + JSON.stringify(foreign({ pidNamespace: "pid:[1]", pid: 1, containerId: null })), + ); + await expect( + acquireRecoverableLock(path, { purpose: "next", waitMs: 50, containerRunning: neverAsked }), + ).rejects.toBeInstanceOf(LockBusyError); + expect(readLockHolder(path)?.purpose).toBe("other writer"); + }); + + it("treats a live process with another start time as a reused pid", async () => { + const self = currentLockHolder("self"); + if (!self.startTime) return; // Start times are read from /proc, on Linux only. + const reused = foreign({ pid: process.pid, startTime: "1" }); + expect(await lockHolderGone(reused, self, neverAsked)).toBe(true); + expect(await lockHolderGone(foreign({ pid: process.pid }), self, neverAsked)).toBe(false); + }); + + it("lets the operator clear a lock and refuses a release after that", async () => { + const path = lockPath(); + const release = await acquireRecoverableLock(path, { purpose: "stuck writer" }); + expect(clearRecoverableLock(path)?.purpose).toBe("stuck writer"); + expect(clearRecoverableLock(path)).toBeUndefined(); + const next = await acquireRecoverableLock(path, { purpose: "next" }); + expect(() => release()).toThrow(/taken from this holder/); + expect(readLockHolder(path)?.purpose).toBe("next"); + next(); + }); +}); diff --git a/packages/core/src/services/recoverable-lock.ts b/packages/core/src/services/recoverable-lock.ts new file mode 100644 index 000000000..597cde8a7 --- /dev/null +++ b/packages/core/src/services/recoverable-lock.ts @@ -0,0 +1,283 @@ +import { execFile } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + fsyncSync, + linkSync, + lstatSync, + openSync, + readFileSync, + readlinkSync, + readSync, + renameSync, + unlinkSync, + writeSync, +} from "node:fs"; +import { hostname } from "node:os"; +import { basename, dirname } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { z } from "zod"; +import { fsyncDirectory } from "../release-files"; + +/** + * A lock file that records who holds it, so that a holder that died can be told + * apart from one that is still working. It is held only around short file + * updates, never across a Docker effect. + * + * Holders run in different PID namespaces: the agent and its fixed CLI in the + * agent's container, the configuration helper in its own container and the + * host CLI on the host. A lock is reclaimed only when its holder is provably + * gone: the machine booted since, or the holder shares this PID namespace and + * its process has ended, or the container it ran in is no longer running. + * Anything else is refused with the holder named, for `openmapx control clear`. + */ +const holderSchema = z.strictObject({ + version: z.literal(1), + purpose: z.string().min(1).max(128), + pid: z.number().int().positive().safe(), + startTime: z.string().max(32).nullable(), + bootId: z.string().max(64).nullable(), + pidNamespace: z.string().max(128).nullable(), + containerId: z + .string() + .regex(/^[a-f0-9]{64}$/) + .nullable(), + acquiredAt: z.number().int().nonnegative().safe(), +}); +export type LockHolder = z.infer; + +export class LockBusyError extends Error { + constructor( + readonly path: string, + readonly holder: LockHolder | undefined, + ) { + super( + holder + ? `${basename(path)} is held by ${holder.purpose} (pid ${holder.pid}${holder.containerId ? ` in container ${holder.containerId.slice(0, 12)}` : ""}, since ${new Date(holder.acquiredAt).toISOString()}). If that process is gone, run \`openmapx control status\` and \`openmapx control clear --lock ${basename(path)}\`.` + : `${basename(path)} is busy`, + ); + } +} + +function readOptional(path: string): string | null { + try { + return readFileSync(path, "utf8").trim() || null; + } catch { + return null; + } +} + +/** Field 22 of /proc//stat, after the parenthesised command name. */ +function processStartTime(pid: number | "self"): string | null { + const stat = readOptional(`/proc/${pid}/stat`); + if (!stat) return null; + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + return fields[19] ?? null; +} + +function currentContainerId(): string | null { + const mounts = readOptional("/proc/self/mountinfo"); + return mounts?.match(/\/containers\/([a-f0-9]{64})\//)?.[1] ?? null; +} + +function currentPidNamespace(): string | null { + if (process.platform !== "linux") return `${process.platform}:${hostname()}`; + try { + return readlinkSync("/proc/self/ns/pid"); + } catch { + return null; + } +} + +export function currentLockHolder(purpose: string, now = Date.now()): LockHolder { + return { + version: 1, + purpose, + pid: process.pid, + startTime: processStartTime("self"), + bootId: readOptional("/proc/sys/kernel/random/boot_id"), + pidNamespace: currentPidNamespace(), + containerId: currentContainerId(), + acquiredAt: now, + }; +} + +function processAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +export type ContainerRunning = (containerId: string) => Promise; + +/** Docker's view of a container, or undefined when Docker cannot answer. */ +export const dockerContainerRunning: ContainerRunning = (containerId) => + new Promise((resolve) => { + execFile( + "docker", + ["inspect", "--format", "{{.State.Running}}", containerId], + { timeout: 5_000 }, + (error, stdout, stderr) => { + if (!error) return resolve(stdout.trim() === "true"); + resolve(/no such (object|container)/i.test(stderr) ? false : undefined); + }, + ); + }); + +/** Whether `holder` is provably gone, judged from `self`'s place. */ +export async function lockHolderGone( + holder: LockHolder, + self: LockHolder, + containerRunning: ContainerRunning = dockerContainerRunning, +): Promise { + if (holder.bootId && self.bootId && holder.bootId !== self.bootId) return true; + if (holder.pidNamespace && holder.pidNamespace === self.pidNamespace) { + if (!processAlive(holder.pid)) return true; + const startTime = processStartTime(holder.pid); + return Boolean(holder.startTime && startTime && holder.startTime !== startTime); + } + if (holder.containerId && holder.containerId !== self.containerId) + return (await containerRunning(holder.containerId)) === false; + return false; +} + +function readHolderFile(path: string): { holder?: LockHolder; ino: number } | undefined { + let descriptor: number; + try { + descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + try { + const stat = fstatSync(descriptor); + if (!stat.isFile() || stat.size > 4096) return { ino: stat.ino }; + const buffer = Buffer.alloc(stat.size); + readSync(descriptor, buffer, 0, buffer.length, 0); + try { + return { holder: holderSchema.parse(JSON.parse(buffer.toString("utf8"))), ino: stat.ino }; + } catch { + // A holder that died between creating and writing its file left it empty. + return { ino: stat.ino }; + } + } finally { + closeSync(descriptor); + } +} + +export function readLockHolder(path: string): LockHolder | undefined { + return readHolderFile(path)?.holder; +} + +/** + * Moves a stale lock aside. Reclaimers race through `rename`; the one that + * finds it moved a different file than the one it judged puts that file back. + */ +function reclaim(path: string, ino: number): void { + const aside = `${path}.stale-${randomBytes(6).toString("hex")}`; + try { + renameSync(path, aside); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if (lstatSync(aside).ino !== ino) { + try { + linkSync(aside, path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + } + unlinkSync(aside); + fsyncDirectory(dirname(path)); +} + +export interface RecoverableLockOptions { + purpose: string; + /** How long to wait for a live holder, 5 seconds by default. */ + waitMs?: number; + signal?: AbortSignal; + containerRunning?: ContainerRunning; +} + +export async function acquireRecoverableLock( + path: string, + options: RecoverableLockOptions, +): Promise<() => void> { + const deadline = performance.now() + (options.waitMs ?? 5_000); + for (;;) { + options.signal?.throwIfAborted(); + const self = currentLockHolder(options.purpose); + let descriptor: number | undefined; + try { + descriptor = openSync( + path, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } + if (descriptor !== undefined) { + let ino: number; + try { + writeSync(descriptor, JSON.stringify(self)); + fsyncSync(descriptor); + ino = fstatSync(descriptor).ino; + } finally { + closeSync(descriptor); + } + fsyncDirectory(dirname(path)); + let released = false; + return () => { + if (released) return; + released = true; + const current = readHolderFile(path); + if (current?.ino !== ino) throw new Error(`${basename(path)} was taken from this holder`); + unlinkSync(path); + fsyncDirectory(dirname(path)); + }; + } + const existing = readHolderFile(path); + if (!existing) continue; + if ( + !existing.holder || + (await lockHolderGone(existing.holder, self, options.containerRunning)) + ) { + // An unreadable holder record is only trusted as stale once it has stayed + // unreadable past the wait, so a holder still writing it is not displaced. + if (existing.holder || performance.now() >= deadline) { + reclaim(path, existing.ino); + continue; + } + } + if (performance.now() >= deadline) throw new LockBusyError(path, existing.holder); + await delay(25, undefined, { signal: options.signal }); + } +} + +export async function withRecoverableLock( + path: string, + options: RecoverableLockOptions, + run: () => Promise | T, +): Promise { + const release = await acquireRecoverableLock(path, options); + try { + return await run(); + } finally { + release(); + } +} + +/** The operator exit for a lock whose holder cannot be proved gone. */ +export function clearRecoverableLock(path: string): LockHolder | undefined { + const existing = readHolderFile(path); + if (!existing) return undefined; + reclaim(path, existing.ino); + return existing.holder; +} From 8cae76ba9710e17b5c77fceb873df257c20e7f42 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 12:15:39 +0100 Subject: [PATCH 02/42] feat(control): hold target leases in the agent's store and add openmapx control Every holder of a target lease now goes through the operations agent's own lease store (state/ops-agent/leases.json), written under the recoverable lock. Data-manager uses typed lease operations over its agent token; the host CLI runs the fixed CLI in the agent's container with a host-CLI token the agent creates, and writes the store itself only while the agent is stopped. The fixed CLI, the configuration helper and the catalog bootstrap use the store directly. Stop, restart, compose down, removal and backup create and restore take a lease on their targets, through the same admission path in the agent and in the host CLI; a held target answers busy and names the holder. Community service ids are leased like built-in ones. The shared admission directory on the data volume is gone, and a policy gate keeps it gone. Execution witnesses, data-manager terminals and the feed-proxy terminal move to the observing role's private state; data-manager trusts the caller its token names instead of adopting leases, and transit candidates travel with the child's grant. Lifecycle receipts move to state/ops-agent/lifecycle. The configuration fence names the agent's lease. openmapx control status, clear and resolve show and end held operations; every change is appended to state/ops-agent/control-audit.jsonl. Resolving a maintenance also settles the data-manager fence. services recover-update and the in-process lifecycle exclusion are removed. --- .../0064_configuration_fence_owner_lease.sql | 4 + .../src/db/migrations/meta/0064_snapshot.json | 7588 +++++++++++++++++ apps/api/src/db/migrations/meta/_journal.json | 7 + .../admin-artifact-transfers.test.ts | 2 +- .../services/__tests__/admin-job-ops.test.ts | 12 +- ...configuration-maintenance-postgres.test.ts | 337 +- .../coordinator-lifecycle-postgres.test.ts | 303 +- .../services/__tests__/crash-points/driver.ts | 30 + .../__tests__/crash-points/support/engine.ts | 35 + .../data-configuration-fence-postgres.test.ts | 209 +- ...orm-configuration-runtime-postgres.test.ts | 16 +- .../retained-maintenance-job-postgres.test.ts | 20 +- .../migration-postgres.test.ts | 6 +- .../extension-workflow-postgres.test.ts | 6 +- .../services/integration-application.test.ts | 6 +- .../standalone-integrations/install.test.ts | 6 +- .../transit-feed-credentials-postgres.test.ts | 6 +- apps/ops-agent/src/__fixtures__/shutdown.ts | 2 +- .../artifact-transfer/import-runtime.test.ts | 14 +- .../src/artifact-transfer/ingress.test.ts | 2 +- .../src/artifact-transfer/motis.test.ts | 2 +- apps/ops-agent/src/artifact-transfer/store.ts | 8 +- .../src/backup-prune-runtime.test.ts | 4 +- apps/ops-agent/src/bootstrap.test.ts | 6 +- .../ops-agent/src/catalog-approval-runtime.ts | 10 +- apps/ops-agent/src/catalog-pins.test.ts | 2 +- apps/ops-agent/src/community-repositories.ts | 7 +- .../src/configuration-handoff-application.ts | 2 +- .../src/configuration-handoff-control.test.ts | 8 +- .../src/configuration-handoff-control.ts | 7 +- .../src/configuration-handoff-store.ts | 4 +- .../src/configuration-handoff.test.ts | 2 +- .../src/configuration-maintenance-store.ts | 367 +- .../configuration-writer-concurrency.test.ts | 26 +- apps/ops-agent/src/control-admission.test.ts | 5 +- .../src/control-terminal-restart.test.ts | 13 +- .../src/core-service-runtime.test.ts | 2 +- apps/ops-agent/src/data-manager-terminal.ts | 32 + .../src/data-transit-resource-source.test.ts | 60 +- .../src/data-workflow-runtime.test.ts | 10 +- apps/ops-agent/src/data-workflow-runtime.ts | 3 +- apps/ops-agent/src/docker-runtime.ts | 1 - apps/ops-agent/src/index.ts | 18 +- apps/ops-agent/src/ingress-rate-limit.test.ts | 1 + apps/ops-agent/src/integration-store.test.ts | 15 +- apps/ops-agent/src/journal.ts | 3 +- apps/ops-agent/src/lease-operations.test.ts | 131 + apps/ops-agent/src/lease-operations.ts | 121 + .../ops-agent/src/lifecycle-exclusion.test.ts | 65 - apps/ops-agent/src/lifecycle-exclusion.ts | 213 - apps/ops-agent/src/production-runtime.test.ts | 218 +- apps/ops-agent/src/production-runtime.ts | 52 +- apps/ops-agent/src/runtime.ts | 2 - apps/ops-agent/src/server-foundation.test.ts | 120 +- apps/ops-agent/src/server-review-2.test.ts | 38 +- apps/ops-agent/src/server.test.ts | 34 +- apps/ops-agent/src/server.ts | 143 +- .../src/service-lifecycle-runtime.ts | 5 +- .../test-fixtures/data-owned-http-process.ts | 2 +- .../staged-coordinator-request-process.mjs | 36 - .../terminal-retention-process.ts | 2 +- .../src/traffic-target-ownership.test.ts | 12 +- .../src/trusted-config-ownership.test.ts | 14 +- .../cli/__tests__/cli-build-artifacts.test.ts | 1 - .../cli/__tests__/cli-control-context.test.ts | 1 - packages/cli/__tests__/cli-control.test.ts | 128 + .../__tests__/cli-coordinator-ready.test.ts | 1 - .../cli/__tests__/cli-host-effect.test.ts | 100 + .../__tests__/cli-service-lifecycle.test.ts | 33 +- .../__tests__/cli-service-recovery.test.ts | 57 - .../cli/__tests__/cli-services-update.test.ts | 2 +- packages/cli/src/commands/backup.ts | 18 +- packages/cli/src/commands/compose.ts | 10 +- packages/cli/src/commands/control.ts | 240 + packages/cli/src/commands/services.ts | 35 +- packages/cli/src/index.ts | 2 + packages/cli/src/lib/artifact-ownership.ts | 3 +- packages/cli/src/lib/control-context.ts | 1 - packages/cli/src/lib/control-resolve.ts | 84 + packages/cli/src/lib/data-execution.ts | 2 + packages/cli/src/lib/data-update-command.ts | 3 +- packages/cli/src/lib/host-effect.ts | 100 + packages/cli/src/lib/host-leases.ts | 54 + packages/cli/src/lib/service-lifecycle.ts | 2 +- packages/cli/src/lib/service-recovery.ts | 41 - packages/cli/src/lib/transitous-api-keys.ts | 3 +- .../src/build-artifacts/child-phase.test.ts | 59 +- .../core/src/build-artifacts/child-phase.ts | 8 +- .../configuration-handoff-records.ts | 1 - ...configuration-maintenance-final-records.ts | 6 +- ...nfiguration-maintenance-forward-records.ts | 47 - ...configuration-maintenance-owner-records.ts | 4 +- .../configuration-pre-effect.test.ts | 15 +- .../build-artifacts/control-authority.test.ts | 118 +- .../src/build-artifacts/control-authority.ts | 910 +- .../src/build-artifacts/control-prune.test.ts | 23 +- .../core/src/build-artifacts/control-prune.ts | 87 +- .../src/build-artifacts/control-records.ts | 3 + .../data-configuration-fence.ts | 477 +- .../data-execution-candidate.ts | 450 +- .../build-artifacts/data-execution-journal.ts | 46 +- .../build-artifacts/feed-proxy-validation.ts | 2 - .../fixed-configuration-maintenance.test.ts | 64 +- .../build-artifacts/fixed-data-execution.ts | 9 +- .../generation-hardlinks-owner.ts | 41 +- packages/core/src/build-artifacts/index.ts | 27 +- .../core/src/build-artifacts/lease-remote.ts | 74 + .../src/build-artifacts/lease-store.test.ts | 169 + .../core/src/build-artifacts/lease-store.ts | 181 + .../build-artifacts/motis-portable.test.ts | 1 - .../src/build-artifacts/peer-admission.ts | 134 +- .../src/build-artifacts/peer-process.test.ts | 101 +- .../publication-native.test.ts | 37 +- .../src/build-artifacts/publication.test.ts | 2 +- .../core/src/build-artifacts/publication.ts | 10 +- .../service-lifecycle-recovery.test.ts | 266 - .../service-lifecycle-recovery.ts | 160 - .../src/build-artifacts/target-lease.test.ts | 318 +- .../core/src/build-artifacts/target-lease.ts | 4684 +++------- .../artifact-publication-process.ts | 2 +- .../test-fixtures/peer-process.ts | 39 +- .../service-lifecycle-recovery.ts | 289 - packages/core/src/catalog-pins/bootstrap.ts | 7 +- packages/core/src/catalog-pins/store.test.ts | 6 +- packages/core/src/ops/contract.test.ts | 20 + packages/core/src/ops/contract.ts | 14 +- packages/core/src/ops/lease-records.ts | 101 + .../configuration-generation-readers.test.ts | 2 +- .../services/__tests__/infra-state.test.ts | 6 +- .../__tests__/recoverable-lock.test.ts | 26 + packages/core/src/services/agent-state.ts | 23 +- .../src/services/configuration-ownership.ts | 90 +- .../data-manager/configuration-maintenance.ts | 33 +- .../__fixtures__/reviewed-surface.json | 8 +- .../contracts/configuration-maintenance.ts | 25 +- .../data-manager/contracts/executions.ts | 24 - .../data-manager/contracts/operations.ts | 18 +- .../generated/data-manager.generated.ts | 145 +- .../data-manager/generated/models/index.ts | 53 +- .../inspectDataConfigurationFenceParams.ts | 9 + ...erUnadmittedConfigurationPreparation200.ts | 21 +- ...dmittedConfigurationPreparation200Child.ts | 21 + ...onfigurationPreparation200ChildExecutor.ts | 11 + ...nfigurationPreparation200ChildPrincipal.ts | 11 + ...mittedConfigurationPreparation200Input.ts} | 10 +- ...ationPreparation200InputCoordinatorMode.ts | 10 + ...gurationPreparation200InputOwnerPrepare.ts | 21 + ...Preparation200InputOwnerPrepareExecutor.ts | 11 + ...reparation200InputOwnerPreparePrincipal.ts | 11 + ...admittedConfigurationPreparation200Kind.ts | 9 + ...ittedConfigurationPreparation200Payload.ts | 37 - ...ConfigurationPreparation200PayloadChild.ts | 21 - ...ationPreparation200PayloadChildExecutor.ts | 11 - ...tionPreparation200PayloadChildPrincipal.ts | 11 - ...eparation200PayloadInputCoordinatorMode.ts | 10 - ...nPreparation200PayloadInputOwnerPrepare.ts | 21 - ...tion200PayloadInputOwnerPrepareExecutor.ts | 11 - ...ion200PayloadInputOwnerPreparePrincipal.ts | 11 - ...dConfigurationPreparation200PayloadKind.ts | 9 - ...onfigurationPreparation200PayloadOrigin.ts | 16 - ...rationPreparation200PayloadOriginIssuer.ts | 11 - ...ationPreparation200PayloadOriginVersion.ts | 9 - ...nfigurationPreparation200PayloadVersion.ts | 9 - ...ittedConfigurationPreparation200Version.ts | 9 + ...recoverUnadmittedGenerationHardlinks200.ts | 12 - ...UnadmittedGenerationHardlinks200Payload.ts | 30 - ...ittedGenerationHardlinks200PayloadChild.ts | 21 - ...erationHardlinks200PayloadChildExecutor.ts | 11 - ...rationHardlinks200PayloadChildPrincipal.ts | 11 - ...mittedGenerationHardlinks200PayloadKind.ts | 9 - ...ttedGenerationHardlinks200PayloadOrigin.ts | 16 - ...nerationHardlinks200PayloadOriginIssuer.ts | 11 - ...erationHardlinks200PayloadOriginVersion.ts | 9 - ...ttedGenerationHardlinks200PayloadParent.ts | 21 - ...rationHardlinks200PayloadParentExecutor.ts | 11 - ...ationHardlinks200PayloadParentPrincipal.ts | 11 - ...tedGenerationHardlinks200PayloadVersion.ts | 9 - ...recoverUnadmittedGenerationHardlinks400.ts | 20 - ...recoverUnadmittedGenerationHardlinks401.ts | 20 - ...recoverUnadmittedGenerationHardlinks403.ts | 20 - ...recoverUnadmittedGenerationHardlinks409.ts | 20 - ...recoverUnadmittedGenerationHardlinks413.ts | 20 - ...recoverUnadmittedGenerationHardlinks415.ts | 20 - ...recoverUnadmittedGenerationHardlinks429.ts | 20 - ...recoverUnadmittedGenerationHardlinks500.ts | 20 - ...recoverUnadmittedGenerationHardlinks503.ts | 20 - ...ecoverUnadmittedGenerationHardlinksBody.ts | 16 - ...rUnadmittedGenerationHardlinksBodyInput.ts | 13 - ...rationHardlinksBodyInputCoordinatorMode.ts | 10 - ...dmittedGenerationHardlinksBodyInputKind.ts | 9 - ...erUnadmittedGenerationHardlinksBodyPlan.ts | 9 - ...ttedGenerationHardlinksBodyPlanPlanItem.ts | 23 - .../generated/operation-bindings.generated.ts | 9 - .../core/src/services/data-manager/index.ts | 1 + packages/core/src/services/index.ts | 14 +- .../core/src/services/recoverable-lock.ts | 23 +- .../service-lifecycle-receipt.test.ts | 13 +- .../src/services/service-lifecycle-receipt.ts | 5 +- packages/db-schema/src/data-manager.ts | 3 +- scripts/check-ops-authority.ts | 40 + services/data-manager/__tests__/auth.test.ts | 1 - .../transitous/catalog-review.test.ts | 2 +- .../catalog-validation-current.test.ts | 1 - .../catalog-workflow-postgres.test.ts | 30 +- services/data-manager/openapi.json | 1234 +-- .../src/jobs/execution/api.test.ts | 20 +- .../data-manager/src/jobs/execution/api.ts | 24 +- .../src/jobs/execution/dispatch.ts | 3 +- .../src/jobs/execution/fixed-cli.test.ts | 8 +- .../src/jobs/execution/full-update.test.ts | 14 +- .../src/jobs/execution/hardlinks.test.ts | 20 +- .../legacy-admission-postgres.test.ts | 102 +- .../src/jobs/execution/ops-children.test.ts | 64 +- .../src/jobs/execution/ops-children.ts | 23 +- .../src/jobs/execution/owned-lease.ts | 10 + .../src/jobs/execution/receiver.ts | 12 +- .../execution/runner-cancellation.test.ts | 39 +- .../src/jobs/execution/runtime.test.ts | 36 +- .../src/jobs/execution/runtime.ts | 118 +- .../src/jobs/execution/source-mutation.ts | 5 +- .../src/jobs/execution/store-postgres.test.ts | 1 - .../transitous/catalog-activation-record.ts | 5 +- .../src/jobs/transitous/catalog-activation.ts | 4 +- .../src/jobs/transitous/catalog-review.ts | 4 +- .../transitous/catalog-rollback-retention.ts | 11 +- .../src/jobs/transitous/catalog-validation.ts | 5 +- .../src/jobs/transitous/isolate-producer.ts | 9 +- services/data-manager/src/principals.ts | 1 - services/data-manager/src/service.ts | 6 +- 229 files changed, 13202 insertions(+), 10324 deletions(-) create mode 100644 apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql create mode 100644 apps/api/src/db/migrations/meta/0064_snapshot.json create mode 100644 apps/ops-agent/src/data-manager-terminal.ts create mode 100644 apps/ops-agent/src/lease-operations.test.ts create mode 100644 apps/ops-agent/src/lease-operations.ts delete mode 100644 apps/ops-agent/src/lifecycle-exclusion.test.ts delete mode 100644 apps/ops-agent/src/lifecycle-exclusion.ts create mode 100644 packages/cli/__tests__/cli-control.test.ts create mode 100644 packages/cli/__tests__/cli-host-effect.test.ts delete mode 100644 packages/cli/__tests__/cli-service-recovery.test.ts create mode 100644 packages/cli/src/commands/control.ts create mode 100644 packages/cli/src/lib/control-resolve.ts create mode 100644 packages/cli/src/lib/host-effect.ts create mode 100644 packages/cli/src/lib/host-leases.ts delete mode 100644 packages/cli/src/lib/service-recovery.ts delete mode 100644 packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts create mode 100644 packages/core/src/build-artifacts/lease-remote.ts create mode 100644 packages/core/src/build-artifacts/lease-store.test.ts create mode 100644 packages/core/src/build-artifacts/lease-store.ts delete mode 100644 packages/core/src/build-artifacts/service-lifecycle-recovery.test.ts delete mode 100644 packages/core/src/build-artifacts/service-lifecycle-recovery.ts delete mode 100644 packages/core/src/build-artifacts/test-fixtures/service-lifecycle-recovery.ts create mode 100644 packages/core/src/ops/lease-records.ts create mode 100644 packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Child.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildExecutor.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildPrincipal.ts rename packages/core/src/services/data-manager/generated/models/{recoverUnadmittedConfigurationPreparation200PayloadInput.ts => recoverUnadmittedConfigurationPreparation200Input.ts} (59%) create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputCoordinatorMode.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepare.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Kind.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Payload.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChild.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildExecutor.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildPrincipal.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadKind.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOrigin.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginIssuer.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginVersion.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadVersion.ts create mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Version.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200Payload.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChild.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildExecutor.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildPrincipal.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadKind.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOrigin.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginIssuer.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginVersion.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParent.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentExecutor.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentPrincipal.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadVersion.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks400.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks401.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks403.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks409.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks413.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks415.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks429.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks500.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks503.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBody.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInput.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputKind.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlan.ts delete mode 100644 packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlanPlanItem.ts create mode 100644 services/data-manager/src/jobs/execution/owned-lease.ts diff --git a/apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql b/apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql new file mode 100644 index 000000000..e693102d8 --- /dev/null +++ b/apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql @@ -0,0 +1,4 @@ +-- Fences name the agent's lease now; rows bound to the old signed certificates cannot be kept. +DELETE FROM "data_manager"."configuration_fence";--> statement-breakpoint +ALTER TABLE "data_manager"."configuration_fence" ADD COLUMN "owner_lease_id" uuid NOT NULL;--> statement-breakpoint +ALTER TABLE "data_manager"."configuration_fence" DROP COLUMN "owner_certificate_digest"; \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0064_snapshot.json b/apps/api/src/db/migrations/meta/0064_snapshot.json new file mode 100644 index 000000000..c5df3929f --- /dev/null +++ b/apps/api/src/db/migrations/meta/0064_snapshot.json @@ -0,0 +1,7588 @@ +{ + "id": "12f39f65-a4b7-4cf2-9250-1d2565604fb2", + "prevId": "6612793a-c6b1-45e2-ad89-200a4214a323", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_action_receipt": { + "name": "account_action_receipt", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mail_requested": { + "name": "mail_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_action_actor_time_idx": { + "name": "account_action_actor_time_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_action_target_time_idx": { + "name": "account_action_target_time_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_action_receipt_actor_id_user_id_fk": { + "name": "account_action_receipt_actor_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "account_action_receipt_target_id_user_id_fk": { + "name": "account_action_receipt_target_id_user_id_fk", + "tableFrom": "account_action_receipt", + "tableTo": "user", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account_admin_guard": { + "name": "account_admin_guard", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "active_count": { + "name": "active_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_admin_guard_singleton": { + "name": "account_admin_guard_singleton", + "value": "\"account_admin_guard\".\"id\"=1" + }, + "account_admin_guard_nonnegative": { + "name": "account_admin_guard_nonnegative", + "value": "\"account_admin_guard\".\"active_count\">=0" + } + }, + "isRLSEnabled": false + }, + "public.account_mail_outbox": { + "name": "account_mail_outbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_mail_action_kind_uq": { + "name": "account_mail_action_kind_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "account_mail_pending_idx": { + "name": "account_mail_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_mail_outbox_request_key_account_action_receipt_key_fk": { + "name": "account_mail_outbox_request_key_account_action_receipt_key_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "account_action_receipt", + "columnsFrom": [ + "request_key" + ], + "columnsTo": [ + "key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "account_mail_outbox_user_id_user_id_fk": { + "name": "account_mail_outbox_user_id_user_id_fk", + "tableFrom": "account_mail_outbox", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "account_mail_state_check": { + "name": "account_mail_state_check", + "value": "\"account_mail_outbox\".\"state\" in ('queued','sending','sent','failed','unknown')" + }, + "account_mail_kind_check": { + "name": "account_mail_kind_check", + "value": "\"account_mail_outbox\".\"kind\" in ('verification','password_setup')" + } + }, + "isRLSEnabled": false + }, + "public.admin_action_reauthentication": { + "name": "admin_action_reauthentication", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "starting_session_id": { + "name": "starting_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "nonce_digest": { + "name": "nonce_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "completed_session_id": { + "name": "completed_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_method": { + "name": "completed_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "admin_reauth_actor_expiry_idx": { + "name": "admin_reauth_actor_expiry_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "admin_reauth_request_uq": { + "name": "admin_reauth_request_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_action_reauthentication_actor_id_user_id_fk": { + "name": "admin_action_reauthentication_actor_id_user_id_fk", + "tableFrom": "admin_action_reauthentication", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "admin_reauth_state_check": { + "name": "admin_reauth_state_check", + "value": "\"admin_action_reauthentication\".\"state\" in ('pending','completed','consumed','superseded','expired')" + }, + "admin_reauth_purpose_check": { + "name": "admin_reauth_purpose_check", + "value": "\"admin_action_reauthentication\".\"purpose\" in ('account.verify','account.password.set','account.reset.send','account.invite','credentials.export','credentials.import','transit.credentials','platform.domain.apply','privacy.export')" + } + }, + "isRLSEnabled": false + }, + "public.admin_audit_log": { + "name": "admin_audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "audit_actorId_idx": { + "name": "audit_actorId_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_action_idx": { + "name": "audit_action_idx", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "audit_createdAt_idx": { + "name": "audit_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_audit_log_actor_id_user_id_fk": { + "name": "admin_audit_log_actor_id_user_id_fk", + "tableFrom": "admin_audit_log", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job": { + "name": "admin_job", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "adminJob_status_idx": { + "name": "adminJob_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_type_idx": { + "name": "adminJob_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_createdAt_idx": { + "name": "adminJob_createdAt_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "adminJob_one_active_system_maintenance_uq": { + "name": "adminJob_one_active_system_maintenance_uq", + "columns": [ + { + "expression": "(true)", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"admin_job\".\"type\" IN ('system.update', 'system.diagnostics') AND \"admin_job\".\"status\" IN ('queued', 'running', 'cancel_pending')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_created_by_user_id_fk": { + "name": "admin_job_created_by_user_id_fk", + "tableFrom": "admin_job", + "tableTo": "user", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.admin_job_log": { + "name": "admin_job_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "stream": { + "name": "stream", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'stdout'" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "adminJobLog_jobId_idx": { + "name": "adminJobLog_jobId_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "admin_job_log_job_id_admin_job_id_fk": { + "name": "admin_job_log_job_id_admin_job_id_fk", + "tableFrom": "admin_job_log", + "tableTo": "admin_job", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.app_logs": { + "name": "app_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "msg": { + "name": "msg", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "app_logs_level_source_idx": { + "name": "app_logs_level_source_idx", + "columns": [ + { + "expression": "level", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "app_logs_created_at_idx": { + "name": "app_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_application": { + "name": "auth_policy_application", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_application_actor_id_user_id_fk": { + "name": "auth_policy_application_actor_id_user_id_fk", + "tableFrom": "auth_policy_application", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.auth_policy_state": { + "name": "auth_policy_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "activation_version": { + "name": "activation_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activated_by": { + "name": "activated_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "activated_at": { + "name": "activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "auth_policy_state_activated_by_user_id_fk": { + "name": "auth_policy_state_activated_by_user_id_fk", + "tableFrom": "auth_policy_state", + "tableTo": "user", + "columnsFrom": [ + "activated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "auth_policy_singleton": { + "name": "auth_policy_singleton", + "value": "\"auth_policy_state\".\"id\"=1 AND \"auth_policy_state\".\"activation_version\"=1" + } + }, + "isRLSEnabled": false + }, + "public.capability_binding": { + "name": "capability_binding", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_capability_binding_service": { + "name": "idx_capability_binding_service", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "capability_binding_integration_id_capability_pk": { + "name": "capability_binding_integration_id_capability_pk", + "columns": [ + "integration_id", + "capability" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.configuration_migration_preview": { + "name": "configuration_migration_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "consumed_by_job_id": { + "name": "consumed_by_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "configuration_migration_preview_actor_kind_idx": { + "name": "configuration_migration_preview_actor_kind_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "configuration_migration_preview_expiry_idx": { + "name": "configuration_migration_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "configuration_migration_preview_actor_id_user_id_fk": { + "name": "configuration_migration_preview_actor_id_user_id_fk", + "tableFrom": "configuration_migration_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "configuration_migration_preview_bounds": { + "name": "configuration_migration_preview_bounds", + "value": "\"configuration_migration_preview\".\"kind\" in ('migration','credentials') and length(\"configuration_migration_preview\".\"id\") = 36 and \"configuration_migration_preview\".\"intent_fingerprint\" ~ '^[a-f0-9]{64}$' and octet_length(\"configuration_migration_preview\".\"ciphertext\") <= 1572864 and length(\"configuration_migration_preview\".\"iv\") = 16 and length(\"configuration_migration_preview\".\"tag\") = 24 and \"configuration_migration_preview\".\"expires_at\" > \"configuration_migration_preview\".\"created_at\" and \"configuration_migration_preview\".\"expires_at\" <= \"configuration_migration_preview\".\"created_at\" + interval '10 minutes' and (\"configuration_migration_preview\".\"consumed_by_job_id\" is null or length(\"configuration_migration_preview\".\"consumed_by_job_id\") = 36)" + } + }, + "isRLSEnabled": false + }, + "public.data_disclosure_event": { + "name": "data_disclosure_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "recipient_id": { + "name": "recipient_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_name": { + "name": "recipient_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_role": { + "name": "recipient_role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipient_country": { + "name": "recipient_country", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipient_privacy_url": { + "name": "recipient_privacy_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_code": { + "name": "operation_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category_code": { + "name": "category_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose_code": { + "name": "purpose_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "legal_basis_code": { + "name": "legal_basis_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transfer_safeguard_code": { + "name": "transfer_safeguard_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_reference_digest": { + "name": "external_reference_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_disclosure_event_user_occurred_idx": { + "name": "data_disclosure_event_user_occurred_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_recipient_idx": { + "name": "data_disclosure_event_recipient_idx", + "columns": [ + { + "expression": "recipient_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_disclosure_event_idempotency_idx": { + "name": "data_disclosure_event_idempotency_idx", + "columns": [ + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_disclosure_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_disclosure_event_user_id_user_id_fk": { + "name": "data_disclosure_event_user_id_user_id_fk", + "tableFrom": "data_disclosure_event", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_disclosure_event_code_check": { + "name": "data_disclosure_event_code_check", + "value": "\"data_disclosure_event\".\"recipient_id\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"operation_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"category_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"purpose_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$' and \"data_disclosure_event\".\"legal_basis_code\" ~ '^[a-z0-9][a-z0-9._-]{0,127}$'" + } + }, + "isRLSEnabled": false + }, + "data_manager.configuration_fence": { + "name": "configuration_fence", + "schema": "data_manager", + "columns": { + "maintenance_id": { + "name": "maintenance_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "fence_id": { + "name": "fence_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "plan_digest": { + "name": "plan_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_lease_id": { + "name": "owner_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "input": { + "name": "input", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prepared_at": { + "name": "prepared_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "settled_at": { + "name": "settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "configuration_fence_one_prepared_uq": { + "name": "configuration_fence_one_prepared_uq", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"configuration_fence\".\"state\" = 'prepared'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "configuration_fence_fence_id_unique": { + "name": "configuration_fence_fence_id_unique", + "nullsNotDistinct": false, + "columns": [ + "fence_id" + ] + }, + "configuration_fence_operation_key_unique": { + "name": "configuration_fence_operation_key_unique", + "nullsNotDistinct": false, + "columns": [ + "operation_key" + ] + } + }, + "policies": {}, + "checkConstraints": { + "configuration_fence_state": { + "name": "configuration_fence_state", + "value": "(\"data_manager\".\"configuration_fence\".\"state\" = 'prepared' AND \"data_manager\".\"configuration_fence\".\"outcome\" IS NULL AND \"data_manager\".\"configuration_fence\".\"settled_at\" IS NULL) OR (\"data_manager\".\"configuration_fence\".\"state\" = 'settled' AND \"data_manager\".\"configuration_fence\".\"outcome\" IN ('applied','unused','contained') AND \"data_manager\".\"configuration_fence\".\"settled_at\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "data_manager.feed_state": { + "name": "feed_state", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_fetched_at": { + "name": "last_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_imported_at": { + "name": "last_imported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_message": { + "name": "validation_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.job_stages": { + "name": "job_stages", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "job_stages_job_id_idx": { + "name": "job_stages_job_id_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "job_stages_job_id_jobs_id_fk": { + "name": "job_stages_job_id_jobs_id_fk", + "tableFrom": "job_stages", + "tableTo": "jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.jobs": { + "name": "jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "triggered_by": { + "name": "triggered_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_version": { + "name": "execution_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operation_fingerprint": { + "name": "operation_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "execution_fingerprint": { + "name": "execution_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deadline_at": { + "name": "deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "requested_deadline_at": { + "name": "requested_deadline_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "execution_parent": { + "name": "execution_parent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_input": { + "name": "execution_input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_state": { + "name": "execution_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_children": { + "name": "execution_children", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "ownership_version": { + "name": "ownership_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "data_manager_jobs_execution_key_uq": { + "name": "data_manager_jobs_execution_key_uq", + "columns": [ + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"jobs\".\"operation_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_manager_jobs_execution_bounds": { + "name": "data_manager_jobs_execution_bounds", + "value": "\"data_manager\".\"jobs\".\"ownership_version\" >= 0 AND jsonb_typeof(\"data_manager\".\"jobs\".\"execution_children\") = 'array' AND jsonb_array_length(\"data_manager\".\"jobs\".\"execution_children\") <= 32 AND octet_length(\"data_manager\".\"jobs\".\"execution_children\"::text) <= 131072" + }, + "data_manager_jobs_execution_shape": { + "name": "data_manager_jobs_execution_shape", + "value": "\"data_manager\".\"jobs\".\"execution_version\" IS NULL OR coalesce((\"data_manager\".\"jobs\".\"execution_version\" = 2 AND \"data_manager\".\"jobs\".\"operation_key\" ~ '^opk1_[A-Za-z0-9_-]{16,64}$' AND \"data_manager\".\"jobs\".\"operation_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"execution_fingerprint\" ~ '^[a-f0-9]{64}$' AND \"data_manager\".\"jobs\".\"admitted_at\" >= 0 AND \"data_manager\".\"jobs\".\"deadline_at\" > \"data_manager\".\"jobs\".\"admitted_at\" AND \"data_manager\".\"jobs\".\"execution_input\" IS NOT NULL AND \"data_manager\".\"jobs\".\"execution_state\" IS NOT NULL), false)" + } + }, + "isRLSEnabled": false + }, + "data_manager.offline_package_artifact_references": { + "name": "offline_package_artifact_references", + "schema": "data_manager", + "columns": { + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_length": { + "name": "byte_length", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "retained_at": { + "name": "retained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_artifact_refs_package_idx": { + "name": "offline_package_artifact_refs_package_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_artifact_refs_principal_age_idx": { + "name": "offline_package_artifact_refs_principal_age_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retained_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "offline_package_artifact_references_principal_package_id_pk": { + "name": "offline_package_artifact_references_principal_package_id_pk", + "columns": [ + "principal", + "package_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_job_owners": { + "name": "offline_package_job_owners", + "schema": "data_manager", + "columns": { + "job_id": { + "name": "job_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "principal": { + "name": "principal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "offline_package_job_owners_principal_idx": { + "name": "offline_package_job_owners_principal_idx", + "columns": [ + { + "expression": "principal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "offline_package_job_owners_job_id_offline_package_jobs_id_fk": { + "name": "offline_package_job_owners_job_id_offline_package_jobs_id_fk", + "tableFrom": "offline_package_job_owners", + "tableTo": "offline_package_jobs", + "schemaTo": "data_manager", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "offline_package_job_owners_job_id_principal_pk": { + "name": "offline_package_job_owners_job_id_principal_pk", + "columns": [ + "job_id", + "principal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "data_manager.offline_package_jobs": { + "name": "offline_package_jobs", + "schema": "data_manager", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "package_id": { + "name": "package_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request": { + "name": "request", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "offline_package_jobs_live_request_key_uq": { + "name": "offline_package_jobs_live_request_key_uq", + "columns": [ + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_manager\".\"offline_package_jobs\".\"status\" IN ('preparing', 'ready-to-download')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_status_created_idx": { + "name": "offline_package_jobs_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "offline_package_jobs_package_id_idx": { + "name": "offline_package_jobs_package_id_idx", + "columns": [ + { + "expression": "package_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request": { + "name": "data_subject_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_session_id": { + "name": "actor_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "encrypted_locator": { + "name": "encrypted_locator", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "locator_type": { + "name": "locator_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user_id'" + }, + "account_state": { + "name": "account_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'current'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "registered_at": { + "name": "registered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "preservation_at": { + "name": "preservation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_at": { + "name": "snapshot_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "due_at": { + "name": "due_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "extension": { + "name": "extension", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "identity_method": { + "name": "identity_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_verified_at": { + "name": "identity_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refusal_code": { + "name": "refusal_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_user_state_idx": { + "name": "data_subject_request_user_state_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_due_idx": { + "name": "data_subject_request_due_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "due_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_active_user_kind_idx": { + "name": "data_subject_request_active_user_kind_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request\".\"user_id\" is not null and \"data_subject_request\".\"state\" in ('open', 'in_progress', 'waiting_on_requester')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_user_id_user_id_fk": { + "name": "data_subject_request_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_user_id_user_id_fk": { + "name": "data_subject_request_actor_user_id_user_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "data_subject_request_actor_session_id_session_id_fk": { + "name": "data_subject_request_actor_session_id_session_id_fk", + "tableFrom": "data_subject_request", + "tableTo": "session", + "columnsFrom": [ + "actor_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_kind_check": { + "name": "data_subject_request_kind_check", + "value": "\"data_subject_request\".\"kind\" in ('access', 'portability', 'access_and_portability')" + }, + "data_subject_request_channel_check": { + "name": "data_subject_request_channel_check", + "value": "\"data_subject_request\".\"channel\" in ('self_service', 'email', 'post', 'representative', 'internal')" + }, + "data_subject_request_state_check": { + "name": "data_subject_request_state_check", + "value": "\"data_subject_request\".\"state\" in ('open', 'in_progress', 'waiting_on_requester', 'completed', 'refused')" + }, + "data_subject_request_version_check": { + "name": "data_subject_request_version_check", + "value": "\"data_subject_request\".\"version\" > 0" + }, + "data_subject_request_locator_type_check": { + "name": "data_subject_request_locator_type_check", + "value": "\"data_subject_request\".\"locator_type\" in ('user_id', 'email', 'username', 'erasure_reference', 'other_reference')" + }, + "data_subject_request_account_state_check": { + "name": "data_subject_request_account_state_check", + "value": "\"data_subject_request\".\"account_state\" in ('current', 'inaccessible', 'deleted', 'unknown')" + }, + "data_subject_request_identity_check": { + "name": "data_subject_request_identity_check", + "value": "(\"data_subject_request\".\"identity_method\" is null and \"data_subject_request\".\"identity_verified_at\" is null) or (\"data_subject_request\".\"identity_method\" in ('account_session', 'manual') and \"data_subject_request\".\"identity_verified_at\" is not null)" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_event": { + "name": "data_subject_request_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_kind": { + "name": "actor_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_version": { + "name": "payload_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_event_request_idx": { + "name": "data_subject_request_event_request_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_event_idempotency_idx": { + "name": "data_subject_request_event_idempotency_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"data_subject_request_event\".\"idempotency_key\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_event_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_event_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_event", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_event_payload_version_check": { + "name": "data_subject_request_event_payload_version_check", + "value": "\"data_subject_request_event\".\"payload_version\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_notification": { + "name": "data_subject_request_notification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "channel": { + "name": "channel", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'email'" + }, + "locale": { + "name": "locale", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'en'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_code": { + "name": "last_error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_notification_event_idx": { + "name": "data_subject_request_notification_event_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "template", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "channel", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_state_idx": { + "name": "data_subject_request_notification_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_notification_recipient_idx": { + "name": "data_subject_request_notification_recipient_idx", + "columns": [ + { + "expression": "recipient_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_notification_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_notification_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "data_subject_request_notification_recipient_user_id_user_id_fk": { + "name": "data_subject_request_notification_recipient_user_id_user_id_fk", + "tableFrom": "data_subject_request_notification", + "tableTo": "user", + "columnsFrom": [ + "recipient_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_notification_template_check": { + "name": "data_subject_request_notification_template_check", + "value": "\"data_subject_request_notification\".\"template\" in ('acknowledgement', 'waiting_on_requester', 'extension', 'completed', 'refused', 'admin_new_request', 'admin_deadline')" + }, + "data_subject_request_notification_channel_check": { + "name": "data_subject_request_notification_channel_check", + "value": "\"data_subject_request_notification\".\"channel\" in ('email')" + }, + "data_subject_request_notification_state_check": { + "name": "data_subject_request_notification_state_check", + "value": "\"data_subject_request_notification\".\"state\" in ('pending', 'sending', 'sent', 'retryable', 'failed', 'canceled')" + }, + "data_subject_request_notification_attempts_check": { + "name": "data_subject_request_notification_attempts_check", + "value": "\"data_subject_request_notification\".\"attempts\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.data_subject_request_preservation": { + "name": "data_subject_request_preservation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_cutoff_at": { + "name": "source_cutoff_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "captured_at": { + "name": "captured_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'held'" + }, + "outcome_code": { + "name": "outcome_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "data_subject_request_preservation_request_registration_idx": { + "name": "data_subject_request_preservation_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_preservation_registration_status_idx": { + "name": "data_subject_request_preservation_registration_status_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_preservation_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_preservation_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_preservation", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.data_subject_request_task": { + "name": "data_subject_request_task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "request_id": { + "name": "request_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "public_code": { + "name": "public_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolution_code": { + "name": "resolution_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "data_subject_request_task_request_registration_idx": { + "name": "data_subject_request_task_request_registration_idx", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "data_subject_request_task_status_idx": { + "name": "data_subject_request_task_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "data_subject_request_task_request_id_data_subject_request_id_fk": { + "name": "data_subject_request_task_request_id_data_subject_request_id_fk", + "tableFrom": "data_subject_request_task", + "tableTo": "data_subject_request", + "columnsFrom": [ + "request_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "data_subject_request_task_status_check": { + "name": "data_subject_request_task_status_check", + "value": "\"data_subject_request_task\".\"status\" in ('open', 'resolved')" + } + }, + "isRLSEnabled": false + }, + "public.health_history": { + "name": "health_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response_time": { + "name": "response_time", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "healthHistory_serviceId_checkedAt_idx": { + "name": "healthHistory_serviceId_checkedAt_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_extension": { + "name": "installed_extension", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_trust": { + "name": "source_trust", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'community'" + }, + "installation_kind": { + "name": "installation_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'catalog'" + }, + "provenance": { + "name": "provenance", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_operation_id": { + "name": "last_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedExtension_sourceTrust_idx": { + "name": "installedExtension_sourceTrust_idx", + "columns": [ + { + "expression": "source_trust", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_installed_by_user_id_fk": { + "name": "installed_extension_installed_by_user_id_fk", + "tableFrom": "installed_extension", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "installed_extension_installation_kind_check": { + "name": "installed_extension_installation_kind_check", + "value": "\"installed_extension\".\"installation_kind\" in ('catalog','standalone')" + }, + "installed_extension_standalone_provenance_check": { + "name": "installed_extension_standalone_provenance_check", + "value": "\"installed_extension\".\"installation_kind\" <> 'standalone' or coalesce((\"installed_extension\".\"source_trust\" = 'community' and \"installed_extension\".\"id\" ~ '^local-[0-9a-f-]{36}$' and jsonb_typeof(\"installed_extension\".\"provenance\") = 'object' and \"installed_extension\".\"provenance\"->>'kind' in ('artifact','source','uploaded-artifact','legacy_unknown') and \"installed_extension\".\"provenance\"->>'contentDigest' ~ '^[a-f0-9]{64}$'),false)" + } + }, + "isRLSEnabled": false + }, + "public.installed_extension_component": { + "name": "installed_extension_component", + "schema": "", + "columns": { + "extension_id": { + "name": "extension_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "component_id": { + "name": "component_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "installedExtensionComponent_componentId_idx": { + "name": "installedExtensionComponent_componentId_idx", + "columns": [ + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "installedExtensionComponent_kind_componentId_key": { + "name": "installedExtensionComponent_kind_componentId_key", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "component_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_extension_component_extension_id_installed_extension_id_fk": { + "name": "installed_extension_component_extension_id_installed_extension_id_fk", + "tableFrom": "installed_extension_component", + "tableTo": "installed_extension", + "columnsFrom": [ + "extension_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "installed_extension_component_extension_id_kind_component_id_pk": { + "name": "installed_extension_component_extension_id_kind_component_id_pk", + "columns": [ + "extension_id", + "kind", + "component_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installed_integration": { + "name": "installed_integration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repository": { + "name": "repository", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_version": { + "name": "installed_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'registry'" + }, + "installed_at": { + "name": "installed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "installed_by": { + "name": "installed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "installedIntegration_sourceType_idx": { + "name": "installedIntegration_sourceType_idx", + "columns": [ + { + "expression": "source_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installed_integration_installed_by_user_id_fk": { + "name": "installed_integration_installed_by_user_id_fk", + "tableFrom": "installed_integration", + "tableTo": "user", + "columnsFrom": [ + "installed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_config": { + "name": "integration_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integrationConfig_integrationId_idx": { + "name": "integrationConfig_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationConfig_integrationId_unq": { + "name": "integrationConfig_integrationId_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_secret": { + "name": "integration_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integrationSecret_integrationId_idx": { + "name": "integrationSecret_integrationId_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_secret_updated_by_user_id_fk": { + "name": "integration_secret_updated_by_user_id_fk", + "tableFrom": "integration_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integrationSecret_integrationId_key_unq": { + "name": "integrationSecret_integrationId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "integration_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jwks": { + "name": "jwks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key": { + "name": "private_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "alg": { + "name": "alg", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "crv": { + "name": "crv", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.labeled_place": { + "name": "labeled_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "labeledPlace_userId_idx": { + "name": "labeledPlace_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "labeled_place_user_id_user_id_fk": { + "name": "labeled_place_user_id_user_id_fk", + "tableFrom": "labeled_place", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "labeledPlace_userId_label_unq": { + "name": "labeledPlace_userId_label_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "label" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair": { + "name": "mangrove_keypair", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "passphrase_ciphertext": { + "name": "passphrase_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "recipients_ciphertext": { + "name": "recipients_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_user_id_user_id_fk": { + "name": "mangrove_keypair_user_id_user_id_fk", + "tableFrom": "mangrove_keypair", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mangrove_keypair_wrap": { + "name": "mangrove_keypair_wrap", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrap_type": { + "name": "wrap_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity_string": { + "name": "identity_string", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk": { + "name": "mangrove_keypair_wrap_user_id_mangrove_keypair_user_id_fk", + "tableFrom": "mangrove_keypair_wrap", + "tableTo": "mangrove_keypair", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "user_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_auth_handoff": { + "name": "mobile_auth_handoff", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "callback_code_hash": { + "name": "callback_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_key_version": { + "name": "token_key_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mobile_auth_handoff_user_expires_idx": { + "name": "mobile_auth_handoff_user_expires_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mobile_auth_handoff_user_id_user_id_fk": { + "name": "mobile_auth_handoff_user_id_user_id_fk", + "tableFrom": "mobile_auth_handoff", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "mobile_auth_handoff_callback_code_hash_unique": { + "name": "mobile_auth_handoff_callback_code_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "callback_code_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_access_token": { + "name": "oauth_access_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "refresh_id": { + "name": "refresh_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthAccessToken_clientId_idx": { + "name": "oauthAccessToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_sessionId_idx": { + "name": "oauthAccessToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_userId_idx": { + "name": "oauthAccessToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_authorizationCodeId_idx": { + "name": "oauthAccessToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthAccessToken_refreshId_idx": { + "name": "oauthAccessToken_refreshId_idx", + "columns": [ + { + "expression": "refresh_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_access_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_access_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_session_id_session_id_fk": { + "name": "oauth_access_token_session_id_session_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_access_token_user_id_user_id_fk": { + "name": "oauth_access_token_user_id_user_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_access_token_refresh_id_oauth_refresh_token_id_fk": { + "name": "oauth_access_token_refresh_id_oauth_refresh_token_id_fk", + "tableFrom": "oauth_access_token", + "tableTo": "oauth_refresh_token", + "columnsFrom": [ + "refresh_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_access_token_token_unique": { + "name": "oauth_access_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret": { + "name": "client_secret", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "client_discovery_id": { + "name": "client_discovery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "skip_consent": { + "name": "skip_consent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "enable_end_session": { + "name": "enable_end_session", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "subject_type": { + "name": "subject_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "client_credentials_scopes": { + "name": "client_credentials_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "contacts": { + "name": "contacts", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "tos": { + "name": "tos", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_id": { + "name": "software_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_version": { + "name": "software_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "software_statement": { + "name": "software_statement", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "post_logout_redirect_uris": { + "name": "post_logout_redirect_uris", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_uri": { + "name": "backchannel_logout_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "backchannel_logout_session_required": { + "name": "backchannel_logout_session_required", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "application_type": { + "name": "application_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks": { + "name": "jwks", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "jwks_uri": { + "name": "jwks_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "grant_types": { + "name": "grant_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "response_types": { + "name": "response_types", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "require_pkce": { + "name": "require_pkce", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens": { + "name": "dpop_bound_access_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClient_userId_idx": { + "name": "oauthClient_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_user_id_user_id_fk": { + "name": "oauth_client_user_id_user_id_fk", + "tableFrom": "oauth_client", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_client_client_id_unique": { + "name": "oauth_client_client_id_unique", + "nullsNotDistinct": false, + "columns": [ + "client_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_assertion": { + "name": "oauth_client_assertion", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client_resource": { + "name": "oauth_client_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "oauthClientResource_clientId_resourceId_uidx": { + "name": "oauthClientResource_clientId_resourceId_uidx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_clientId_idx": { + "name": "oauthClientResource_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthClientResource_resourceId_idx": { + "name": "oauthClientResource_resourceId_idx", + "columns": [ + { + "expression": "resource_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_client_resource_client_id_oauth_client_client_id_fk": { + "name": "oauth_client_resource_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_client_resource_resource_id_oauth_resource_identifier_fk": { + "name": "oauth_client_resource_resource_id_oauth_resource_identifier_fk", + "tableFrom": "oauth_client_resource", + "tableTo": "oauth_resource", + "columnsFrom": [ + "resource_id" + ], + "columnsTo": [ + "identifier" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_consent": { + "name": "oauth_consent", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthConsent_clientId_idx": { + "name": "oauthConsent_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthConsent_userId_idx": { + "name": "oauthConsent_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_consent_client_id_oauth_client_client_id_fk": { + "name": "oauth_consent_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_consent_user_id_user_id_fk": { + "name": "oauth_consent_user_id_user_id_fk", + "tableFrom": "oauth_consent", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_refresh_token": { + "name": "oauth_refresh_token", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_code_id": { + "name": "authorization_code_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resources": { + "name": "resources", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "requested_user_info_claims": { + "name": "requested_user_info_claims", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotated_at": { + "name": "rotated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_response": { + "name": "rotation_replay_response", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rotation_replay_expires_at": { + "name": "rotation_replay_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "auth_time": { + "name": "auth_time", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "confirmation": { + "name": "confirmation", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauthRefreshToken_clientId_idx": { + "name": "oauthRefreshToken_clientId_idx", + "columns": [ + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_sessionId_idx": { + "name": "oauthRefreshToken_sessionId_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_userId_idx": { + "name": "oauthRefreshToken_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauthRefreshToken_authorizationCodeId_idx": { + "name": "oauthRefreshToken_authorizationCodeId_idx", + "columns": [ + { + "expression": "authorization_code_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "oauth_refresh_token_client_id_oauth_client_client_id_fk": { + "name": "oauth_refresh_token_client_id_oauth_client_client_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "oauth_client", + "columnsFrom": [ + "client_id" + ], + "columnsTo": [ + "client_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "oauth_refresh_token_session_id_session_id_fk": { + "name": "oauth_refresh_token_session_id_session_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "oauth_refresh_token_user_id_user_id_fk": { + "name": "oauth_refresh_token_user_id_user_id_fk", + "tableFrom": "oauth_refresh_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_refresh_token_token_unique": { + "name": "oauth_refresh_token_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_resource": { + "name": "oauth_resource", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_ttl": { + "name": "access_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refresh_token_ttl": { + "name": "refresh_token_ttl", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "signing_algorithm": { + "name": "signing_algorithm", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signing_key_id": { + "name": "signing_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "allowed_scopes": { + "name": "allowed_scopes", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "custom_claims": { + "name": "custom_claims", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "dpop_bound_access_tokens_required": { + "name": "dpop_bound_access_tokens_required", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "disabled": { + "name": "disabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "policy_version": { + "name": "policy_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "oauth_resource_identifier_unique": { + "name": "oauth_resource_identifier_unique", + "nullsNotDistinct": false, + "columns": [ + "identifier" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parked_location": { + "name": "parked_location", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "vehicle_id": { + "name": "vehicle_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "accuracy_meters": { + "name": "accuracy_meters", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "saved_at": { + "name": "saved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parkedLocation_userId_idx": { + "name": "parkedLocation_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parked_location_user_id_user_id_fk": { + "name": "parked_location_user_id_user_id_fk", + "tableFrom": "parked_location", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parked_location_vehicle_id_personal_vehicle_id_fk": { + "name": "parked_location_vehicle_id_personal_vehicle_id_fk", + "tableFrom": "parked_location", + "tableTo": "personal_vehicle", + "columnsFrom": [ + "vehicle_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parkedLocation_userId_vehicleId_unq": { + "name": "parkedLocation_userId_vehicleId_unq", + "nullsNotDistinct": true, + "columns": [ + "user_id", + "vehicle_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.passkey": { + "name": "passkey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "credential_id": { + "name": "credential_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "device_type": { + "name": "device_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backed_up": { + "name": "backed_up", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "aaguid": { + "name": "aaguid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "passkey_userId_idx": { + "name": "passkey_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "passkey_credentialID_idx": { + "name": "passkey_credentialID_idx", + "columns": [ + { + "expression": "credential_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "passkey_user_id_user_id_fk": { + "name": "passkey_user_id_user_id_fk", + "tableFrom": "passkey", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_timeline_connection": { + "name": "personal_timeline_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_origin": { + "name": "public_origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_api_key": { + "name": "encrypted_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_iv": { + "name": "encryption_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encryption_tag": { + "name": "encryption_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "upstream_user_id": { + "name": "upstream_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_email": { + "name": "upstream_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream_time_zone": { + "name": "upstream_time_zone", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "distance_unit": { + "name": "distance_unit", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'connected'" + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "validated_at": { + "name": "validated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_read_at": { + "name": "last_read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalTimelineConnection_userId_idx": { + "name": "personalTimelineConnection_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_timeline_connection_user_id_user_id_fk": { + "name": "personal_timeline_connection_user_id_user_id_fk", + "tableFrom": "personal_timeline_connection", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalTimelineConnection_userId_unq": { + "name": "personalTimelineConnection_userId_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.personal_vehicle": { + "name": "personal_vehicle", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "powertrain": { + "name": "powertrain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "preset_id": { + "name": "preset_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ev": { + "name": "ev", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fuel_consumption_l_per_100km": { + "name": "fuel_consumption_l_per_100km", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "personalVehicle_userId_idx": { + "name": "personalVehicle_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "personal_vehicle_user_id_user_id_fk": { + "name": "personal_vehicle_user_id_user_id_fk", + "tableFrom": "personal_vehicle", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "personalVehicle_userId_name_unq": { + "name": "personalVehicle_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.platform_configuration_preview": { + "name": "platform_configuration_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "preview_fingerprint": { + "name": "preview_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "auth_policy_revision": { + "name": "auth_policy_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_revision": { + "name": "source_revision", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expected_current_revision_id": { + "name": "expected_current_revision_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "consumed_by_job_id": { + "name": "consumed_by_job_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "retired_at": { + "name": "retired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "platform_preview_actor_key_uq": { + "name": "platform_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "platform_preview_fingerprint_uq": { + "name": "platform_preview_fingerprint_uq", + "columns": [ + { + "expression": "preview_fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "platform_preview_expiry_idx": { + "name": "platform_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "platform_configuration_preview_actor_id_user_id_fk": { + "name": "platform_configuration_preview_actor_id_user_id_fk", + "tableFrom": "platform_configuration_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "platform_preview_bounds": { + "name": "platform_preview_bounds", + "value": "length(\"platform_configuration_preview\".\"operation_key\") between 1 and 128 and \"platform_configuration_preview\".\"request_fingerprint\" ~ '^[a-f0-9]{64}$' and \"platform_configuration_preview\".\"preview_fingerprint\" ~ '^[a-f0-9]{64}$' and \"platform_configuration_preview\".\"auth_policy_revision\" ~ '^[a-f0-9]{64}$' and octet_length(\"platform_configuration_preview\".\"preview\"::text) <= 131072 and jsonb_typeof(\"platform_configuration_preview\".\"preview\") = 'object' and \"platform_configuration_preview\".\"expires_at\" > \"platform_configuration_preview\".\"created_at\" and \"platform_configuration_preview\".\"expires_at\" <= \"platform_configuration_preview\".\"created_at\" + interval '10 minutes' and ((\"platform_configuration_preview\".\"consumed_at\" is null) = (\"platform_configuration_preview\".\"consumed_by_job_id\" is null)) and (\"platform_configuration_preview\".\"retired_at\" is null or (\"platform_configuration_preview\".\"consumed_at\" is not null and \"platform_configuration_preview\".\"retired_at\" >= \"platform_configuration_preview\".\"consumed_at\"))" + } + }, + "isRLSEnabled": false + }, + "public.saved_list": { + "name": "saved_list", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedList_userId_idx": { + "name": "savedList_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_list_user_id_user_id_fk": { + "name": "saved_list_user_id_user_id_fk", + "tableFrom": "saved_list", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "savedList_userId_name_unq": { + "name": "savedList_userId_name_unq", + "nullsNotDistinct": false, + "columns": [ + "user_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.saved_place": { + "name": "saved_place", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "list_id": { + "name": "list_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "lng": { + "name": "lng", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "place_id": { + "name": "place_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "field_contributors": { + "name": "field_contributors", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "savedPlace_listId_idx": { + "name": "savedPlace_listId_idx", + "columns": [ + { + "expression": "list_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "saved_place_list_id_saved_list_id_fk": { + "name": "saved_place_list_id_saved_list_id_fk", + "tableFrom": "saved_place", + "tableTo": "saved_list", + "columnsFrom": [ + "list_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_config": { + "name": "service_config", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "serviceConfig_serviceId_idx": { + "name": "serviceConfig_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceConfig_serviceId_unq": { + "name": "serviceConfig_serviceId_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_repository": { + "name": "service_repository", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sha": { + "name": "last_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pinned_ref": { + "name": "pinned_ref", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "managed_by_extension": { + "name": "managed_by_extension", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_transaction_id": { + "name": "last_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "service_repository_url_unique": { + "name": "service_repository_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.service_secret": { + "name": "service_secret", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "serviceSecret_serviceId_idx": { + "name": "serviceSecret_serviceId_idx", + "columns": [ + { + "expression": "service_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "service_secret_updated_by_user_id_fk": { + "name": "service_secret_updated_by_user_id_fk", + "tableFrom": "service_secret", + "tableTo": "user", + "columnsFrom": [ + "updated_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "serviceSecret_serviceId_key_unq": { + "name": "serviceSecret_serviceId_key_unq", + "nullsNotDistinct": false, + "columns": [ + "service_id", + "key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_auth_assurance": { + "name": "session_auth_assurance", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_independent_password_login": { + "name": "origin_independent_password_login", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "authenticated_at": { + "name": "authenticated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_auth_assurance_user_idx": { + "name": "session_auth_assurance_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "authenticated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_auth_assurance_session_id_session_id_fk": { + "name": "session_auth_assurance_session_id_session_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_auth_assurance_user_id_user_id_fk": { + "name": "session_auth_assurance_user_id_user_id_fk", + "tableFrom": "session_auth_assurance", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "session_auth_assurance_method_check": { + "name": "session_auth_assurance_method_check", + "value": "\"session_auth_assurance\".\"method\" in ('password', 'password_totp', 'password_recovery', 'passkey', 'federated')" + } + }, + "isRLSEnabled": false + }, + "public.share_link": { + "name": "share_link", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "snapshot": { + "name": "snapshot", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "shareLink_userId_idx": { + "name": "shareLink_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_link_user_id_user_id_fk": { + "name": "share_link_user_id_user_id_fk", + "tableFrom": "share_link", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "share_link_token_hash_unique": { + "name": "share_link_token_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "token_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transit_credential_preview": { + "name": "transit_credential_preview", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation_key": { + "name": "operation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "intent_fingerprint": { + "name": "intent_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "iv": { + "name": "iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_ciphertext": { + "name": "authorization_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_iv": { + "name": "authorization_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authorization_tag": { + "name": "authorization_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "credentials_revision": { + "name": "credentials_revision", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "transit_credential_preview_actor_key_uq": { + "name": "transit_credential_preview_actor_key_uq", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "operation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transit_credential_preview_expiry_idx": { + "name": "transit_credential_preview_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "transit_credential_preview_actor_id_user_id_fk": { + "name": "transit_credential_preview_actor_id_user_id_fk", + "tableFrom": "transit_credential_preview", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "transit_credential_preview_bounds": { + "name": "transit_credential_preview_bounds", + "value": "octet_length(\"transit_credential_preview\".\"ciphertext\") <= 1500000 and length(\"transit_credential_preview\".\"intent_fingerprint\") = 64 and length(\"transit_credential_preview\".\"iv\") = 16 and length(\"transit_credential_preview\".\"tag\") = 24" + } + }, + "isRLSEnabled": false + }, + "public.two_factor": { + "name": "two_factor", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backup_codes": { + "name": "backup_codes", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verified": { + "name": "verified", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "failed_verification_count": { + "name": "failed_verification_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "locked_until": { + "name": "locked_until", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "twoFactor_secret_idx": { + "name": "twoFactor_secret_idx", + "columns": [ + { + "expression": "secret", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "twoFactor_userId_idx": { + "name": "twoFactor_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "two_factor_user_id_user_id_fk": { + "name": "two_factor_user_id_user_id_fk", + "tableFrom": "two_factor", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "normalized_email": { + "name": "normalized_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "two_factor_enabled": { + "name": "two_factor_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + }, + "user_normalized_email_unique": { + "name": "user_normalized_email_unique", + "nullsNotDistinct": false, + "columns": [ + "normalized_email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_application_receipt": { + "name": "integration_application_receipt", + "schema": "", + "columns": { + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "capability": { + "name": "capability", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_id": { + "name": "service_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "candidate_id": { + "name": "candidate_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "intent_digest": { + "name": "intent_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "saved_binding_revision": { + "name": "saved_binding_revision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_application_target_time_idx": { + "name": "integration_application_target_time_idx", + "columns": [ + { + "expression": "integration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_application_actor_idx": { + "name": "integration_application_actor_idx", + "columns": [ + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_application_receipt_actor_id_user_id_fk": { + "name": "integration_application_receipt_actor_id_user_id_fk", + "tableFrom": "integration_application_receipt", + "tableTo": "user", + "columnsFrom": [ + "actor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_binding_revision": { + "name": "integration_binding_revision", + "schema": "", + "columns": { + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "data_manager": "data_manager" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 495290e06..61b07b420 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -449,6 +449,13 @@ "when": 1794669803740, "tag": "0063_configuration_fence_table", "breakpoints": true + }, + { + "idx": 64, + "version": "7", + "when": 1794734781139, + "tag": "0064_configuration_fence_owner_lease", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts b/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts index 46dd32378..a3bfbca75 100644 --- a/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts +++ b/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts @@ -55,7 +55,7 @@ it("generated SDK streams through real admin/API and typed/raw agent paths, enfo const audit: unknown[] = []; const agent = buildOpsAgentServer({ audit: (event) => audit.push(event), - tokens: { api: token, "data-manager": "b".repeat(43) }, + tokens: { api: token, "data-manager": "b".repeat(43), "host-cli": "h".repeat(43) }, runtime, artifactTransfers: store, resourcePolicy: { allowGlobal: () => true }, diff --git a/apps/api/src/services/__tests__/admin-job-ops.test.ts b/apps/api/src/services/__tests__/admin-job-ops.test.ts index dd87b909f..740196c7b 100644 --- a/apps/api/src/services/__tests__/admin-job-ops.test.ts +++ b/apps/api/src/services/__tests__/admin-job-ops.test.ts @@ -791,7 +791,11 @@ it.each([false, true])( return { backupId: "original" }; }); let app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, + tokens: { + api: token, + "data-manager": Buffer.alloc(32, 8).toString("base64url"), + "host-cli": "h".repeat(43), + }, journal, controlInstallation: installation, dispatch, @@ -853,7 +857,11 @@ it.each([false, true])( await app.close(); const reloaded = await openOpsJobJournal(path); app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, + tokens: { + api: token, + "data-manager": Buffer.alloc(32, 8).toString("base64url"), + "host-cli": "h".repeat(43), + }, journal: reloaded, controlInstallation: installation, dispatch, diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index eb3d7c5a1..108685508 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -19,12 +19,15 @@ import { setTimeout as nativeDelay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { closeControlInstallation, + heldTargets, + leaseStoreDirectory, lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding, openControlInstallation, + openLeaseStore, pruneControlRecords, + readDataControlTerminalObservation, restoreAdmittedControlExecution, - verifyDataControlTerminalWitness, } from "@openmapx/core/build-artifacts"; import { canonicalJson } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; @@ -47,6 +50,19 @@ import { postgresCommitRelay } from "./fixtures/postgres-commit-relay"; */ const NATIVE_API_APPLICATION_NAME = "openmapx-api"; +/** The lease in the agent's store that holds `target`, or null. */ +function targetLease(infraDir: string, target: string): string | null { + return ( + openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: "test" }) + .list() + .find((lease) => lease.targets.includes(target))?.leaseId ?? null + ); +} +/** One lease as the agent's store records it now. */ +function heldLease(infraDir: string, leaseId: string) { + return openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: "test" }).get(leaseId); +} + /** * A lease renewal a native API sent before it was killed can still commit after * the kill, for example when it was waiting on a row lock that the API's open @@ -216,7 +232,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); const journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal, @@ -1159,8 +1179,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( `${candidate.sealedOperationKey}.json`, ), candidateBytes = retired ? undefined : readFileSync(candidatePath), - rootLock = join(host.infraDir, "data/.admin-target-control/locks/data-manager"), - rootInode = existsSync(rootLock) ? statSync(rootLock).ino : null; + rootLease = () => targetLease(host.infraDir, "data-manager"), + rootInode = rootLease(); expect(rootInode === null).toBe( mode === "after-settle" || preRootHelper || @@ -1256,7 +1276,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ? !existsSync(candidatePath) : readFileSync(candidatePath).equals(candidateBytes!), ).toBe(true); - expect(existsSync(rootLock) ? statSync(rootLock).ino : null).toBe(rootInode); + expect(rootLease()).toBe(rootInode); if (mode === "before-root-start-source") await fixture.db .insert(schema.serviceConfig) @@ -1314,9 +1334,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), authority = restoreAdmittedControlExecution(installation, owner.plan.prepare); core.assertConfigurationMaintenanceRootClosure(authority); - core.verifyRetainedArtifactTargetRootRelease(authority); + await core.verifyRetainedArtifactTargetRootRelease(authority); expect(existsSync(candidatePath)).toBe(false); - expect(existsSync(rootLock)).toBe(false); + expect(rootLease()).toBeNull(); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, @@ -1387,7 +1407,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ).toBe(false); expect(existsSync(candidatePath)).toBe(false); - expect(existsSync(rootLock)).toBe(false); + expect(rootLease()).toBeNull(); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, @@ -1399,7 +1419,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).receipt, core = await import("@openmapx/core/build-artifacts"); expect(final.prepare.operationKey).toBe(candidate.prepareOperationKey); - core.verifyRetainedArtifactTargetRootRelease( + await core.verifyRetainedArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, final.prepare), ); expect((await start().exited).code).toBe(2); @@ -1461,7 +1481,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).equals(helperAdmission!), ).toBe(true); expect(existsSync(candidatePath)).toBe(false); - expect(existsSync(rootLock)).toBe(false); + expect(rootLease()).toBeNull(); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, @@ -1520,7 +1540,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).equals(helperAdmission!), ).toBe(true); expect(existsSync(candidatePath)).toBe(false); - expect(existsSync(rootLock)).toBe(false); + expect(rootLease()).toBeNull(); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, @@ -1569,7 +1589,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( receipt: { state: "prepared", decision: null, generation: "not_started" }, }); expect(readFileSync(candidatePath).equals(candidateBytes!)).toBe(true); - expect(statSync(rootLock).ino).toBe(rootInode); + expect(rootLease()).toBe(rootInode); expect(preparePosts).toBe(1); expect(settlePosts).toBe(0); expect(dm!.settlePosts).toBe(0); @@ -1628,7 +1648,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(result._configurationMaintenanceSettle).toEqual( originalResult._configurationMaintenanceSettle, ); - expect(finalOwner.rootCertificateDigest).toBe(originalOwner.rootCertificateDigest); + expect(finalOwner.leaseId).toBe(originalOwner.leaseId); } if (knownRollback) { expect(result._configurationMaintenanceRollback).toEqual( @@ -1661,9 +1681,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect( existsSync(join(host.queue, ".candidates", `${candidate.sealedOperationKey}.json`)), ).toBe(false); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect(host.calls.filter((args) => args.includes("up"))).toHaveLength( unused || contained ? 0 : preRootHelper ? 2 : 1, ); @@ -2273,9 +2291,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(candidate.revisionId).not.toBe(host.initial.revisionId); expect(dm!.preparePosts).toBe(0); expect(actualPosts.get("configurationMaintenance.prepare") ?? 0).toBe(0); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect(await fixture.db.select().from(schema.serviceSecret)).toHaveLength(80); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, @@ -2554,9 +2570,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, }), ).toThrow(); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, @@ -2626,9 +2640,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await prospective.stageProspectiveConfigurationMaintenance(candidate); expect(starts).toBe(1); expect(dm!.preparePosts).toBe(0); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); const helperPath = join( host.infraDir, "control-private/ops-agent/executions", @@ -2833,9 +2845,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, ); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect(coordinator.State).toBe("running"); expect(ctx.checkpointResult!._configurationCandidate).toMatchObject({ candidateId: candidate.candidateId, @@ -3252,9 +3262,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( host.initial.revisionId, ); expect(readdirSync(host.queue).filter((name) => name.startsWith("cfg1_"))).toEqual([]); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); }); }, 60000, @@ -3326,9 +3334,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, }, }); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect( host.calls.some( (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), @@ -3390,10 +3396,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( core = await import("@openmapx/core/build-artifacts"), authority = restoreAdmittedControlExecution(installation, prepared.prepare), lease = core.restoreArtifactTargetLeaseForSettlement(authority), - leaseDirectory = join(host.infraDir, "data/.admin-target-control/leases", lease.leaseId), - ownerPath = join(leaseDirectory, "owner.json"), - ownerBytes = readFileSync(ownerPath), - ownerInode = statSync(leaseDirectory).ino, + ownerRecord = () => JSON.stringify(heldLease(host.infraDir, lease.leaseId)), + ownerBytes = ownerRecord(), originalAdmissionPath = join( host.infraDir, "control-private/ops-agent/executions", @@ -3453,13 +3457,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ).toBe(false); expect(dm!.preparePosts).toBe(1); - expect(JSON.parse(readFileSync(ownerPath, "utf8")).payload.certificate).toEqual( - JSON.parse(ownerBytes.toString()).payload.certificate, - ); - expect(JSON.parse(readFileSync(ownerPath, "utf8")).payload.origin).toEqual( - JSON.parse(ownerBytes.toString()).payload.origin, - ); - expect(statSync(leaseDirectory).ino).toBe(ownerInode); + // The root lease still names the same owner and targets. + const after = heldLease(host.infraDir, lease.leaseId), + before = JSON.parse(ownerBytes); + expect(after?.owner).toEqual(before.owner); + expect(after?.targets).toEqual(before.targets); expect(readFileSync(originalAdmissionPath)).toEqual(originalAdmission); const step = prepared.steps[0]!, projected = (await import("../admin-job-ops")).readProjection(ctx.checkpointResult) @@ -3472,9 +3474,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( receiptDelivery: expect.anything(), }); expect(await maintenance.executeConfigurationMaintenanceStep(candidate, 0)).toEqual(result); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); }); }, 60000); it.each([false, true])( @@ -3606,12 +3606,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), ), ).toBe(false); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis"))).toBe( - true, - ); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); + expect(heldTargets(host.infraDir).includes("motis")).toBe(true); if (recreate) { expect(prepared.steps[2]).toMatchObject({ phase: "maintenance-recreate", @@ -3753,7 +3749,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); const restartedRuntime = await host.reopenRuntime(); app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), journal, @@ -3890,9 +3890,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(owner.final).toBeNull(); expect(owner.inspection).toBeNull(); expect(owner.receipt.dmFence!.state).toBe("prepared"); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); await expect(dm!.receiver.admitLocal({ kind: "download-fonts" })).rejects.toThrow( "configuration_maintenance_busy", ); @@ -3946,9 +3944,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( dmFence: { state: "settled" }, }); expect(owner.apiControls.steps).toHaveLength(1); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect( existsSync( join(host.queue, ".candidates", `${owner.plan.operation.applyOperationKey}.json`), @@ -3984,6 +3980,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -4223,9 +4220,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( followUp: "complete", dmFence: { state: "settled" }, }); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); } else if (mode === "component") { const shapes = await import( "../../../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records" @@ -4338,119 +4333,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, 180000, ); - it("rejects a real durable nonempty middle-owner grant with only the original pending decision witness", async () => { - dm = await ownedDataManager({ - sql: fixture.sql, - infraDir: host.infraDir, - serviceIds: host.loaded.map((service) => service.manifest.id), - }); - vi.stubEnv("DATA_MANAGER_URL", dm.baseUrl); - vi.stubEnv("DATA_MANAGER_TOKEN_FILE", dataManagerTokenFile(dm.token)); - host.setRows([ - host.row, - { - ...host.row, - Id: "c".repeat(64), - State: "running", - ConfigImage: "example/data-manager:1", - Labels: { ...host.row.Labels, "com.docker.compose.service": "data-manager" }, - }, - ]); - const ctx = await job(), - observation = new AbortController(); - ctx.signal = observation.signal; - await owned(ctx, async () => { - const candidate = await prospective.prepareProspectiveConfiguration({ - actorId: ctx.jobId, - operationKey: `opk1_${ctx.jobId.replaceAll("-", "")}`, - expectedCurrentRevisionId: host.initial.revisionId, - followUp: "service-apply", - overlay: { - purpose: "service-secret", - coordinatorMode: "require-running", - changes: [ - { - kind: "service-secret", - serviceId: "data-manager", - key: "TOKEN", - action: "set", - value: "synthetic-no-decision-credential", - }, - ], - }, - }), - maintenance = await import("../configuration-maintenance"), - prepared = await maintenance.prepareConfigurationMaintenance(candidate), - witnessPath = join( - host.infraDir, - "data/.admin-target-control/executions/ops-agent", - `${prepared.prepare.operationKey}.configuration-maintenance.json`, - ), - pending = readFileSync(witnessPath), - decision = await prospective.commitProspectiveConfigurationMaintenance(candidate); - expect(JSON.parse(pending.toString()).payload).toMatchObject({ - decision: null, - generation: null, - }); - await maintenance.decideConfigurationMaintenance(decision); - await maintenance.executeConfigurationMaintenanceStep(candidate, 0); - const nativeFetch = globalThis.fetch, - core = await import("@openmapx/core/build-artifacts"); - let denied = 0, - validGrant = false, - leafKey = ""; - const transport = vi.spyOn(globalThis, "fetch").mockImplementation(async (...args) => { - if ( - !String(args[0]).endsWith("/executions/generation-hardlinks") || - args[1]?.method !== "POST" - ) - return nativeFetch(...args); - const headers = new Headers(args[1].headers), - grant = core.inspectArtifactDelegation({ - installation, - dataRoot: join(host.infraDir, "data"), - assertion: headers.get("x-openmapx-control-delegation")!, - }); - leafKey = grant.execution.operationKey; - validGrant = - grant.kind === "generation-hardlinks" && - grant.parent.operationKey === prepared.steps[1]!.operationKey && - grant.owner.operationKey === prepared.prepare.operationKey; - const current = readFileSync(witnessPath); - // Named native shared-publication rollback only; both packets are the - // actual producer's signatures. No private source/grant is fabricated. - writeFileSync(witnessPath, pending); - try { - const response = await nativeFetch(...args); - denied = response.status; - observation.abort(); - return response; - } finally { - writeFileSync(witnessPath, current); - } - }); - try { - await expect( - maintenance.executeConfigurationMaintenanceStep(candidate, 1), - ).rejects.toMatchObject({ name: "OpsObservationPending" }); - } finally { - transport.mockRestore(); - } - expect(validGrant).toBe(true); - expect(denied).toBe(409); - expect( - await fixture.sql`SELECT id FROM data_manager.jobs WHERE kind='generation-hardlinks' AND operation_key=${leafKey}`, - ).toHaveLength(0); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); - expect( - host.calls.some( - (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), - ), - ).toBe(false); - }); - }, 120000); it.each(["expired", "host-source"])( "refuses actual durable nonempty committed continue after %s drift without admitting any effect", async (reason) => { @@ -4528,9 +4410,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ).toBe(false); expect(dm!.preparePosts).toBe(1); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); }); } finally { vi.useRealTimers(); @@ -4648,9 +4528,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( : { status: 409, errorClass: "conflict" }, ]); expect(readFileSync(path)).toEqual(before); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); expect(dm!.preparePosts).toBe(1); }); }, @@ -4754,7 +4632,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); const restartedRuntime = await host.reopenRuntime(); app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), journal, @@ -4791,9 +4673,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await expect(dm!.receiver.admitLocal({ kind: "download-fonts" })).rejects.toThrow( "configuration_maintenance_busy", ); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); expect( host.calls.some( (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), @@ -4929,9 +4809,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( dmFence: { state: "settled" }, }); expect(readFileSync(admissionPath)).toEqual(originalAdmission); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect( host.calls.some( (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), @@ -5047,7 +4925,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), records.configurationMaintenanceOwnerRecordSchema, ), - released = core.verifyRetainedArtifactTargetRootRelease(authority), + released = await core.verifyRetainedArtifactTargetRootRelease(authority), originalFinal = JSON.stringify(owner.final), originalInspection = JSON.stringify(owner.inspection), [originalSql] = @@ -5066,7 +4944,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); const restartedRuntime = await host.reopenRuntime(); app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), journal, @@ -5092,9 +4974,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await fixture.sql`SELECT state,outcome,settled_at,xmin::text AS xmin FROM data_manager.configuration_fence WHERE fence_id=${prepared.dmFence!.fenceId}::uuid` )[0], ).toEqual(originalSql); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect( host.calls.some( (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), @@ -5208,9 +5088,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await fixture.sql`SELECT state,outcome,settled_at,xmin::text AS xmin FROM data_manager.configuration_fence WHERE fence_id=${prepared.dmFence!.fenceId}::uuid` )[0], ).toEqual(beforeSql); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); expect( host.calls.some( (args) => args.includes("up") || args.includes("pull") || args.includes("restart"), @@ -5244,7 +5122,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); const runtime = await host.reopenRuntime(); app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime, resourceClaimer: host.createResourceClaimer(runtime), journal, @@ -5393,14 +5275,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const core = await import("@openmapx/core/build-artifacts"), originalAuthority = restoreAdmittedControlExecution(installation, prepared.prepare), lease = core.restoreArtifactTargetLeaseForSettlement(originalAuthority), - ownerPath = join( - host.infraDir, - "data/.admin-target-control/leases", - lease.leaseId, - "owner.json", - ), - ownerBytes = readFileSync(ownerPath), - ownerInode = statSync(ownerPath).ino; + ownerRecord = () => JSON.stringify(heldLease(host.infraDir, lease.leaseId)), + ownerBytes = ownerRecord(); core.closeArtifactTargetLeaseHandle(lease); await expect( maintenance.decideConfigurationMaintenance({ @@ -5486,8 +5362,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect( readFileSync(join(executionsPath, `${prepared.prepare.operationKey}.json`)), ).toEqual(rootBytes); - expect(readFileSync(ownerPath)).toEqual(ownerBytes); - expect(statSync(ownerPath).ino).toBe(ownerInode); + expect(ownerRecord()).toEqual(ownerBytes); const [guardAfter] = await fixture.sql`SELECT xmin::text AS version FROM data_manager.configuration_fence WHERE fence_id=${prepared.dmFence!.fenceId}::uuid`; expect(guardAfter?.version).toBe(guardBefore?.version); @@ -5516,6 +5391,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -5562,6 +5438,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -5578,8 +5455,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( } finally { nodeFs.renameSync(hiddenOwner, privateOwner); } - expect(readFileSync(ownerPath)).toEqual(ownerBytes); - expect(statSync(ownerPath).ino).toBe(ownerInode); + expect(ownerRecord()).toEqual(ownerBytes); expect(readdirSync(executionsPath).sort()).toEqual(executionsBefore); } expect(dm!.preparePosts).toBe(1); @@ -5592,9 +5468,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, ); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); }); }, 60000, @@ -5813,7 +5687,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( if (bound) { const child = await dm!.store.lookup(bound.child.execution.operationKey); try { - verifyDataControlTerminalWitness(installation, bound.child.execution, child?.status); + readDataControlTerminalObservation( + installation, + bound.child.execution, + child?.status, + ); witness = "verified"; } catch (failure) { witness = String(failure); @@ -5823,11 +5701,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( phase: "hardlinks", ordinal: 0, }); - const files = attempt?.leaseId - ? readdirSync(join(host.infraDir, "data/.admin-target-control/leases", attempt.leaseId)) - : []; + const held = attempt?.leaseId ? heldLease(host.infraDir, attempt.leaseId) : undefined; throw Error( - `${String(error)}; witness=${String(witness)}; lease files=${JSON.stringify(files)}; DM replies: ${JSON.stringify(await dm?.diagnostics())}`, + `${String(error)}; witness=${String(witness)}; lease=${JSON.stringify(held)}; DM replies: ${JSON.stringify(await dm?.diagnostics())}`, ); }); expect(result).toMatchObject({ applied: true, entries: 1, linked: 1, via: "data-manager" }); @@ -5837,9 +5713,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(host.calls.some((args) => args.includes("up") || args.includes("restart"))).toBe( false, ); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis"))).toBe( - false, - ); + expect(heldTargets(host.infraDir).includes("motis")).toBe(false); }, 60000, ); @@ -5936,9 +5810,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ), ).toBe(true); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); }); const next = await job(); await expect(owned(next, async () => undefined)).rejects.toThrow( @@ -5963,7 +5835,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const bin = join(host.root, "fixture-bin"), trace = join(host.root, "docker-argv.jsonl"), marker = join(host.root, "pull-started"), - receiptDirectory = join(host.infraDir, "data/ops-agent/lifecycle"), + receiptDirectory = join(host.infraDir, "state/ops-agent/lifecycle"), recreated = ["recreate", "crash-recreate", "missing-receipt"].includes(mode), crash = "fs.writeFileSync(" + @@ -6044,10 +5916,8 @@ if(args.includes("--force-recreate")){${mode === "recreate" ? "process.exit(9);" .map((line) => JSON.parse(line) as string[]); expect(calls.filter((args) => args.includes("pull"))).toHaveLength(1); expect(calls.filter((args) => args.includes("up"))).toHaveLength(recreated ? 1 : 0); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/motis"))).toBe( - true, - ); - expect(() => core.verifyRetainedArtifactTargetRootRelease(root)).toThrow(); + expect(heldTargets(host.infraDir).includes("motis")).toBe(true); + await expect(core.verifyRetainedArtifactTargetRootRelease(root)).rejects.toThrow(); if (mode === "missing-receipt") { await expect(receiptRead).rejects.toThrow(); expect(readdirSync(receiptDirectory)).toEqual([]); @@ -6183,7 +6053,7 @@ if(args[0]==="compose"&&args.includes("config")&&args.includes("--format")){impo expect(lifecycleCalls[0]?.slice(-2)).toEqual(["pull", "motis"]); expect(lifecycleCalls[1]?.slice(-4)).toEqual(["up", "-d", "--force-recreate", "motis"]); if (!competing) { - const directory = join(host.infraDir, "data/ops-agent/lifecycle"), + const directory = join(host.infraDir, "state/ops-agent/lifecycle"), [name] = readdirSync(directory); expect(name).toBeDefined(); const later = Date.now() + 30 * 24 * 3600000, @@ -6312,11 +6182,10 @@ if(args[0]==="compose"&&args.includes("config")&&args.includes("--format")){impo ), ).selected, ).toEqual(["data-manager", "motis"]); - expect(existsSync(join(host.infraDir, services.CONFIGURATION_DIRECTORY, ".writer"))).toBe( + expect(existsSync(join(host.infraDir, services.CONFIGURATION_DIRECTORY, "writer.lock"))).toBe( false, ); - for (const id of ["data-manager", "motis"]) - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks", id))).toBe(false); + expect(heldTargets(host.infraDir)).toEqual([]); }, 60000); }, ); diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index a842cdbad..dedcae862 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -19,8 +19,11 @@ import { acquireArtifactTargetLease, assertCoordinatorPullLeaseCompatibility, closeControlInstallation, + heldTargets, + leaseStoreDirectory, lookupControlChildPhase, openControlInstallation, + openLeaseStore, ownedCoordinatorPullBaseline, releaseArtifactTargetLease, restoreAdmittedControlExecution, @@ -427,7 +430,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); agent = buildOpsAgentServer({ operationTimeoutMs: 180000, - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: { claim: async (...args) => { @@ -782,9 +789,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( release.resolve(); } } - async function restartNativeStagedAgent( - options: { stopAfterFirstTargetRelease?: "held" | "unwound" } = {}, - ) { + async function restartNativeStagedAgent() { await agent.close(); const bridge = Fastify({ logger: false, bodyLimit: 64 * 1024 }); bridge.post("/docker", async (request, reply) => { @@ -821,11 +826,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( env: process.env, stdio: ["pipe", "pipe", "pipe"], }), - ready = Promise.withResolvers<{ url: string; ownedDM: boolean }>(), - fault = Promise.withResolvers(); - let observedFault = false, - killedForFault = false, - closed = false; + ready = Promise.withResolvers<{ url: string; ownedDM: boolean }>(); + let closed = false; let output = "", stderr = ""; child.stdout.on("data", (chunk) => { @@ -835,10 +837,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( for (const line of lines) { const event = JSON.parse(line); if (event.event === "ready") ready.resolve(event); - if (event.event === "pre-effect-release-unlinked") { - observedFault = true; - fault.resolve(); - } if (event.event === "operation") { posts.set(event.kind, (posts.get(event.kind) ?? 0) + 1); if (event.status >= 400) @@ -875,9 +873,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( token: host.token, serviceIds: host.loaded.map((service) => service.manifest.id), dockerBridge: `${dockerBridge}/docker`, - ...(options.stopAfterFirstTargetRelease - ? { stopAfterFirstTargetRelease: options.stopAfterFirstTargetRelease } - : {}), }), ); try { @@ -890,26 +885,14 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( await bridge.close(); throw error; } - const killAfterFault = async () => { - await fault.promise; - expect(observedFault).toBe(true); - expect(child.kill("SIGKILL")).toBe(true); - killedForFault = true; - const result = await exited; - expect(result).toEqual({ code: null, signal: "SIGKILL" }); - }; - const stop = async () => { + return async () => { if (closed) return; - if (observedFault && !killedForFault) await killAfterFault(); - if (!killedForFault) child.kill("SIGTERM"); + child.kill("SIGTERM"); const result = await exited; await bridge.close(); closed = true; - expect(result).toEqual( - killedForFault ? { code: null, signal: "SIGKILL" } : { code: 0, signal: null }, - ); + expect(result).toEqual({ code: 0, signal: null }); }; - return Object.assign(stop, { fault: fault.promise, killAfterFault }); } it("reconstructs staged ownership in a NEW native production agent before a fresh user Update", async () => { const { pullPath } = await canceledSuccessfulCoordinatorStage(), @@ -1349,13 +1332,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( "utf8", ), ), - owner = JSON.parse( - readFileSync( - join(host.infraDir, "data/.admin-target-control/leases", lease.leaseId, "owner.json"), - "utf8", - ), - ); - expect(owner.payload.certificate.payload.serviceIds).toContain("data-manager"); + owner = openLeaseStore({ + directory: leaseStoreDirectory(host.infraDir), + purpose: "test", + }).get(lease.leaseId); + expect(owner?.targets).toContain("data-manager"); const request = await api.inject({ method: "POST", url: "/api/admin/services/bulk-action", @@ -1395,8 +1376,18 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( authority = restoreAdmittedControlExecution(installation, actual.execution), snapshot = () => Object.fromEntries( - ["data/.admin-target-control", "control-private/ops-agent/observations"].flatMap( - (base) => { + [ + [ + "leases", + JSON.stringify( + openLeaseStore({ + directory: leaseStoreDirectory(host.infraDir), + purpose: "test", + }).list(), + ), + ], + ].concat( + ["control-private/ops-agent/observations"].flatMap((base) => { const directory = join(host.infraDir, base); return readdirSync(directory, { recursive: true }) .map(String) @@ -1409,12 +1400,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, ]; }); - }, + }), ), ); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/admission-index-v1"))).toBe( - false, - ); const before = snapshot(), writes = vi.spyOn(filesystem, "mkdirSync"); syncBuiltinESMExports(); @@ -1469,33 +1457,33 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceIds: host.loaded.map((service) => service.manifest.id).sort(), }), lease = await acquireArtifactTargetLease({ - dataRoot: join(host.infraDir, "data"), owner: authority, serviceIds: host.loaded.map((service) => service.manifest.id).sort(), }); expect(lease.serviceIds).toContain("data-manager"); let pullObserver: ReturnType | undefined; + const leases = () => + JSON.stringify( + openLeaseStore({ directory: leaseStoreDirectory(host.infraDir), purpose: "test" }).list(), + ); const snapshot = () => Object.fromEntries( - [ - "data/.admin-target-control/locks", - "data/.admin-target-control/leases", - "data/.admin-target-control/admission-index-v1", - "control-private/ops-agent/observations", - ].flatMap((base) => { - const directory = join(host.infraDir, base); - return readdirSync(directory, { recursive: true }) - .map(String) - .sort() - .map((name) => { - const path = join(directory, name), - stat = statSync(path); - return [ - `${base}/${name}`, - `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, - ]; - }); - }), + [["leases", leases()]].concat( + ["control-private/ops-agent/observations"].flatMap((base) => { + const directory = join(host.infraDir, base); + return readdirSync(directory, { recursive: true }) + .map(String) + .sort() + .map((name) => { + const path = join(directory, name), + stat = statSync(path); + return [ + `${base}/${name}`, + `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, + ]; + }); + }), + ), ); // Make a forbidden Docker dispatch conclusively observable without // progressing to the separately held maintenance target. This controls @@ -1546,22 +1534,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( actualPulls[0].payload.execution, ); pullObserver = observer; - const journal = join( - host.infraDir, - "data/.admin-target-control/leases", - lease.leaseId, - ".journal-writer", - ); - mkdirSync(journal, { mode: 0o700 }); - try { - const before = snapshot(), - started = performance.now(); - expect(() => assertCoordinatorPullLeaseCompatibility(observer)).toThrow(); - expect(performance.now() - started).toBeLessThan(5000); - expect(snapshot()).toEqual(before); - } finally { - rmdirSync(journal); - } + const before = snapshot(), + started = performance.now(); + expect(() => assertCoordinatorPullLeaseCompatibility(observer)).toThrow(); + expect(performance.now() - started).toBeLessThan(5000); + expect(snapshot()).toEqual(before); await delay(Math.max(0, deadlineAt - Date.now() + 25)); expect(Date.now()).toBeGreaterThan(deadlineAt); const expired = snapshot(); @@ -1585,21 +1562,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const released = snapshot(); assertCoordinatorPullLeaseCompatibility(pullObserver); expect(snapshot()).toEqual(released); - const slot = join( - host.infraDir, - "data/.admin-target-control/admission-index-v1/active/slot-255", - ); - mkdirSync(slot, { mode: 0o700 }); - writeFileSync(join(slot, "generation.json"), JSON.stringify({ generation: randomUUID() }), { - mode: 0o600, - }); - try { - const partial = snapshot(); - expect(() => assertCoordinatorPullLeaseCompatibility(releasedObserver)).toThrow(); - expect(snapshot()).toEqual(partial); - } finally { - rmSync(slot, { recursive: true }); - } // A second independent component root holds only a certified unrelated // target. Its body/clock/certificate are issued by the real normal host // producer; this is kernel observation, not a native CLI-effect claim. @@ -1641,7 +1603,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceIds: ["motis"], }), unrelatedLease = await acquireArtifactTargetLease({ - dataRoot: join(host.infraDir, "data"), owner: unrelatedAuthority, serviceIds: ["motis"], }); @@ -1828,7 +1789,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(effect.outcome).toBe("failed_without_effect"); const originalRoot = restoreAdmittedControlExecution(installation, admitted.execution); expect(lookupControlChildPhase(originalRoot, { phase: "lifecycle", ordinal: 0 })).toBeNull(); - verifyRetainedArtifactTargetRootRelease(originalRoot); + await verifyRetainedArtifactTargetRootRelease(originalRoot); const repaired = await api.inject({ method: "POST", url: "/api/admin/services/bulk-action", @@ -1906,9 +1867,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ), ).toBeNull(); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(true); release.resolve(); await terminalReached.promise; let state = await client.status("service.update", admitted.operationId); @@ -1921,7 +1880,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(state).toMatchObject({ state: "failed", errorClass: "configuration_prerequisite" }); const authority = restoreAdmittedControlExecution(installation, source.execution); expect(lookupControlChildPhase(authority, { phase: "lifecycle", ordinal: 0 })).toBeNull(); - verifyRetainedArtifactTargetRootRelease(authority); + await verifyRetainedArtifactTargetRootRelease(authority); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, ); @@ -2017,148 +1976,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, 420000, ); - it.each(["held", "unwound"] as const)( - "observes native pre-effect first-unlink SIGKILL with journal guard %s", - async (guard) => { - await canceledSuccessfulCoordinatorStage(); - const first = await restartNativeStagedAgent({ stopAfterFirstTargetRelease: guard }), - { createApiOpsClient } = await import("../ops-client"), - client = createApiOpsClient(), - deadlineAt = Date.now() + 120000, - operation = { - kind: "service.update" as const, - serviceId: "motis", - expectedConfigurationRevisionId: host.initial.revisionId, - }, - operationKey = createControlRootOperationKey({ - notAfter: deadlineAt, - nonce: randomBytes(16), - }), - request = client - .execute(operation, { operationKey, execution: { requestedDeadlineAt: deadlineAt } }) - .catch(() => undefined); - try { - await Promise.race([ - first.fault, - delay(45000).then(() => { - throw Error("Actual native release fault not reached"); - }), - ]); - const privateRoot = join(host.infraDir, "control-private/ops-agent"), - admissionPath = join(privateRoot, "executions", `${operationKey}.json`), - admissionBytes = readFileSync(admissionPath), - source = JSON.parse(admissionBytes.toString("utf8")).payload, - effect = JSON.parse( - readFileSync(join(privateRoot, "effects", `${operationKey}.json`), "utf8"), - ).payload, - terminal = JSON.parse( - readFileSync( - join(privateRoot, "observations", `terminal-${operationKey}.json`), - "utf8", - ), - ).payload; - expect(effect).toMatchObject({ - identity: source.execution, - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/app-api"))).toBe( - false, - ); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); - const leaseRef = JSON.parse( - readFileSync(join(privateRoot, "execution-leases", `${operationKey}.json`), "utf8"), - ), - journalGuard = join( - host.infraDir, - "data/.admin-target-control/leases", - leaseRef.leaseId, - ".journal-writer", - ); - expect(existsSync(journalGuard)).toBe(guard === "held"); - console.info( - "actual native pre-effect kill", - JSON.stringify({ - terminalPhase: terminal.phase, - originalDeadline: source.execution.deadlineAt, - firstTargetAbsent: true, - dmStillHeld: true, - }), - ); - await first.killAfterFault(); - await request; - await first(); - const next = await restartNativeStagedAgent(); - try { - const observed = await createApiOpsClient() - .lookup(operation, operationKey, { - execution: { requestedDeadlineAt: deadlineAt }, - fingerprint: source.execution.fingerprint, - }) - .catch((error: unknown) => { - if ( - guard === "held" && - error instanceof Error && - "errorClass" in error && - error.errorClass === "recovery_required" - ) - return undefined; - throw error; - }); - if (guard === "held") { - if (observed) expect(observed.errorClass).toBe("recovery_required"); - expect(existsSync(journalGuard)).toBe(true); - expect( - existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager")), - ).toBe(true); - expect(readFileSync(admissionPath)).toEqual(admissionBytes); - return; - } - expect(observed).toBeDefined(); - if (!observed) throw Error("Missing original pre-effect failure"); - console.info( - "actual native pre-effect restored", - JSON.stringify({ - state: observed.state, - errorClass: observed.errorClass, - originalDeadlinePreserved: observed.deadlineAt === source.execution.deadlineAt, - }), - ); - expect(observed).toMatchObject({ - state: "failed", - errorClass: "configuration_prerequisite", - operationId: source.execution.executionId, - deadlineAt: source.execution.deadlineAt, - fingerprint: source.execution.fingerprint, - }); - expect(observed.updatedAt).toBe(new Date(terminal.terminalAt).toISOString()); - expect(readFileSync(admissionPath)).toEqual(admissionBytes); - verifyRetainedArtifactTargetRootRelease( - restoreAdmittedControlExecution(installation, source.execution), - ); - const repaired = await api.inject({ - method: "POST", - url: "/api/admin/services/bulk-action", - headers: { - "idempotency-key": randomUUID(), - "x-openmapx-local-admin": "task012-coordinator-local-admin", - }, - payload: { action: "update", serviceIds: ["data-manager"] }, - }); - expect((await finished(repaired.json().jobId)).status).toBe("success"); - } finally { - await next(); - } - } finally { - await first(); - await request; - } - }, - 240000, - ); it("refuses an ordinary unrelated configuration write until explicit guarded DM repair and a NEW request", async () => { await fixture.db .insert(schema.serviceConfig) diff --git a/apps/api/src/services/__tests__/crash-points/driver.ts b/apps/api/src/services/__tests__/crash-points/driver.ts index 4686b9ae6..7f9df3df0 100644 --- a/apps/api/src/services/__tests__/crash-points/driver.ts +++ b/apps/api/src/services/__tests__/crash-points/driver.ts @@ -338,6 +338,8 @@ export class CrashHost { stopRole: async (service) => this.stopRoleProcess(ROLE_OF_SERVICE[service] as Role), roleState: (service) => this.roleState(ROLE_OF_SERVICE[service] as Role), runContainer: (container, request) => this.runContainer(container, request.env), + execRole: (service, container, command) => + this.execRole(ROLE_OF_SERVICE[service] as Role, container, command), }); this.proxies = { "api->agent": new FaultProxy(`http://127.0.0.1:${ports.agent}`), @@ -867,6 +869,34 @@ export class CrashHost { return "running"; } + /** + * `compose exec` in the agent's container: the command runs from the agent's + * bundle with the role's environment and mounts, as `docker exec` would. + */ + private async execRole(role: Role, container: ContainerRecord, command: string[]) { + if (role !== "agent") + return { code: 126, stdout: "", stderr: "crash-point engine execs only in the agent\n" }; + const bundles = this.bundles as RuntimeBundles; + const argv = command.map((value, index) => + index === 0 && value === "node" + ? process.execPath + : value.startsWith("dist/") + ? join(bundles.opsAgent, value.slice("dist/".length)) + : value, + ); + const launched = launchContained({ + name: `exec-${role}-${Date.now()}`, + workDir: this.workDir, + argv, + env: this.roleEnvironment(role, container), + binds: this.binds(container), + ...this.identity(container), + cwd: join(REPO, "apps/ops-agent"), + }); + const { code } = await launched.exited; + return { code: code ?? 137, stdout: launched.stdout(), stderr: launched.stderr() }; + } + /** `docker run` of a release image: its command runs from the matching bundle. */ private runContainer(container: ContainerRecord, callerEnv: Record) { const bundles = this.bundles as RuntimeBundles; diff --git a/apps/api/src/services/__tests__/crash-points/support/engine.ts b/apps/api/src/services/__tests__/crash-points/support/engine.ts index 95ad3bfa0..8a39c74aa 100644 --- a/apps/api/src/services/__tests__/crash-points/support/engine.ts +++ b/apps/api/src/services/__tests__/crash-points/support/engine.ts @@ -121,6 +121,12 @@ export interface EngineHooks { stopRole(service: string): Promise; /** Whether a role service's process is up and answering. */ roleState(service: string): "running" | "starting" | "exited"; + /** `compose exec` of a command in a role service's running container. */ + execRole( + service: string, + container: ContainerRecord, + command: string[], + ): Promise<{ code: number; stdout: string; stderr: string }>; /** `docker run` of an image's command, such as the configuration helper. */ runContainer( container: ContainerRecord, @@ -1039,6 +1045,9 @@ export class FakeDockerEngine { const rest = args.slice(index + 1); if (sub === "config") return this.realCompose([...global, "config", ...rest], request); if (sub === "version") return { code: 0, stdout: "Docker Compose version v2.40.3\n" }; + // A command in a running container changes no container, so it never + // waits behind another caller's Compose effect. + if (sub === "exec") return this.composeExec(global, rest, request); // Mutating compose commands of different callers never interleave, as in // Docker where each container operation is atomic on the daemon. const run = async () => { @@ -1309,6 +1318,11 @@ export class FakeDockerEngine { services.includes(c.Config.Labels["com.docker.compose.service"] as string), ); if (quiet) return { code: 0, stdout: rows.map((c) => `${c.Id}\n`).join("") }; + if (args.includes("--services")) + return { + code: 0, + stdout: rows.map((c) => `${c.Config.Labels["com.docker.compose.service"]}\n`).join(""), + }; const format = flagValue(args, "--format"); const objects = rows.map((c) => ({ ID: c.Id, @@ -1332,6 +1346,27 @@ export class FakeDockerEngine { return { code: 0, stdout: objects.map((o) => `${o.Name} ${o.State}\n`).join("") }; } + /** `compose exec [-T] ` in a role service's running container. */ + private async composeExec(global: string[], args: string[], request: Request): Promise { + let index = 0; + while (args[index] === "-T" || args[index] === "--no-TTY") index++; + const service = args[index]; + const command = args.slice(index + 1); + if (!service || args[index]?.startsWith("-") || command.length === 0) + return this.unsupportedAnswer(["compose", "exec", ...args]); + const { project } = await this.composeModel(global, request); + const container = this.composeContainers(project).find( + (c) => c.Config.Labels["com.docker.compose.service"] === service, + ); + if ( + !container?.State.Running || + !ROLE_SERVICES.has(service) || + this.hooks.roleState(service) !== "running" + ) + return { code: 1, stderr: `service "${service}" is not running\n` }; + return this.hooks.execRole(service, container, command); + } + private servicesOf(args: string[], skipValue: string[]): string[] { const services: string[] = []; for (let i = 0; i < args.length; i++) { diff --git a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts index b9e7f269a..0857c1d42 100644 --- a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts +++ b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts @@ -34,6 +34,26 @@ import { openOpsJobJournal } from "../../../../../apps/ops-agent/src/journal"; import { buildOpsAgentServer } from "../../../../../apps/ops-agent/src/server"; import { releaseProvenance } from "../../../../../packages/core/src/services/__fixtures__/release-provenance"; +/** What the agent records after data-manager's prepare reply; data-manager never writes the store. */ +async function settledByAgent( + lease: control.TargetLeaseAuthority, + child: import("@openmapx/core/ops").ExecutionRef, + receipt: unknown, +) { + await control.settleIssuedLeaseChild(lease, child, { + outcome: "committed", + publication: "committed", + terminalReceiptDigest: control.controlDigest({ kind: "data-configuration-fence", receipt }), + }); +} +/** One delegated child as the agent's lease store records it now. */ +function storeChild(infraDir: string, leaseId: string, delegationId: string) { + return control + .openLeaseStore({ directory: control.leaseStoreDirectory(infraDir), purpose: "test" }) + .get(leaseId) + ?.children.find((child) => child.key === delegationId); +} + // Keep real filesystem checks and writes; expose only the publication boundary // so an acknowledged rename can lose its reply in the recovery tests. vi.mock("node:fs", async (importOriginal) => { @@ -100,7 +120,6 @@ describe.skipIf( try { const fence = createDataConfigurationMaintenance(f.sql, f.dm); await fence.startup(); - await f.adoptPreparationWithoutAdmission(); await f.failPreparation(); const { cancelAdminJobOperations } = await import("../admin-job-ops"), { jobRunner } = await import("../job-runner"), @@ -248,7 +267,7 @@ describe.skipIf( existsSync(join(f.host.queue, ".candidates", `${candidate.sealedOperationKey}.json`)), ).toBe(false); expect(readFileSync(canonicalPath)).toEqual(canonical); - expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); expect(f.data.preparePosts).toBe(0); expect(f.host.calls.some((args) => args.includes("up") || args.includes("restart"))).toBe( false, @@ -285,40 +304,6 @@ describe.skipIf( 90000, ); - it("serializes receiver admission with original-root cancellation and negative settlement", async () => { - const f = await ownedFenceFixture({ productionRegistration: true }), - authority = await import( - "../../../../../packages/core/src/build-artifacts/control-authority" - ), - retain = authority.retainAdmittedControlExecution; - let cancellation: Promise | undefined; - const admission = vi - .spyOn(authority, "retainAdmittedControlExecution") - .mockImplementation((installation, input) => { - if ( - installation.role === "data-manager" && - input.kind === "configuration-maintenance.prepare" && - !cancellation - ) { - cancellation = control.cancelArtifactTargetLease(f.lease); - void cancellation.catch(() => {}); - } - return retain(installation, input); - }); - try { - const fence = createDataConfigurationMaintenance(f.sql, f.dm); - await fence.startup(); - await expect( - fence.prepare(f.input, ["x-openmapx-control-delegation", f.assertion], "ops-agent"), - ).resolves.toMatchObject({ state: "prepared" }); - expect(cancellation).toBeDefined(); - await expect(cancellation!).rejects.toThrow("Lease journal writer is busy"); - } finally { - admission.mockRestore(); - await f.close(); - } - }, 60000); - it("keeps ownership when negative receiver evidence is live, changed, admitted, or unavailable", async () => { const f = await ownedFenceFixture({ productionRegistration: true }); let clock: ReturnType | undefined; @@ -336,10 +321,10 @@ describe.skipIf( try { const fence = createDataConfigurationMaintenance(f.sql, f.dm); await fence.startup(); - await f.adoptPreparationWithoutAdmission(); expect((await request()).status).not.toBe(200); // live original + // Data-manager answers from the expired delegation and its own SQL; it + // never reads the agent's lease store. The agent cancels before asking. clock = vi.spyOn(Date, "now").mockReturnValue(f.owner.deadlineAt + 1); - expect((await request()).status).not.toBe(200); // issuer not frozen control.cancelControlExecution(f.authority); await control.cancelArtifactTargetLease(f.lease); expect((await request(f.input, { authorization: "Bearer wrong" })).status).toBe(401); @@ -347,22 +332,6 @@ describe.skipIf( (await request(f.input, { "x-openmapx-control-delegation": "changed" })).status, ).not.toBe(200); expect((await request({ ...f.input, candidateId: randomUUID() })).status).not.toBe(200); - const delegation = JSON.parse(Buffer.from(f.assertion, "base64url").toString("utf8")), - adoptionFile = join( - f.root, - "control-private/data-manager/adoptions", - `${delegation.delegationId}.json`, - ), - adoption = readFileSync(adoptionFile, "utf8"); - writeFileSync( - adoptionFile, - JSON.stringify({ ...JSON.parse(adoption), state: "pending-adoption" }), - ); - try { - expect((await request()).status).not.toBe(200); - } finally { - writeFileSync(adoptionFile, adoption); - } for (const identity of ["key", "id"] as const) { const id = identity === "id" ? f.child.executionId : randomUUID(), key = @@ -375,8 +344,8 @@ describe.skipIf( } } // A fence row for the same maintenance means the preparation was admitted. - await f.sql`INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,owner_certificate_digest,input,state) - VALUES(${f.input.maintenanceId},${randomUUID()},${`opk1_${randomUUID().replaceAll("-", "")}`},${f.input.candidateId},${f.input.planDigest},${"0".repeat(64)},${JSON.stringify(f.input)}::jsonb,'prepared')`; + await f.sql`INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,owner_lease_id,input,state) + VALUES(${f.input.maintenanceId},${randomUUID()},${`opk1_${randomUUID().replaceAll("-", "")}`},${f.input.candidateId},${f.input.planDigest},${randomUUID()},${JSON.stringify(f.input)}::jsonb,'prepared')`; try { expect((await request()).status).not.toBe(200); } finally { @@ -401,16 +370,9 @@ describe.skipIf( await tx`SELECT pg_advisory_xact_lock(${0x4f4d58},2)`; expect((await request()).status).not.toBe(200); }); - expect(() => control.verifyRetainedArtifactTargetRootRelease(f.authority)).toThrow(); - const assertion = JSON.parse(Buffer.from(f.assertion, "base64url").toString("utf8")), - childFile = join( - f.root, - "data/.admin-target-control/leases", - assertion.leaseId, - "children", - `${assertion.delegationId}.json`, - ); - expect(JSON.parse(readFileSync(childFile, "utf8")).payload.phase).toBe("held"); + await expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).rejects.toThrow(); + const assertion = JSON.parse(Buffer.from(f.assertion, "base64url").toString("utf8")); + expect(storeChild(f.root, assertion.leaseId, assertion.delegationId)?.state).toBe("held"); } finally { clock?.mockRestore(); await f.close(); @@ -426,7 +388,6 @@ describe.skipIf( try { const fence = createDataConfigurationMaintenance(f.sql, f.dm); await fence.startup(); - await f.adoptPreparationWithoutAdmission(); await f.failPreparation(); const { createApiOpsClient } = await import("../ops-client"), client = createApiOpsClient(), @@ -442,7 +403,7 @@ describe.skipIf( pollIntervalMs: 1, }), ).toBe("pending"); - expect(() => control.verifyRetainedArtifactTargetRootRelease(f.authority)).toThrow(); + await expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).rejects.toThrow(); vi.stubEnv("DATA_MANAGER_URL", f.data.baseUrl); const cancel = client.cancel.bind(client), legacy = vi.spyOn(client, "cancel").mockImplementation(async (...args) => { @@ -456,7 +417,7 @@ describe.skipIf( pollIntervalMs: 1, }), ).toBe("pending"); - expect(() => control.verifyRetainedArtifactTargetRootRelease(f.authority)).toThrow(); + await expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).rejects.toThrow(); expect( await cancelAdminJobOperations(f.ctx.checkpointResult, client, { context: f.ctx, @@ -516,7 +477,7 @@ describe.skipIf( expect(finished?.status).toBe("failed"); expect(finished?.finished_at).not.toBeNull(); expect(finished?.result._cancelRequestedAt).toBeDefined(); - expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); expect(await client.status("configurationMaintenance.prepare", f.owner.executionId)).toEqual( original, ); @@ -526,7 +487,8 @@ describe.skipIf( ), ).toBe(controlTerminalReceiptDigest(original)); expect(f.data.preparePosts).toBe(0); - expect(f.data.recoveryPosts).toBe(1); + // Asking data-manager again is read-only and answers the same. + expect(f.data.recoveryPosts).toBeGreaterThanOrEqual(1); expect( await f.sql`SELECT id FROM data_manager.jobs WHERE id=${f.child.executionId}`, ).toHaveLength(0); @@ -559,7 +521,7 @@ describe.skipIf( } }, 60000); - it.each(["child", "owner", "release"] as const)( + it.each(["owner", "release"] as const)( "rejoins only original negative recovery after lost %s publication acknowledgement", async (boundary) => { const f = await ownedFenceFixture({ productionRegistration: true }); @@ -568,7 +530,6 @@ describe.skipIf( try { const fence = createDataConfigurationMaintenance(f.sql, f.dm); await fence.startup(); - await f.adoptPreparationWithoutAdmission(); await f.failPreparation(); const canonicalPath = join( f.root, @@ -576,13 +537,8 @@ describe.skipIf( `terminal-${f.owner.operationKey}.json`, ), canonical = readFileSync(canonicalPath), - assertion = JSON.parse(Buffer.from(f.assertion, "base64url").toString("utf8")), - suffix = - boundary === "child" - ? `${assertion.delegationId}.json` - : boundary === "owner" - ? `${f.input.maintenanceId}.json` - : "admission.json", + // The agent's own owner record, and the lease store it releases the root in. + suffix = boundary === "owner" ? `${f.input.maintenanceId}.json` : "leases.json", nativeRename = (await vi.importActual("node:fs")).renameSync, maintenance = await import( "../../../../../apps/ops-agent/src/configuration-maintenance-store" @@ -600,15 +556,13 @@ describe.skipIf( f.host.runtime, f.authority, ); - // The DM publication can be read from the exact child after its lost - // HTTP reply, so that case may finish without another operator request. - if (boundary === "child") - await expect(recovery).resolves.toMatchObject({ maintenanceId: f.input.maintenanceId }); - else await expect(recovery).rejects.toThrow(); + // A lost write ends this attempt; the retry reuses what was kept and + // asks data-manager no second time. + await expect(recovery).rejects.toThrow(); fault.mockRestore(); expect(faulted).toBe(true); await maintenance.recoverUnadmittedConfigurationMaintenance(f.host.runtime, f.authority); - expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); expect(readFileSync(canonicalPath)).toEqual(canonical); expect(f.data.preparePosts).toBe(0); expect(f.data.recoveryPosts).toBe(1); @@ -624,26 +578,18 @@ describe.skipIf( 60000, ); - it("settles only the expired adopted but unadmitted original preparation through receiver recovery", async () => { + it("settles only the expired unadmitted original preparation through receiver recovery", async () => { const f = await ownedFenceFixture({ productionRegistration: true }); let clock: ReturnType | undefined; try { const fence = createDataConfigurationMaintenance(f.sql, f.dm); await fence.startup(); - await f.adoptPreparationWithoutAdmission(); const assertion = JSON.parse(Buffer.from(f.assertion, "base64url").toString("utf8")), - adoption = join( - f.root, - "control-private/data-manager/adoptions", - `${assertion.delegationId}.json`, - ), - originalAdoption = readFileSync(adoption, "utf8"), execution = join( f.root, "control-private/data-manager/executions", `${f.child.operationKey}.json`, ); - expect(JSON.parse(originalAdoption).state).toBe("ready"); expect(existsSync(execution)).toBe(false); clock = vi.spyOn(Date, "now").mockReturnValue(f.owner.deadlineAt + 1); control.cancelControlExecution(f.authority); @@ -662,29 +608,16 @@ describe.skipIf( ); expect(response.status).toBe(200); const proof = await response.json(); - expect(proof.payload).toMatchObject({ + expect(proof).toMatchObject({ kind: "configuration-prepare-not-admitted", input: f.input, child: f.child, }); - expect(readFileSync(adoption, "utf8")).toBe(originalAdoption); expect(existsSync(execution)).toBe(false); - const child = JSON.parse( - readFileSync( - join( - f.root, - "data/.admin-target-control/leases", - assertion.leaseId, - "children", - `${assertion.delegationId}.json`, - ), - "utf8", - ), + // data-manager answers; the agent settles its child from that answer. + expect(storeChild(f.root, assertion.leaseId, assertion.delegationId)?.state).not.toBe( + "settled", ); - expect(child.payload).toMatchObject({ - phase: "settled", - settlement: { outcome: "failed_without_effect", publication: "not_started" }, - }); expect( await f.sql`SELECT id FROM data_manager.jobs WHERE id=${f.child.executionId}`, ).toHaveLength(0); @@ -694,7 +627,7 @@ describe.skipIf( await expect( maintenance.recoverUnadmittedConfigurationMaintenance(f.host.runtime, f.authority), ).resolves.toMatchObject({ maintenanceId: f.input.maintenanceId }); - expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); } finally { clock?.mockRestore(); await f.close(); @@ -714,7 +647,10 @@ describe.skipIf( "ops-agent", ); expect(receipt.state).toBe("prepared"); - expect(control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child)).toMatchObject({ + await settledByAgent(f.lease, f.child, receipt); + expect( + await control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child), + ).toMatchObject({ outcome: "committed", terminalReceiptDigest: control.controlDigest({ kind: "data-configuration-fence", receipt }), }); @@ -739,7 +675,10 @@ describe.skipIf( "ops-agent", ); expect(receipt.state).toBe("prepared"); - expect(control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child)).toMatchObject({ + await settledByAgent(f.lease, f.child, receipt); + expect( + await control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child), + ).toMatchObject({ outcome: "committed", terminalReceiptDigest: control.controlDigest({ kind: "data-configuration-fence", receipt }), }); @@ -765,7 +704,8 @@ describe.skipIf( expect(rows).toEqual([ { state: "prepared", operation_key: child.operationKey, outcome: null }, ]); - expect(control.inspectArtifactTargetLeaseChildSettlement(lease, child)).toMatchObject({ + await settledByAgent(lease, child, receipt); + expect(await control.inspectArtifactTargetLeaseChildSettlement(lease, child)).toMatchObject({ outcome: "committed", terminalReceiptDigest: control.controlDigest({ kind: "data-configuration-fence", receipt }), }); @@ -963,9 +903,6 @@ describe.skipIf( expect( await fence.prepare(input, ["x-openmapx-control-delegation", assertion], "ops-agent"), ).toEqual(receipt); - expect(control.inspectArtifactTargetLeaseChildSettlement(lease, child)?.publication).toBe( - "committed", - ); await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); } finally { await f.close(); @@ -1046,7 +983,11 @@ async function ownedFenceFixture( runtime: host.runtime, resourceClaimer: host.claimer, controlInstallation: agent, - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, journal: await openOpsJobJournal(join(root, "state/ops-agent/jobs-v1.json")), }), tokenFile = join(host.root, "api-token"); @@ -1247,7 +1188,6 @@ async function ownedFenceFixture( // The original API producer must reach the actual agent-issued child grant. const { input, assertion } = await Promise.race([captured.promise, pending]); const child = control.inspectArtifactDelegation({ - dataRoot: join(root, "data"), assertion, installation: dm, }).execution, @@ -1277,29 +1217,6 @@ async function ownedFenceFixture( ); await preparation.catch(() => {}); }, - /** The receiver's state after a PREPARE stopped between adopting the - * signed lease and admitting its execution, for example because the Data - * Manager was killed there: a ready adoption and no admitted execution. */ - async adoptPreparationWithoutAdmission() { - const rawHeaders = ["x-openmapx-control-delegation", assertion], - dataRoot = join(root, "data"), - peer = control.verifyControlPeer(dm, { - rawHeaders, - transportPrincipal: "ops-agent", - execution: child, - kind: "configuration-maintenance.prepare", - }), - grant = control.inspectArtifactDelegation({ dataRoot, assertion, installation: dm }); - control.closeArtifactTargetLeaseHandle( - await control.adoptArtifactTargetLease({ - dataRoot, - assertion, - peer, - execution: child, - serviceIds: grant.serviceIds, - }), - ); - }, async close() { response.reject(Error("Named fixture HTTP transport closed")); await preparation.catch(() => {}); @@ -1377,7 +1294,7 @@ it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( await f.sql`SELECT * FROM data_manager.configuration_fence WHERE maintenance_id=${f.input.maintenanceId}::uuid`, ).toEqual([]); // The begun leaf has no recorded outcome, so the agent's root stays held. - expect(control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child)).toBeNull(); + expect(await control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.child)).toBeNull(); await expect(control.releaseArtifactTargetLease(f.lease)).rejects.toThrow(); } finally { release(); @@ -1474,7 +1391,6 @@ it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( plan: requests.map((request) => request.slot), }); foreignLease = await control.acquireArtifactTargetLease({ - dataRoot: join(f.root, "data"), serviceIds: ["foreign-consumer"], owner: authority, }); @@ -1492,6 +1408,7 @@ it.skipIf(!process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL)( plan, }, ["x-openmapx-control-delegation", assertion], + "ops-agent", ), ).rejects.toMatchObject({ statusCode: 403 }); expect(await f.sql`SELECT id FROM data_manager.jobs WHERE execution_version=2`).toEqual([]); diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 2f9a7b893..3d64b4b00 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -762,7 +762,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( join(host.infraDir, "state/ops-agent/jobs-v1.json"), ); const app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal, @@ -1916,11 +1920,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); expect(dm.settlePosts).toBe(0); await expect(dm.assertDataAdmission()).rejects.toThrow(); - expect(() => + await expect( verifyRetainedArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), ), - ).toThrow(); + ).rejects.toThrow(); return; } if (version === "launch-native-killed") { @@ -1944,11 +1948,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).toHaveLength(1); expect(dm.settlePosts).toBe(0); await expect(dm.assertDataAdmission()).rejects.toThrow(); - expect(() => + await expect( verifyRetainedArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), ), - ).toThrow(); + ).rejects.toThrow(); const { inspectConfigurationHandoff } = await import( "../../../../ops-agent/src/configuration-handoff-store" ), @@ -2026,7 +2030,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( owner.plan.handoff!.handoffId, ), }; - verifyRetainedArtifactTargetSettlement( + await verifyRetainedArtifactTargetSettlement( actual.installation, { leaseId: actual.intent.leaseId, diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index 41a115405..2ae865417 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -140,7 +140,7 @@ dbDescribe("retained finite maintenance requests", () => { journal, controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); agent.addHook("preHandler", async (request, reply) => { const operation = ( @@ -353,7 +353,7 @@ dbDescribe("retained finite maintenance requests", () => { resourceClaimer: host.createResourceClaimer(runtime), controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); agent.addHook("onSend", async (request, reply, payload) => { const op = ( @@ -516,7 +516,7 @@ dbDescribe("retained finite maintenance requests", () => { resourceClaimer: host.claimer, journal, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); const tokenFile = join(host.root, "render-api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); @@ -625,7 +625,7 @@ dbDescribe("retained finite maintenance requests", () => { resourceClaimer: host.claimer, journal, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); let lost = false; agent.addHook("onSend", async (request, reply, payload) => { @@ -817,7 +817,11 @@ dbDescribe("retained finite maintenance requests", () => { }); const journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); const agent = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal, @@ -1028,7 +1032,11 @@ dbDescribe("retained finite maintenance requests", () => { }), journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); const agent = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime, resourceClaimer: host.claimer, journal, diff --git a/apps/api/src/services/configuration-migration/migration-postgres.test.ts b/apps/api/src/services/configuration-migration/migration-postgres.test.ts index f24a7df76..f58a81620 100644 --- a/apps/api/src/services/configuration-migration/migration-postgres.test.ts +++ b/apps/api/src/services/configuration-migration/migration-postgres.test.ts @@ -105,7 +105,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceIds: host.loaded.map((s) => s.manifest.id), }); agent = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), diff --git a/apps/api/src/services/extension-workflow-postgres.test.ts b/apps/api/src/services/extension-workflow-postgres.test.ts index dce47a31d..a133027be 100644 --- a/apps/api/src/services/extension-workflow-postgres.test.ts +++ b/apps/api/src/services/extension-workflow-postgres.test.ts @@ -83,7 +83,11 @@ beforeAll(async () => { serviceIds: host.loaded.map((s) => s.manifest.id), }); agent = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), diff --git a/apps/api/src/services/integration-application.test.ts b/apps/api/src/services/integration-application.test.ts index 91bce51b7..92e56501a 100644 --- a/apps/api/src/services/integration-application.test.ts +++ b/apps/api/src/services/integration-application.test.ts @@ -128,7 +128,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( serviceIds: host.loaded.map((s) => s.manifest.id), }); agent = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal: await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")), diff --git a/apps/api/src/services/standalone-integrations/install.test.ts b/apps/api/src/services/standalone-integrations/install.test.ts index b27f772c9..9a1477137 100644 --- a/apps/api/src/services/standalone-integrations/install.test.ts +++ b/apps/api/src/services/standalone-integrations/install.test.ts @@ -73,7 +73,11 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( installArtifactTransferRuntime(state.host.runtime, store); installStandaloneIntegrationRuntime(state.host.runtime, { rootDir: state.host.root, store }); agent = buildOpsAgentServer({ - tokens: { api: state.host.token, "data-manager": "b".repeat(43) }, + tokens: { + api: state.host.token, + "data-manager": "b".repeat(43), + "host-cli": "h".repeat(43), + }, runtime: state.host.runtime, resourceClaimer: state.host.claimer, journal: await openOpsJobJournal(join(state.host.infraDir, "state/ops-agent/jobs-v1.json")), diff --git a/apps/api/src/services/transit-feed-credentials-postgres.test.ts b/apps/api/src/services/transit-feed-credentials-postgres.test.ts index ee2a072a8..659c45699 100644 --- a/apps/api/src/services/transit-feed-credentials-postgres.test.ts +++ b/apps/api/src/services/transit-feed-credentials-postgres.test.ts @@ -116,7 +116,11 @@ describe.skipIf(!enabled)( await installPinnedCredentialCatalog(host.root, installation); journal = await openOpsJobJournal(join(host.infraDir, "state/ops-agent/jobs-v1.json")); agent = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal, diff --git a/apps/ops-agent/src/__fixtures__/shutdown.ts b/apps/ops-agent/src/__fixtures__/shutdown.ts index 0ac064b34..3fedee8e3 100644 --- a/apps/ops-agent/src/__fixtures__/shutdown.ts +++ b/apps/ops-agent/src/__fixtures__/shutdown.ts @@ -7,7 +7,7 @@ let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); process.on("message", (message) => { if (message === "release") release(); }); const app = await startOpsAgentServer({ - tokens: { api: "shutdown-fixture-token", "data-manager": "shutdown-dm-token" }, + tokens: { api: "shutdown-fixture-token", "data-manager": "shutdown-dm-token", "host-cli": "h".repeat(43) }, resourcePolicy: createRegistryResourcePolicy({ serviceIds: [], integrationIds: [], trustedData: denyAllTrustedOpsData }), dispatch: async () => { process.send?.({ event: "working" }); diff --git a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts index 08f8c9d6e..90dc9a7eb 100644 --- a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts +++ b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts @@ -71,7 +71,7 @@ async function fixture() { resourceClaimer: host.claimer, controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); let lose = false; app.addHook("onSend", async (request, reply, payload) => { @@ -321,7 +321,7 @@ it("imports through the real registered server and original target root after a fingerprint: status.fingerprint, deadlineAt: status.deadlineAt, }); - verifyRetainedArtifactTargetRootRelease( + await verifyRetainedArtifactTargetRootRelease( restoreAdmittedControlExecution(f.installation, execution), ); expect( @@ -473,7 +473,7 @@ it("reconciles the original prepared publication after exact foreign-inode repai resourceClaimer: f.createResourceClaimer(runtime), controlInstallation: installation, artifactTransfers: store, - tokens: { api: f.token, "data-manager": "d".repeat(43) }, + tokens: { api: f.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); cleanups.push(() => app.close()); const freshClient = new OpsClient({ @@ -500,7 +500,9 @@ it("reconciles the original prepared publication after exact foreign-inode repai fingerprint: status.fingerprint, deadlineAt: status.deadlineAt, }); - verifyRetainedArtifactTargetRootRelease(restoreAdmittedControlExecution(installation, execution)); + await verifyRetainedArtifactTargetRootRelease( + restoreAdmittedControlExecution(installation, execution), + ); expect( journal.records().filter((job) => job.operation.kind === "artifactTransfer.import"), ).toHaveLength(1); @@ -629,7 +631,7 @@ it("observes committed publication after result loss without rewriting its origi fingerprint: original.fingerprint, deadlineAt: original.deadlineAt, }); - verifyRetainedArtifactTargetRootRelease( + await verifyRetainedArtifactTargetRootRelease( restoreAdmittedControlExecution(next.installation, execution), ); expect(readFileSync(join(f.infraDir, "data/osrm-graph/region.osrm"), "utf8")).toBe(files); @@ -676,7 +678,7 @@ async function reassembleCommittedFixture(f: Awaited> resourceClaimer: f.createResourceClaimer(runtime), controlInstallation: installation, artifactTransfers: store, - tokens: { api: f.token, "data-manager": "d".repeat(43) }, + tokens: { api: f.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); cleanups.push(() => app.close()); const freshClient = new OpsClient({ diff --git a/apps/ops-agent/src/artifact-transfer/ingress.test.ts b/apps/ops-agent/src/artifact-transfer/ingress.test.ts index 3ee885c2c..a24a1f2b9 100644 --- a/apps/ops-agent/src/artifact-transfer/ingress.test.ts +++ b/apps/ops-agent/src/artifact-transfer/ingress.test.ts @@ -47,7 +47,7 @@ it.skipIf(process.env.RUN_ARTIFACT_INGRESS_TESTS !== "1")( const runtime = createUnavailableRuntime(); installArtifactTransferRuntime(runtime, store); const agent = buildOpsAgentServer({ - tokens: { api: token, "data-manager": "b".repeat(43) }, + tokens: { api: token, "data-manager": "b".repeat(43), "host-cli": "h".repeat(43) }, runtime, artifactTransfers: store, resourcePolicy: { allowGlobal: () => true }, diff --git a/apps/ops-agent/src/artifact-transfer/motis.test.ts b/apps/ops-agent/src/artifact-transfer/motis.test.ts index e7e3f06c0..ca1072d98 100644 --- a/apps/ops-agent/src/artifact-transfer/motis.test.ts +++ b/apps/ops-agent/src/artifact-transfer/motis.test.ts @@ -462,7 +462,7 @@ it("imports an actual portable candidate through the production agent with desti resourceClaimer: host.claimer, controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); const token = join(host.root, "portable-token"); writeFileSync(token, host.token, { mode: 0o600 }); diff --git a/apps/ops-agent/src/artifact-transfer/store.ts b/apps/ops-agent/src/artifact-transfer/store.ts index f4c29fae1..b602690c6 100644 --- a/apps/ops-agent/src/artifact-transfer/store.ts +++ b/apps/ops-agent/src/artifact-transfer/store.ts @@ -18,13 +18,13 @@ import { artifactFingerprint, assertArtifactTargetLeaseExecution, executionAuthorityState, - inspectArtifactTargetLeaseChildSettlement, + inspectLocalLeaseChildSettlement, lookupControlChildPhaseBinding, parseBundleManifest, readGenerationHardlinksOwnerPlan, type ValidatedBuildBundle, validateBuildBundle, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, } from "@openmapx/core/build-artifacts"; import { type ArtifactFilePublicationResult, @@ -1372,7 +1372,7 @@ export class ArtifactTransferStore { if (plan.reservation) { const binding = lookupControlChildPhaseBinding(authority, plan.reservation), settlement = binding - ? inspectArtifactTargetLeaseChildSettlement(lease, binding.child.execution) + ? inspectLocalLeaseChildSettlement(lease, binding.child.execution) : null; if ( (!binding && result.followUp !== "failed") || @@ -1534,7 +1534,7 @@ export class ArtifactTransferStore { publication.archiveSha256 !== operation.expectedSha256 ) failed(409, "Restored original publication required"); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); const retained = this.lock(() => { const meta = this.owned(operation.stageId, operation.ownerId, "prepared-artifact"), entry = meta.claims.find( diff --git a/apps/ops-agent/src/backup-prune-runtime.test.ts b/apps/ops-agent/src/backup-prune-runtime.test.ts index f486eb0ee..cad726473 100644 --- a/apps/ops-agent/src/backup-prune-runtime.test.ts +++ b/apps/ops-agent/src/backup-prune-runtime.test.ts @@ -74,7 +74,7 @@ it.each([7, 30, 365])( journal, resourceClaimer: host.claimer, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); const tokenFile = join(host.root, "api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); @@ -178,7 +178,7 @@ it("previews and prunes only the unchanged original eligible set through the pro resourceClaimer: host.claimer, journal, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, }); const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }), client = new OpsClient({ baseUrl, tokenFile }); diff --git a/apps/ops-agent/src/bootstrap.test.ts b/apps/ops-agent/src/bootstrap.test.ts index 7ad6317bb..6807dc9ae 100644 --- a/apps/ops-agent/src/bootstrap.test.ts +++ b/apps/ops-agent/src/bootstrap.test.ts @@ -123,7 +123,11 @@ it("production bootstrap retains health and observation when the release store i try { app = await startOpsAgentServer( { - tokens: { api: token, "data-manager": Buffer.alloc(32, 2).toString("base64url") }, + tokens: { + api: token, + "data-manager": Buffer.alloc(32, 2).toString("base64url"), + "host-cli": "h".repeat(43), + }, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, journal: { records: () => [retained], replace: async () => undefined }, dispatch, diff --git a/apps/ops-agent/src/catalog-approval-runtime.ts b/apps/ops-agent/src/catalog-approval-runtime.ts index ae01fa1db..fe34f7d4a 100644 --- a/apps/ops-agent/src/catalog-approval-runtime.ts +++ b/apps/ops-agent/src/catalog-approval-runtime.ts @@ -17,7 +17,6 @@ import { retainedControlParent, verifyControl, verifyControlEffectSettlement, - verifyDataControlTerminalWitness, verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { @@ -46,6 +45,7 @@ import { } from "../../../packages/core/src/build-artifacts/control-authority"; import { verifyDataTransitCandidateForExecution } from "../../../packages/core/src/build-artifacts/data-execution-candidate"; import { AnchoredDirectory } from "../../../packages/core/src/build-artifacts/filesystem"; +import { dataManagerTerminal } from "./data-manager-terminal"; import type { OpsRuntime } from "./runtime"; const installedRoots = new WeakMap(); @@ -186,7 +186,7 @@ export function installCatalogApprovalRuntime( currentValidation.selected.validationDigest !== controlDigest(validation) ) throw Error("Validation was invalidated before slot reuse"); - const validated = verifyDataControlTerminalWitness(installation, validation.sourceExecution); + const validated = await dataManagerTerminal(installation, validation.sourceExecution); if ( validated.status.status !== "ok" || validated.status.result?.kind !== "catalog.validate" || @@ -222,11 +222,7 @@ export function installCatalogApprovalRuntime( activation.promotion.adoption, activation.promotion.execution, ), - grant = inspectArtifactDelegation({ - dataRoot: own.installation.dataRoot, - installation, - assertion, - }); + grant = inspectArtifactDelegation({ installation, assertion }); if ( grant.phase !== "promote-primary" || controlDigest(grant.parent) !== controlDigest(activation.parent) || diff --git a/apps/ops-agent/src/catalog-pins.test.ts b/apps/ops-agent/src/catalog-pins.test.ts index d7acbdb8b..2bad8fc91 100644 --- a/apps/ops-agent/src/catalog-pins.test.ts +++ b/apps/ops-agent/src/catalog-pins.test.ts @@ -52,7 +52,7 @@ it("serializes the largest legal paired pins through the actual HTTP server/clie }); const runtime = createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: root }); const app = buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dmToken }, + tokens: { api: apiToken, "data-manager": dmToken, "host-cli": "h".repeat(43) }, runtime, resourcePolicy: createRegistryResourcePolicy({ serviceIds: ["motis"], diff --git a/apps/ops-agent/src/community-repositories.ts b/apps/ops-agent/src/community-repositories.ts index 913724f26..35f8590c0 100644 --- a/apps/ops-agent/src/community-repositories.ts +++ b/apps/ops-agent/src/community-repositories.ts @@ -21,7 +21,6 @@ import { type GitShallowCloneOptions, gitShallowCloneSnapshot, } from "@openmapx/core/server"; -import type { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; import type { OwnedServiceResolver } from "./owned-services"; import { OpsAgentBusyError, type OpsRuntime } from "./runtime"; import type { TrustedConfigurationAuthoritySnapshot } from "./trusted-config-runtime"; @@ -629,7 +628,6 @@ export function installCommunityRepositoryRuntime( runtime: OpsRuntime, store: CommunityRepositoryStore, owned: OwnedServiceResolver, - lifecycle: ServiceLifecycleExclusion, ): void { runtime["serviceRepository.inspect"] = (operation, context) => store.inspect(operation, context.signal); @@ -645,9 +643,8 @@ export function installCommunityRepositoryRuntime( if (typeof id !== "string") throw new Error("Repository service authority unavailable"); return id; }); - return lifecycle.run(ids, context.signal, () => - store.remove(operation, context.signal, owned, ids), - ); + // The agent holds the lease on every registered service while this runs. + return store.remove(operation, context.signal, owned, ids); }; runtime["communityServices.inspect"] = () => store.list(); } diff --git a/apps/ops-agent/src/configuration-handoff-application.ts b/apps/ops-agent/src/configuration-handoff-application.ts index 86ddb1a09..c0c1a0ef7 100644 --- a/apps/ops-agent/src/configuration-handoff-application.ts +++ b/apps/ops-agent/src/configuration-handoff-application.ts @@ -81,7 +81,7 @@ function ownerFor(child: ConfigurationHandoffChild) { if ( canonicalJson(owner.origin) !== canonicalJson(intent.origin) || canonicalJson(owner.plan.prepare) !== canonicalJson(intent.ownerPrepare) || - owner.rootCertificateDigest !== intent.ownerCertificateDigest || + owner.leaseId !== intent.leaseId || !step || canonicalJson(step.operation) !== canonicalJson(intent.operation) || !source?.grant || diff --git a/apps/ops-agent/src/configuration-handoff-control.test.ts b/apps/ops-agent/src/configuration-handoff-control.test.ts index 704435e25..adee9b428 100644 --- a/apps/ops-agent/src/configuration-handoff-control.test.ts +++ b/apps/ops-agent/src/configuration-handoff-control.test.ts @@ -24,7 +24,7 @@ vi.mock("node:timers/promises", async (original) => ({ new Promise((resolve) => setTimeout(resolve, milliseconds)), })); -import { ArtifactTargetJournalBusyError } from "@openmapx/core/build-artifacts"; +import { LeaseStoreBusyError } from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey } from "@openmapx/core/ops"; import { closeConfigurationHandoffChild, @@ -83,7 +83,7 @@ afterEach(() => vi.useRealTimers()); it("retries only a nominal pre-adoption refusal with the original identity", async () => { const f = fixture(); mock.adoptArtifactTargetLease.mockRejectedValueOnce( - new ArtifactTargetJournalBusyError(Error("held journal guard")), + new LeaseStoreBusyError("/state/leases.lock", undefined), ); const pending = openConfigurationHandoffChild(f.input); void pending.catch(() => {}); @@ -109,7 +109,7 @@ it("retries only a nominal pre-adoption refusal with the original identity", asy it("stops at the five-second contention budget without adopting or retaining authority", async () => { const f = fixture(), - error = new ArtifactTargetJournalBusyError(Error("persistent guard")); + error = new LeaseStoreBusyError("/state/leases.lock", undefined); mock.adoptArtifactTargetLease.mockRejectedValue(error); const failed = expect(openConfigurationHandoffChild(f.input)).rejects.toBe(error); await vi.advanceTimersByTimeAsync(5050); @@ -122,7 +122,7 @@ it("stops at the five-second contention budget without adopting or retaining aut it("stops at the original deadline before the five-second wait budget", async () => { const f = fixture(), - error = new ArtifactTargetJournalBusyError(Error("held guard")); + error = new LeaseStoreBusyError("/state/leases.lock", undefined); f.input.execution.deadlineAt = Date.now() + 20; mock.adoptArtifactTargetLease.mockRejectedValue(error); const failed = expect(openConfigurationHandoffChild(f.input)).rejects.toBe(error); diff --git a/apps/ops-agent/src/configuration-handoff-control.ts b/apps/ops-agent/src/configuration-handoff-control.ts index adef31260..872590ab6 100644 --- a/apps/ops-agent/src/configuration-handoff-control.ts +++ b/apps/ops-agent/src/configuration-handoff-control.ts @@ -4,7 +4,6 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { - ArtifactTargetJournalBusyError, adoptArtifactTargetLease, assertArtifactTargetLeaseExecution, assertConfigurationHandoffHostSources as assertHostSources, @@ -16,6 +15,7 @@ import { closeControlInstallation, controlDigest, executionAuthorityState, + LeaseStoreBusyError, openFixedChildInstallation, readConfigurationHandoffSpawnIntent, readPrivateRecord, @@ -50,7 +50,7 @@ const children = new WeakMap< } >(); function sourceReadBusy(error: unknown): boolean { - return error instanceof ArtifactTargetJournalBusyError; + return error instanceof LeaseStoreBusyError; } /** Retry only the caller's certified pre-write contention. The original * deadline remains authoritative; no lock is removed or stolen. */ @@ -139,14 +139,13 @@ export async function openConfigurationHandoffChild(input: { execution, () => adoptArtifactTargetLease({ - dataRoot: join(infraDir, "data"), assertion: input.assertion, peer, execution, serviceIds: intent.operation.serviceIds, }), // Any other error may follow partial adoption and is never retried. - (error) => error instanceof ArtifactTargetJournalBusyError, + (error) => error instanceof LeaseStoreBusyError, ); const authority = retainAdmittedControlExecution(installation, { execution, diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index 70cf710c9..48d43d53a 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -195,14 +195,13 @@ async function prepareConfigurationHandoffLaunch( ) throw Error("Original retained helper grant required"); const observed = inspectArtifactDelegation({ - dataRoot: local.installation.dataRoot, installation: local.installation.installation, assertion: attempt.assertion, }); if ( bytes(observed.execution) !== bytes(bound.execution) || observed.recipient !== "configuration-helper" || - observed.ownerCertificateDigest !== owned.owner.rootCertificateDigest || + observed.leaseId !== owned.owner.leaseId || bytes(observed.owner) !== bytes(owned.owner.plan.prepare) ) throw Error("Original helper target ownership changed"); @@ -233,7 +232,6 @@ async function prepareConfigurationHandoffLaunch( sourceDigest, environmentNames, preparerAdmissionDigest: local.receiptDigest, - ownerCertificateDigest: owned.owner.rootCertificateDigest, leaseId: observed.leaseId, delegationId: observed.delegationId, assertion: attempt.assertion, diff --git a/apps/ops-agent/src/configuration-handoff.test.ts b/apps/ops-agent/src/configuration-handoff.test.ts index 8ed88e1e7..8ae2d009b 100644 --- a/apps/ops-agent/src/configuration-handoff.test.ts +++ b/apps/ops-agent/src/configuration-handoff.test.ts @@ -391,7 +391,7 @@ it("uses actual server job state when excluding a live log observer from drain", release = Promise.withResolvers(), checked = Promise.withResolvers(); const app = buildOpsAgentServer({ - tokens: { api, "data-manager": dm }, + tokens: { api, "data-manager": dm, "host-cli": "h".repeat(43) }, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, dispatch: async (operation, _runtime, context) => { if (operation.kind === "service.logs.follow") { diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index ede13432a..985c9fd1d 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -76,7 +76,7 @@ import { executionAuthorityState, type FixedConfigurationMaintenancePlan, inspectArtifactDelegation, - inspectArtifactTargetLeaseChildSettlement, + inspectLocalLeaseChildSettlement, inspectRetainedOwnedArtifactTargetChildSettlement, issueControlChildPhase, lookupControlChildPhaseBinding, @@ -94,8 +94,8 @@ import { verifyControl, verifyControlEffectSettlement, verifyControlRecoveryAuthority, + verifyLocalArtifactTargetRootRelease, verifyMaintenanceCoordinatorPrerequisite, - verifyRetainedArtifactTargetRootRelease, verifyRetainedLocalControlPeer, withConfigurationMaintenanceDecisionPublication, withConfigurationMaintenanceOwnerInitialization, @@ -103,11 +103,15 @@ import { withConfigurationMaintenancePhasePublication, writePrivateRecord, } from "@openmapx/core/build-artifacts"; +import { + type ConfigurationPrepareNonAdmission, + configurationPrepareNonAdmissionSchema, +} from "@openmapx/core/data-manager-contracts"; import { DataManagerHttpError, inspectDataConfigurationFence, prepareDataConfigurationFence, - recoverUnadmittedConfigurationPreparation, + recoverUnadmittedDataConfigurationFence, settleDataConfigurationFence, } from "@openmapx/core/data-manager-sdk"; import { @@ -169,15 +173,16 @@ import { childSettlementState, configurationMaintenanceChildState, installationState, - publishConfigurationMaintenanceForwardWitness, readCompletedConfigurationHandoff, verifyConfigurationMaintenanceChildTerminal, verifyControlPreparedTerminal, withPrivateEffectWriter, } from "../../../packages/core/src/build-artifacts/control-authority"; import { + inspectLocalLeaseChildSettlement as inspectOwnedChildSettlement, observeConfigurationMaintenanceRuntimeTree, - readConfigurationPrepareNonAdmission, + restoreArtifactTargetLeaseForSettlement as restoreOwnedLease, + settleIssuedLeaseChild, } from "../../../packages/core/src/build-artifacts/target-lease"; import type { DataConfigurationFenceReceipt } from "../../../packages/core/src/services/data-manager/contracts/configuration-maintenance"; @@ -324,6 +329,15 @@ function assertOwner( throw Error("Original coordinator pull/image expectation changed"); } else if (owner.coordinatorPull) throw Error("Unexpected coordinator pull observation"); } +/** The data-manager's fence row for this maintenance, asked with the agent's own token. */ +function inspectOwnedFence(owner: ConfigurationMaintenanceOwnerRecord, signal?: AbortSignal) { + return inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { + transport: configurationDataManagerTransport(), + candidateId: owner.plan.operation.candidateId, + ownerLeaseId: owner.leaseId, + signal, + }); +} function read(authority: VerifiedExecutionAuthority): ConfigurationMaintenanceOwnerRecord { const state = executionAuthorityState(authority), name = ownerName(authority), @@ -372,7 +386,7 @@ export function retainOwnedConfigurationMaintenance( installationId: state.installation.installation.installationId, origin: state.origin, admissionDigest: state.receiptDigest, - rootCertificateDigest: root.certificateDigest, + leaseId: root.leaseId, apiParentAttestationDigest: plan.apiParent.assertionDigest, plan, originalContainers: source.originalContainers, @@ -409,7 +423,6 @@ export function retainOwnedConfigurationMaintenance( withConfigurationMaintenanceOwnerInitialization(lease, authority, () => write(authority, owner, true), ); - publishConfigurationMaintenanceForwardWitness(authority); } export function readOwnedConfigurationMaintenance( authority: VerifiedExecutionAuthority, @@ -671,7 +684,6 @@ export async function prepareOwnedConfigurationMaintenanceStepAdmission( throw new ConfigurationMaintenanceControlUnauthorized( "Uninstalled frozen maintenance continuation", ); - publishConfigurationMaintenanceForwardWitness(authority); const lease = restoreConfigurationMaintenancePhaseIssuer(authority, step.phase); try { if (lookupControlChildPhaseBinding(authority, reservation)) @@ -758,11 +770,7 @@ export async function prepareOwnedConfigurationMaintenanceStepAdmission( throw new ConfigurationMaintenanceControlConflict( "Original attempted local phase observation required", ); - const inspected = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - installation, - assertion: attempted.assertion, - }), + const inspected = inspectArtifactDelegation({ installation, assertion: attempted.assertion }), roleParent = verifiedParentForArtifactDelegation( installation, attempted.assertion, @@ -796,7 +804,6 @@ export async function prepareOwnedConfigurationMaintenanceStepAdmission( kind: operation.kind, }); return adoptArtifactTargetLease({ - dataRoot: local.dataRoot, assertion, peer, execution, @@ -1534,7 +1541,7 @@ export function completeConfigurationMaintenanceSettleControl( function completeOwnedConfigurationMaintenanceSettleControl( authority: VerifiedExecutionAuthority, ): ConfigurationMaintenanceReceipt { - const released = verifyRetainedArtifactTargetRootRelease(authority); + const released = verifyLocalArtifactTargetRootRelease(authority); restoreOwnedMaintenanceFinal(authority); assertConfigurationMaintenanceRootClosure(authority); return withPrivateEffectWriter(executionAuthorityState(authority), () => { @@ -1580,7 +1587,7 @@ export function inspectConfigurationMaintenanceSettleControl( throw Error("Original settle canonical source required"); restoreOwnedMaintenanceFinal(source.authority); assertConfigurationMaintenanceRootClosure(source.authority); - if (verifyRetainedArtifactTargetRootRelease(source.authority).releasedAt !== retained.completedAt) + if (verifyLocalArtifactTargetRootRelease(source.authority).releasedAt !== retained.completedAt) throw Error("Original settle completion time changed"); return { receipt: { ...owner.receipt, ...retained.result }, @@ -1659,7 +1666,7 @@ export function recordOwnedConfigurationMaintenanceStepTerminal( proof = configurationMaintenanceChildState(verified, authority), lease = restoreArtifactTargetLeaseForSettlement(authority); try { - const physical = inspectArtifactTargetLeaseChildSettlement(lease, execution); + const physical = inspectLocalLeaseChildSettlement(lease, execution); const handoff = owner.plan.steps[source.ordinal]?.phase === "maintenance-handoff"; if (handoff) verifyControlPreparedTerminal(local, execution); else if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) @@ -1780,7 +1787,7 @@ export function restoreConfigurationMaintenanceControlJournal( if (source.operation.kind === "configurationMaintenance.settle" && !unresolved) { restoreOwnedMaintenanceFinal(authority); assertConfigurationMaintenanceRootClosure(authority); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); } if (keys.has(source.execution.operationKey) || ids.has(source.execution.executionId)) throw new ConfigurationMaintenanceControlConflict("Duplicate original control identity"); @@ -1891,7 +1898,7 @@ export function recordOwnedConfigurationMaintenancePrepared( child: ExecutionRef, receipt: DataConfigurationFenceReceipt, ): void { - const settled = inspectArtifactTargetLeaseChildSettlement(lease, child); + const settled = inspectLocalLeaseChildSettlement(lease, child); if ( settled?.outcome !== "committed" || settled.terminalReceiptDigest !== controlDigest({ kind: "data-configuration-fence", receipt }) @@ -2143,7 +2150,7 @@ export function restoreOwnedMaintenanceFinal( actual.identity.maintenanceId !== owner.plan.operation.maintenanceId || actual.identity.candidateId !== owner.plan.operation.candidateId || actual.identity.planDigest !== owner.plan.planDigest || - actual.identity.rootCertificateDigest !== owner.rootCertificateDigest || + actual.identity.leaseId !== owner.leaseId || canonicalJson(evidence.runtime) !== canonicalJson(inspection.runtime) || inspection.runtime.requiredCheckSetDigest !== finalRecordDigest(domains.inspection, inspection.requiredChecks) || @@ -2193,7 +2200,7 @@ export async function releaseOwnedConfigurationMaintenanceRoot( restoreOwnedMaintenanceFinal(authority); assertConfigurationMaintenanceRootClosure(authority); if (original.receipt.state === "settled") { - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); return original.receipt; } // Physical release precedes this owner's metadata publication. A lost owner @@ -2202,7 +2209,7 @@ export async function releaseOwnedConfigurationMaintenanceRoot( // original settlement-only reconciliation path, which verifies its sources. let physicallyReleased = false; try { - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); physicallyReleased = true; } catch { /* A held/partial original release is reconciled below. */ @@ -2212,13 +2219,12 @@ export async function releaseOwnedConfigurationMaintenanceRoot( leaseId = lease.leaseId; closeArtifactTargetLeaseHandle(lease); const released = await reconcileArtifactTargetLease({ - dataRoot: state.installation.dataRoot, leaseId, authority: verifyControlRecoveryAuthority(authority), }); if (released !== "released") throw Error("Original prepared maintenance root remains fenced"); } - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); assertConfigurationMaintenanceRootClosure(authority); return withPrivateEffectWriter(state, () => { const current = read(authority); @@ -2236,9 +2242,29 @@ export async function releaseOwnedConfigurationMaintenanceRoot( }); } -/** Recovers an expired preparation the data-manager never admitted: the - * data-manager signs that observation, then the never-prepared root releases. */ -export async function recoverUnadmittedConfigurationMaintenance( +const unadmittedRecoveries = new Map< + string, + ReturnType +>(); +/** + * Recovers an expired preparation the data-manager never admitted: the + * data-manager answers that it never admitted it, then the never-prepared root + * releases. Concurrent cancellation polls share one recovery. + */ +export function recoverUnadmittedConfigurationMaintenance( + runtime: OpsRuntime, + authority: VerifiedExecutionAuthority, +) { + const key = executionAuthorityState(authority).execution.operationKey; + const running = unadmittedRecoveries.get(key); + if (running) return running; + const next = recoverUnadmittedConfigurationMaintenanceOnce(runtime, authority).finally(() => + unadmittedRecoveries.delete(key), + ); + unadmittedRecoveries.set(key, next); + return next; +} +async function recoverUnadmittedConfigurationMaintenanceOnce( runtime: OpsRuntime, authority: VerifiedExecutionAuthority, ) { @@ -2255,8 +2281,8 @@ export async function recoverUnadmittedConfigurationMaintenance( ) throw Error("Original expired never-applied preparation required"); try { - verifyRetainedArtifactTargetRootRelease(authority); - assertConfigurationMaintenanceRootClosure(authority); + // The store releases a maintenance root only after its closure check. + verifyLocalArtifactTargetRootRelease(authority); return configurationMaintenanceRecoverySource(authority); } catch { /* Only original held/partially released recovery follows. */ @@ -2283,33 +2309,42 @@ export async function recoverUnadmittedConfigurationMaintenance( // Cancellation closes forward issuance first. A lost response can only // repeat this same negative receiver observation, never PREPARE. cancelControlExecution(authority); - let negative: ReturnType | undefined; + await cancelArtifactTargetLease(lease); + // The data-manager answers that it never admitted the expired preparation + // and can no longer admit it; that answer settles the leaf without effect. + // A retry after a later step failed reuses the answer it kept. + let negative: ConfigurationPrepareNonAdmission; try { - negative = readConfigurationPrepareNonAdmission(authority); - } catch { - /* First recovery. */ - } - if (!negative) { - await cancelArtifactTargetLease(lease); - await recoverUnadmittedConfigurationPreparation(owner.plan.dmPrepare, { - transport: configurationDataManagerTransport(), - headers: { "x-openmapx-control-delegation": attempt.assertion }, - }); - negative = readConfigurationPrepareNonAdmission(authority); + negative = readFenceNonAdmission(authority, owner); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + negative = retainFenceNonAdmission( + authority, + owner, + await recoverUnadmittedDataConfigurationFence(owner.plan.dmPrepare, { + transport: configurationDataManagerTransport(), + controlAssertion: attempt.assertion, + }), + ); } - if (canonicalJson(negative.payload.input) !== canonicalJson(owner.plan.dmPrepare)) + if ( + canonicalJson(negative.input) !== canonicalJson(owner.plan.dmPrepare) || + canonicalJson(negative.child) !== canonicalJson(attempt.execution) + ) throw Error("Original preparation recovery body changed"); + await settleIssuedLeaseChild(lease, attempt.execution, { + outcome: "failed_without_effect", + publication: "not_started", + terminalReceiptDigest: controlDigest(negative), + }); retainConfigurationFenceNotPrepared(authority); if ( (await reconcileArtifactTargetLease({ - dataRoot: state.installation.dataRoot, leaseId: lease.leaseId, authority: verifyControlRecoveryAuthority(authority), })) !== "released" ) throw Error("Original unadmitted maintenance remains fenced"); - assertConfigurationMaintenanceRootClosure(authority); - verifyRetainedArtifactTargetRootRelease(authority); return configurationMaintenanceRecoverySource(authority); } finally { closeArtifactTargetLeaseHandle(lease); @@ -2320,23 +2355,45 @@ export async function recoverUnadmittedConfigurationMaintenance( * the data-manager settled its fence child with, its signed observation that * the prepare was never admitted. */ function configurationMaintenanceRecoverySource(authority: VerifiedExecutionAuthority) { - assertConfigurationMaintenanceRootClosure(authority); - const released = verifyRetainedArtifactTargetRootRelease(authority), + const released = verifyLocalArtifactTargetRootRelease(authority), owner = read(authority), - fence = owner.plan.reservations.find((value) => value.phase === "configuration-fence"), - binding = fence && lookupControlChildPhaseBinding(authority, fence), - settled = binding - ? inspectRetainedOwnedArtifactTargetChildSettlement(authority, binding.child.execution) - : null; - if (settled?.outcome !== "failed_without_effect") - throw Error("Data-manager non-admission required"); + negative = readFenceNonAdmission(authority, owner); return { maintenanceId: owner.plan.operation.maintenanceId, candidateId: owner.plan.operation.candidateId, - receiverEvidenceDigest: settled.terminalReceiptDigest, + receiverEvidenceDigest: controlDigest(negative), releasedAt: released.releasedAt, }; } +const fenceNonAdmissionName = (owner: ConfigurationMaintenanceOwnerRecord) => + `fence-not-admitted-${owner.plan.operation.maintenanceId}.json`; +/** The data-manager's answer that it never admitted the preparation, kept once. */ +function retainFenceNonAdmission( + authority: VerifiedExecutionAuthority, + owner: ConfigurationMaintenanceOwnerRecord, + negative: ConfigurationPrepareNonAdmission, +): ConfigurationPrepareNonAdmission { + const local = executionAuthorityState(authority).installation; + try { + return readFenceNonAdmission(authority, owner); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + writePrivateRecord(local, "observations", fenceNonAdmissionName(owner), negative, true); + return negative; +} +function readFenceNonAdmission( + authority: VerifiedExecutionAuthority, + owner: ConfigurationMaintenanceOwnerRecord, +): ConfigurationPrepareNonAdmission { + return configurationPrepareNonAdmissionSchema.parse( + readPrivateRecord( + executionAuthorityState(authority).installation, + "observations", + fenceNonAdmissionName(owner), + ), + ); +} /** Records that this maintenance never prepared a fence, so its root can * release: the fence child was never issued, or the data-manager settled it * without effect. Anything else stays unknown and keeps the root held. */ @@ -2578,7 +2635,7 @@ export function configurationHandoffPendingMaintenanceIds( const authority = restoreAdmittedControlExecution(installation, owner.plan.prepare); restoreOwnedMaintenanceFinal(authority); assertConfigurationMaintenanceRootClosure(authority); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); return false; } catch { return true; @@ -2664,24 +2721,7 @@ export async function prepareMaintenanceContinueObservation( source.candidateSource, owner.plan.operation.candidateId, ), - issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), - }, - ), - actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal, - }); + actual = await inspectOwnedFence(owner, signal); if ( canonicalJson(actual) !== canonicalJson(owner.preparedFence) || controlDigest(candidate.record) !== owner.plan.candidateRecordDigest || @@ -2750,24 +2790,7 @@ export async function verifyFrozenMaintenanceGenerationSource( canonicalJson(fresh.originalContainers) !== canonicalJson(owner.originalContainers) ) throw Error("Frozen maintenance generation source drifted"); - const issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), - }, - ), - actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal, - }); + const actual = await inspectOwnedFence(owner, signal); if ( canonicalJson(actual) !== canonicalJson(owner.preparedFence) || canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) @@ -2806,7 +2829,7 @@ export async function verifyFrozenMaintenanceHardlinksSource( proof = configurationMaintenanceChildState(verified, authority), lease = restoreArtifactTargetLeaseForSettlement(authority); try { - const physical = inspectArtifactTargetLeaseChildSettlement(lease, generation.execution); + const physical = inspectLocalLeaseChildSettlement(lease, generation.execution); if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) throw Error("Original generation child settlement required"); } finally { @@ -2823,24 +2846,7 @@ export async function verifyFrozenMaintenanceHardlinksSource( if (containers.find((value) => value.serviceId === "data-manager")?.state !== "running") throw Error("Original maintenance coordinator stopped"); await assertConfigurationCoordinatorReady(signal); - const issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), - }, - ), - actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal, - }); + const actual = await inspectOwnedFence(owner, signal); if ( canonicalJson(actual) !== canonicalJson(owner.preparedFence) || canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) || @@ -2909,7 +2915,7 @@ async function verifyFrozenMaintenanceFollowUpSource( ), authority, ), - physical = inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + physical = inspectLocalLeaseChildSettlement(lease, child.execution); if (!physical || canonicalJson(physical) !== canonicalJson(proof.settlement)) throw Error("Original recreation prerequisite settlement required"); } @@ -2925,24 +2931,7 @@ async function verifyFrozenMaintenanceFollowUpSource( throw Error("Original recreation coordinator stopped"); await assertConfigurationCoordinatorReady(signal); } - const issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: Math.min(state.execution.deadlineAt, issuedAt + 15000), - }, - ), - actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal, - }); + const actual = await inspectOwnedFence(owner, signal); if ( canonicalJson(actual) !== canonicalJson(owner.preparedFence) || canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) || @@ -3065,7 +3054,7 @@ export async function inspectFrozenMaintenanceRuntime( }; if ( canonicalJson(original.intent.ownerPrepare) !== canonicalJson(owner.plan.prepare) || - original.intent.ownerCertificateDigest !== owner.rootCertificateDigest || + original.intent.leaseId !== owner.leaseId || original.intent.operation.planDigest !== owner.plan.planDigest || canonicalJson(original.intent.release) !== canonicalJson(owner.plan.release) || published?.publication !== "published" || @@ -3094,24 +3083,7 @@ export async function inspectFrozenMaintenanceRuntime( signal, }); if (owner.plan.hardlinkPlan.plan.length) await assertConfigurationCoordinatorReady(signal); - const issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: issuedAt + 15000, - }, - ), - guard = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal, - }); + const guard = await inspectOwnedFence(owner, signal); if ( canonicalJson(guard) !== canonicalJson(owner.preparedFence) || canonicalJson(containers) !== canonicalJson(await inspectContainers()) || @@ -3131,7 +3103,7 @@ export async function inspectFrozenMaintenanceRuntime( origin: owner.origin, ownerPrepare: owner.plan.prepare, admissionDigest: owner.admissionDigest, - rootCertificateDigest: owner.rootCertificateDigest, + leaseId: owner.leaseId, maintenanceId: owner.plan.operation.maintenanceId, candidateId: owner.plan.operation.candidateId, planDigest: owner.plan.planDigest, @@ -3222,7 +3194,7 @@ export async function inspectFrozenMaintenanceRuntime( origin: owner.origin, ownerPrepare: owner.plan.prepare, admissionDigest: owner.admissionDigest, - rootCertificateDigest: owner.rootCertificateDigest, + leaseId: owner.leaseId, maintenanceId: owner.plan.operation.maintenanceId, candidateId: owner.plan.operation.candidateId, planDigest: owner.plan.planDigest, @@ -3253,8 +3225,7 @@ export async function inspectFrozenMaintenanceRuntime( const fence = owner.plan.reservations.find((value) => value.phase === "configuration-fence"), fenceBinding = fence && lookupControlChildPhaseBinding(authority, fence), physical = - fenceBinding && - inspectArtifactTargetLeaseChildSettlement(lease, fenceBinding.child.execution); + fenceBinding && inspectLocalLeaseChildSettlement(lease, fenceBinding.child.execution); if ( physical?.outcome !== "committed" || physical.publication !== "committed" || @@ -3320,24 +3291,7 @@ export async function inspectFrozenMaintenanceRuntime( canonicalJson(containers) !== canonicalJson(owner.originalContainers) ) throw Error("Unused runtime differs from original owned generation/containers"); - const issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: issuedAt + 15000, - }, - ), - guard = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal, - }); + const guard = await inspectOwnedFence(owner, signal); if ( canonicalJson(guard) !== canonicalJson(owner.preparedFence) || canonicalJson(readOwnedConfigurationMaintenance(authority)) !== canonicalJson(owner) @@ -3570,12 +3524,12 @@ export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, sour } if (owner.receipt.state === "settled" && !owner.preparedFence) { assertConfigurationMaintenanceRootClosure(authority); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); } if (owner.receipt.state === "settled" && owner.preparedFence) { restoreOwnedMaintenanceFinal(authority); assertConfigurationMaintenanceRootClosure(authority); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); } let preflight: "current" | "drifted" | "unavailable" = "unavailable"; try { @@ -3598,24 +3552,7 @@ export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, sour : "drifted"; if (owner.receipt.dmFence) { if (!owner.preparedFence) throw Error("Original prepared guard source unavailable"); - const issuedAt = Date.now(), - witness = signControl( - state.installation.privateKey, - "openmapx-configuration-fence-inspect-v1", - { - installationId: owner.installationId, - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - ownerCertificateDigest: owner.rootCertificateDigest, - issuedAt, - expiresAt: Math.min(context.execution.deadlineAt, issuedAt + 15000), - }, - ), - actual = await inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { - transport: configurationDataManagerTransport(), - witness: Buffer.from(canonicalJson(witness)).toString("base64url"), - signal: context.signal, - }); + const actual = await inspectOwnedFence(owner, context.signal); if (canonicalJson(actual) !== canonicalJson(owner.receipt.dmFence)) throw Error("Original prepared guard source changed"); } @@ -3678,16 +3615,44 @@ export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, sour // The data-manager prepares idempotently: a lost reply repeats the same // body for a short while. A data-manager that stays unreachable leaves // the maintenance held for recovery. - const receipt = await retryDataManager( - context.signal, - Math.min(context.execution.deadlineAt, Date.now() + 15_000), - () => - prepareDataConfigurationFence(plan.dmPrepare, { - transport: configurationDataManagerTransport(), - controlAssertion: assertion, - signal: context.signal, - }), - ); + let receipt: Awaited>; + try { + receipt = await retryDataManager( + context.signal, + Math.min(context.execution.deadlineAt, Date.now() + 15_000), + () => + prepareDataConfigurationFence(plan.dmPrepare, { + transport: configurationDataManagerTransport(), + controlAssertion: assertion, + signal: context.signal, + }), + ); + } catch (error) { + // A busy refusal wrote nothing: the fence leaf failed without effect. + if ( + error instanceof DataManagerHttpError && + error.status === 409 && + error.message.includes("configuration_maintenance_busy") + ) + await settleIssuedLeaseChild(lease, child.execution, { + outcome: "failed_without_effect", + publication: "not_started", + terminalReceiptDigest: controlDigest({ + kind: "data-configuration-fence-refused", + maintenanceId: plan.operation.maintenanceId, + reason: "busy", + }), + }); + throw error; + } + await settleIssuedLeaseChild(lease, child.execution, { + outcome: "committed", + publication: "committed", + terminalReceiptDigest: controlDigest({ + kind: "data-configuration-fence", + receipt: { ...receipt, sequence: 0, state: "prepared" }, + }), + }); recordOwnedConfigurationMaintenancePrepared( authority, lease, diff --git a/apps/ops-agent/src/configuration-writer-concurrency.test.ts b/apps/ops-agent/src/configuration-writer-concurrency.test.ts index 9ddeb98aa..50321981c 100644 --- a/apps/ops-agent/src/configuration-writer-concurrency.test.ts +++ b/apps/ops-agent/src/configuration-writer-concurrency.test.ts @@ -1,5 +1,13 @@ import { type ChildProcess, fork } from "node:child_process"; -import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; @@ -213,23 +221,17 @@ describe("overlapping configuration writers", () => { outcome: "committed", }); }, 20_000); - it("does not steal a killed holder's ownership or allow a timed-out contender to write later", async () => { + it("reclaims the lock of a killed holder and commits the next writer's generation", async () => { const root = fixture(); const infra = join(root, "infra/docker"); const a = start(root, "ops", "a", "", "before"); await a.wait("held"); a.child.kill("SIGKILL"); await new Promise((resolve) => a.child.once("exit", () => resolve())); + // The dead holder's lock file stays until the next writer proves it gone. + expect(existsSync(coreServices.configurationWriterLockPath(infra))).toBe(true); const b = start(root, "ops", "b"); - expect((await b.wait("done")).result).toMatchObject({ - outcome: "not_committed", - reason: "busy", - }); - expect(coreServices.currentConfigurationGeneration(infra)).toBeNull(); - expect( - readdirSync(join(infra, coreServices.CONFIGURATION_GENERATIONS)).some((n) => - n.endsWith(".tmp"), - ), - ).toBe(true); + expect((await b.wait("done")).result).toMatchObject({ outcome: "committed" }); + expect(coreServices.currentConfigurationGeneration(infra)).not.toBeNull(); }, 20_000); }); diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts index 1810b3332..3b2131b9d 100644 --- a/apps/ops-agent/src/control-admission.test.ts +++ b/apps/ops-agent/src/control-admission.test.ts @@ -28,7 +28,7 @@ it("refuses paired approval as a bearer-only root before reserving a worker", as const app = buildOpsAgentServer({ controlInstallation: installation, journal, - tokens: { api: "a".repeat(32), "data-manager": "b".repeat(32) }, + tokens: { api: "a".repeat(32), "data-manager": "b".repeat(32), "host-cli": "h".repeat(43) }, resourcePolicy: { allowGlobal: () => true }, }); try { @@ -135,7 +135,6 @@ it.each(["normal", "ready-failure", "wrong-target", "canceled", "cancel-journal- plan: [plan], }); const lease = await control.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds, owner: authority, }); @@ -152,7 +151,7 @@ it.each(["normal", "ready-failure", "wrong-target", "canceled", "cancel-journal- const token = "b".repeat(32); const journalFile = coreServices.agentStatePaths(root).journal; const options = { - tokens: { api: "a".repeat(32), "data-manager": token }, + tokens: { api: "a".repeat(32), "data-manager": token, "host-cli": "h".repeat(43) }, controlInstallation: agent, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, dispatch: async (_operation: unknown, _runtime: unknown, context: OpsExecutionContext) => { diff --git a/apps/ops-agent/src/control-terminal-restart.test.ts b/apps/ops-agent/src/control-terminal-restart.test.ts index 4ae7516b0..cc9a6d206 100644 --- a/apps/ops-agent/src/control-terminal-restart.test.ts +++ b/apps/ops-agent/src/control-terminal-restart.test.ts @@ -166,7 +166,7 @@ it.each( }); const journal = await openOpsJobJournal(file); app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": "b".repeat(32) }, + tokens: { api: token, "data-manager": "b".repeat(32), "host-cli": "h".repeat(43) }, controlInstallation: installation, journal, dispatch, @@ -310,6 +310,7 @@ it.each( tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32), + "host-cli": "h".repeat(43), }, controlInstallation: installation, journal, @@ -334,6 +335,7 @@ it.each( tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32), + "host-cli": "h".repeat(43), }, controlInstallation: installation, journal: unproved, @@ -476,7 +478,6 @@ it("repairs a borrowed terminal worker after the actual canonical-digest restart plan: [plan], }); const lease = await c.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -498,7 +499,11 @@ it("repairs a borrowed terminal worker after the actual canonical-digest restart }); const journal = await openOpsJobJournal(coreServices.agentStatePaths(root).journal); app = buildOpsAgentServer({ - tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32) }, + tokens: { + api: Buffer.alloc(32, 7).toString("base64url"), + "data-manager": "b".repeat(32), + "host-cli": "h".repeat(43), + }, controlInstallation: installation, journal, dispatch, @@ -506,7 +511,7 @@ it("repairs a borrowed terminal worker after the actual canonical-digest restart }); await app.ready(); expect(journal.records()).toEqual([stopped.job]); - expect(existsSync(join(root, "data/.admin-target-control/locks/motis"))).toBe(true); + expect(c.heldTargets(root).includes("motis")).toBe(true); c.retainControlEffectReceipt(authority, { outcome: "failed_without_effect", publication: "not_started", diff --git a/apps/ops-agent/src/core-service-runtime.test.ts b/apps/ops-agent/src/core-service-runtime.test.ts index cf79c8eef..daae15a5c 100644 --- a/apps/ops-agent/src/core-service-runtime.test.ts +++ b/apps/ops-agent/src/core-service-runtime.test.ts @@ -83,7 +83,7 @@ describe("production core restart workflow", () => { }); const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": "d".repeat(32) }, + tokens: { api: host.token, "data-manager": "d".repeat(32), "host-cli": "h".repeat(43) }, runtime, resourceClaimer: host.claimer, journal, diff --git a/apps/ops-agent/src/data-manager-terminal.ts b/apps/ops-agent/src/data-manager-terminal.ts new file mode 100644 index 000000000..f6c6d5a8a --- /dev/null +++ b/apps/ops-agent/src/data-manager-terminal.ts @@ -0,0 +1,32 @@ +import { + type ControlInstallation, + readDataControlTerminalObservation, + recordDataControlTerminalObservation, +} from "@openmapx/core/build-artifacts"; +import { lookupDataExecution, unwrapDataManagerResponse } from "@openmapx/core/data-manager-sdk"; +import type { ExecutionRef } from "@openmapx/core/ops"; +import { configurationDataManagerTransport } from "./configuration-hardlinks-runtime"; + +/** + * The terminal status of a data-manager execution, as data-manager reports it + * to the agent over its authenticated route. The first terminal answer is + * recorded in the agent's own state and every later read returns that record. + */ +export async function dataManagerTerminal( + installation: ControlInstallation, + execution: ExecutionRef, + signal?: AbortSignal, +) { + try { + return readDataControlTerminalObservation(installation, execution); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const status = unwrapDataManagerResponse( + await lookupDataExecution(execution.operationKey, { + transport: configurationDataManagerTransport(), + signal, + }), + ); + return recordDataControlTerminalObservation(installation, execution, status); +} diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts index 50d80433a..ca4e42c3f 100644 --- a/apps/ops-agent/src/data-transit-resource-source.test.ts +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -111,7 +111,6 @@ async function fixture( reservations: [slot], }), lease = await control.acquireArtifactTargetLease({ - dataRoot: data, serviceIds: ["motis-feed-proxy"], owner: authority, }); @@ -121,6 +120,7 @@ async function fixture( ...slot, lease, childPublicKey: agent.publicKey, + evidence: control.publishedDataTransitCandidate(authority, slot), }); const claimer = createProductionRegistryResourceClaimer({ services: [{ serviceId: "motis-feed-proxy", isBuiltIn: true, enabled: true }], @@ -178,17 +178,27 @@ async function fixture( ...(admissionPolicy === "late" ? { now: () => new Date(Date.now() + 120000) } : {}), controlInstallation: agent, resourceClaimer: claimer, - tokens: { api: "a".repeat(43), "data-manager": token }, + tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, }); await app.listen({ host: "127.0.0.1", port: 0 }); const address = app.server.address(); if (!address || typeof address === "string") throw Error("Missing actual agent address"); const client = new OpsClient({ baseUrl: `http://127.0.0.1:${address.port}`, tokenFile }), - published = join( - data, - ".admin-target-control/executions/data-manager", - `${slot.execution.operationKey}.data-candidate.json`, - ); + /** Changes the candidate data-manager recorded with the child's grant. */ + published = ( + change: (evidence: Record) => Record | undefined, + ) => + control + .openLeaseStore({ directory: control.leaseStoreDirectory(root), purpose: "test" }) + .update((leases) => { + for (const held of leases) + for (const child of held.children) + if (child.execution.operationKey === slot.execution.operationKey && child.evidence) { + const next = change(structuredClone(child.evidence)); + if (next) child.evidence = next; + else delete child.evidence; + } + }); return { root, data, @@ -245,12 +255,6 @@ it.each(["validation", "validation-replaced", "transport", "reload", "marker"] a ) .toMatchObject({ outcome: "contained", publication: "not_started" }); expect(control.inspectFeedProxyValidationFailure(authority)).toMatch(/^[a-f0-9]{64}$/); - await expect - .poll( - () => control.verifyFeedProxyCanonicalTerminal(f.authority, f.bound.execution).state, - { timeout: 15000 }, - ) - .toBe("failed"); const raw = JSON.parse( readFileSync( join( @@ -323,7 +327,7 @@ it.each(["validation", "validation-replaced", "transport", "reload", "marker"] a ).toThrow(); } else { expect( - control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.bound.execution), + await control.inspectArtifactTargetLeaseChildSettlement(f.lease, f.bound.execution), ).toBeNull(); expect(control.inspectFeedProxyValidationFailure(authority)).toBeNull(); expect(() => @@ -382,15 +386,16 @@ it.each([ try { let key = f.slot.execution.operationKey, operation = f.operation; - if (mode === "missing") rmSync(f.published); - if (mode === "forged" || mode === "foreign-target" || mode === "foreign-origin") { - const raw = JSON.parse(readFileSync(f.published, "utf8")); - if (mode === "forged") raw.signature = "a".repeat(86); - else if (mode === "foreign-target") raw.payload.serviceIds = ["motis"]; - else raw.payload.origin.notAfter++; - writeFileSync(f.published, JSON.stringify(raw), { mode: 0o600 }); - } - if (mode === "foreign-root" || mode === "foreign-phase") { + if (mode === "missing") await f.published(() => undefined); + if (mode === "forged" || mode === "foreign-target" || mode === "foreign-origin") + await f.published((evidence) => { + if (mode === "forged") return { ...evidence, version: 3 }; + if (mode === "foreign-target") return { ...evidence, serviceIds: ["motis"] }; + const origin = evidence.origin as { notAfter: number }; + return { ...evidence, origin: { ...origin, notAfter: origin.notAfter + 1 } }; + }); + // A candidate belongs to the child whose grant carries it; another key has none. + if (mode === "foreign-root" || mode === "foreign-phase") key = control.controlChildOperationKey( mode === "foreign-root" ? createControlRootOperationKey({ @@ -401,15 +406,6 @@ it.each([ mode === "foreign-phase" ? "proxy-commit" : "proxy-stage", 0, ); - copyFileSync( - f.published, - join( - f.data, - ".admin-target-control/executions/data-manager", - `${key}.data-candidate.json`, - ), - ); - } if (mode === "foreign-body") operation = { ...operation, candidateId: "ctrl-" + "b".repeat(32) }; if (mode === "inode") { diff --git a/apps/ops-agent/src/data-workflow-runtime.test.ts b/apps/ops-agent/src/data-workflow-runtime.test.ts index e5641370f..af984a688 100644 --- a/apps/ops-agent/src/data-workflow-runtime.test.ts +++ b/apps/ops-agent/src/data-workflow-runtime.test.ts @@ -63,14 +63,11 @@ it("freezes distinct original data/finalization slots and refuses targets before expect(control.readDataUpdatePlan(authority)).toEqual(plan); await expect( control.acquireArtifactTargetLease({ - dataRoot: join(host.infraDir, "data"), owner: authority, serviceIds: ids, }), ).rejects.toThrow("data phase not admitted"); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager"))).toBe( - false, - ); + expect(control.heldTargets(host.infraDir).includes("data-manager")).toBe(false); const child = await control.bindControlChildPhase(authority, plan.dataReservation), key = control.prepareFixedControlChild(authority, plan.dataDelegationId); control.retainFixedCliSpawnIntent(authority, { @@ -90,14 +87,11 @@ it("freezes distinct original data/finalization slots and refuses targets before // An issued original CLI is still not a completed data prerequisite. await expect( control.acquireArtifactTargetLease({ - dataRoot: join(host.infraDir, "data"), owner: authority, serviceIds: ids, }), ).rejects.toThrow(); - expect(existsSync(join(host.infraDir, "data/.admin-target-control/locks/data-manager"))).toBe( - false, - ); + expect(control.heldTargets(host.infraDir).includes("data-manager")).toBe(false); } finally { control.closeControlInstallation(installation); } diff --git a/apps/ops-agent/src/data-workflow-runtime.ts b/apps/ops-agent/src/data-workflow-runtime.ts index 2abf8641a..f51634b29 100644 --- a/apps/ops-agent/src/data-workflow-runtime.ts +++ b/apps/ops-agent/src/data-workflow-runtime.ts @@ -137,7 +137,7 @@ export function installDataWorkflowRuntime( ); control.verifyControlEffectSettlement(installation, bound.execution); if (lease) { - const settled = control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution); + const settled = control.inspectLocalLeaseChildSettlement(lease, bound.execution); if (settled?.outcome !== "committed" || settled.publication !== "committed") throw new OpsExecutionFailure("recovery_required", { dataWorkflow: receipt }); } @@ -152,7 +152,6 @@ export function installDataWorkflowRuntime( const initial = await control.verifyDataUpdatePrerequisites(authority); await assertStagedCoordinatorConfigurationWrite(runtime, operation, context, context.signal); const lease = await control.acquireArtifactTargetLease({ - dataRoot: join(infraDir, "data"), serviceIds: plan.dataReservation.serviceIds, owner: authority, signal: context.signal, diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 574b88cff..306a3121c 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -605,7 +605,6 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { ) => { const remaining = context.execution.deadlineAt - Date.now(); if (remaining <= 0 || context.signal.aborted) throw new OpsExecutionFailure("timeout", {}); - context.beginServiceCreation?.(); if (options.execFile) await run(args, context.signal, Math.min(timeout, remaining)); else await runContainedStreamingProcess("docker", args, { diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index 6ff11317b..b0af492e7 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -1,6 +1,11 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; -import { loadControlStartup } from "@openmapx/core/build-artifacts"; +import { + leaseStoreDirectory, + loadControlStartup, + localLeaseBackend, + openLeaseStore, +} from "@openmapx/core/build-artifacts"; import { initializeManagedCatalogPins } from "@openmapx/core/catalog-pins"; import { readOpsTokenFile } from "@openmapx/core/ops"; import { repoPaths, services } from "@openmapx/core/server"; @@ -144,12 +149,20 @@ async function main(): Promise { await services.withConfigurationGenerationWriter(paths.infraDir, async () => services.prepareAgentStateDirectories(paths.infraDir), ); + // Every lease lives in the agent's own state; the agent applies its rules + // to its own operations and on behalf of data-manager and the host CLI. + const leaseStore = openLeaseStore({ + directory: leaseStoreDirectory(paths.infraDir), + purpose: "ops-agent", + }); const controlStartup = loadControlStartup({ infraDir: paths.infraDir, role: "ops-agent", serviceIds: [...builtInServices, ...communityServices].map((s) => s.manifest.id), container: existsSync("/run/secrets/openmapx-control-verifiers"), + leases: localLeaseBackend(leaseStore), }); + const hostCliToken = services.ensureHostCliToken(paths.infraDir); if (controlStartup.reason) console.error(controlStartup.reason); if (controlStartup.installation) { try { @@ -272,7 +285,8 @@ async function main(): Promise { const initialPolicyAuthority = await policyAuthority(); await startOpsAgentServer( { - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": hostCliToken }, + leaseStore, resourceClaimer: createProductionRegistryResourceClaimer({ services: initialPolicyAuthority.services, integrationIds: initialPolicyAuthority.integrationIds, diff --git a/apps/ops-agent/src/ingress-rate-limit.test.ts b/apps/ops-agent/src/ingress-rate-limit.test.ts index 1eb6274e2..3002e0b78 100644 --- a/apps/ops-agent/src/ingress-rate-limit.test.ts +++ b/apps/ops-agent/src/ingress-rate-limit.test.ts @@ -91,6 +91,7 @@ it("shares the actual server ingress budget across operations and artifact route return "a".repeat(43); }, "data-manager": "b".repeat(43), + "host-cli": "h".repeat(43), }, artifactTransfers: store, }); diff --git a/apps/ops-agent/src/integration-store.test.ts b/apps/ops-agent/src/integration-store.test.ts index 442167a62..740bb86ad 100644 --- a/apps/ops-agent/src/integration-store.test.ts +++ b/apps/ops-agent/src/integration-store.test.ts @@ -14,6 +14,7 @@ import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { closeControlInstallation, + heldTargets, openControlInstallation, provisionControlInstallation, } from "@openmapx/core/build-artifacts"; @@ -293,7 +294,11 @@ describe("integration store", () => { const runtime = createUnavailableRuntime(); installIntegrationRuntime(runtime, f.store); const app = buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": Buffer.alloc(32, 32).toString("base64url") }, + tokens: { + api: apiToken, + "data-manager": Buffer.alloc(32, 32).toString("base64url"), + "host-cli": "h".repeat(43), + }, resourcePolicy: allowAllResources, journal, controlInstallation: installation, @@ -381,7 +386,11 @@ describe("integration store", () => { const runtime = createUnavailableRuntime(); installIntegrationRuntime(runtime, f.store); const app = buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": Buffer.alloc(32, 32).toString("base64url") }, + tokens: { + api: apiToken, + "data-manager": Buffer.alloc(32, 32).toString("base64url"), + "host-cli": "h".repeat(43), + }, resourcePolicy: allowAllResources, journal, controlInstallation: installation, @@ -425,7 +434,7 @@ describe("integration store", () => { ); }); expect(readdirSync(privateRoot, { recursive: true }).sort()).toEqual(before); - expect(existsSync(join(infraDir, "data/.admin-target-control"))).toBe(false); + expect(heldTargets(infraDir)).toEqual([]); expect(record(f, "weather-radar")).toBeUndefined(); } finally { await app.close(); diff --git a/apps/ops-agent/src/journal.ts b/apps/ops-agent/src/journal.ts index 97d9abe9a..e1256ba91 100644 --- a/apps/ops-agent/src/journal.ts +++ b/apps/ops-agent/src/journal.ts @@ -60,7 +60,8 @@ export interface PersistedOpsJob { failureResult?: OpsFailureResult; /** Private checkpoint, never a terminal outcome or authority. */ progressResult?: OpsFailureResult; - role: OpsRole; + /** Only the API and data-manager submit journaled operations. */ + role: Exclude; operation: OpsOperation; operationId: string; operationKey: string; diff --git a/apps/ops-agent/src/lease-operations.test.ts b/apps/ops-agent/src/lease-operations.test.ts new file mode 100644 index 000000000..eb973bf40 --- /dev/null +++ b/apps/ops-agent/src/lease-operations.test.ts @@ -0,0 +1,131 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type ControlInstallation, + closeControlInstallation, + leaseStoreDirectory, + openControlInstallation, + openLeaseStore, + provisionControlInstallation, +} from "@openmapx/core/build-artifacts"; +import type { ExecutionRef } from "@openmapx/core/ops"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { LeaseRefused, runLeaseOperation } from "./lease-operations"; + +let root: string; +let installation: ControlInstallation; +beforeEach(async () => { + root = mkdtempSync(join(tmpdir(), "omx-lease-operations-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await provisionControlInstallation(root); + installation = openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["osrm", "redis", "community-live"], + }); +}); +afterEach(() => { + closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); +}); +const store = () => openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }); +function owner(executor: "data-manager" | "host-cli"): ExecutionRef { + return { + executor, + principal: executor, + executionId: randomUUID(), + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 60_000, + }; +} +const acquire = (role: "data-manager" | "host-cli", execution: ExecutionRef, targets: string[]) => + runLeaseOperation(installation, store(), role, { + kind: "lease.acquire", + owner: execution, + ownerKind: role === "host-cli" ? "service.restart" : "transit-sync", + targets, + }); + +/** The lease an acquire answered with. */ +function leaseOf(result: unknown): string { + const value = result as { outcome?: string; lease?: { leaseId: string } }; + if (value.outcome !== "acquired" || !value.lease) throw Error("Expected a lease"); + return value.lease.leaseId; +} + +describe("lease operations for data-manager and the host CLI", () => { + it("acquires for the caller's own execution on registered targets, community ones included", async () => { + const result = await acquire("host-cli", owner("host-cli"), ["community-live"]); + expect(result).toMatchObject({ outcome: "acquired", lease: { targets: ["community-live"] } }); + await expect(acquire("host-cli", owner("data-manager"), ["osrm"])).rejects.toBeInstanceOf( + LeaseRefused, + ); + await expect( + acquire("data-manager", owner("data-manager"), ["unknown"]), + ).rejects.toBeInstanceOf(LeaseRefused); + }); + + it("answers busy with the holder instead of failing", async () => { + const holder = owner("data-manager"); + await acquire("data-manager", holder, ["osrm"]); + expect(await acquire("host-cli", owner("host-cli"), ["osrm"])).toMatchObject({ + outcome: "busy", + target: "osrm", + holder: { owner: holder, ownerKind: "transit-sync", state: "held" }, + }); + }); + + it("shows and changes only the caller's own leases", async () => { + const leaseId = leaseOf(await acquire("data-manager", owner("data-manager"), ["redis"])); + expect( + await runLeaseOperation(installation, store(), "host-cli", { kind: "lease.read", leaseId }), + ).toEqual({ lease: null }); + expect( + await runLeaseOperation(installation, store(), "host-cli", { + kind: "lease.update", + leaseId, + action: { action: "release" }, + }), + ).toEqual({ lease: null }); + expect(store().get(leaseId)).toBeDefined(); + await expect( + runLeaseOperation(installation, store(), "api", { kind: "lease.read", leaseId }), + ).rejects.toBeInstanceOf(LeaseRefused); + }); + + it("lets the host CLI delegate only to data-manager, from the lease's own owner", async () => { + const cli = owner("host-cli"); + const leaseId = leaseOf( + await runLeaseOperation(installation, store(), "host-cli", { + kind: "lease.acquire", + owner: cli, + ownerKind: "data.update", + targets: ["osrm"], + }), + ); + const child = { + key: randomUUID(), + parentKey: null, + parent: cli, + execution: { ...owner("data-manager"), deadlineAt: cli.deadlineAt }, + kind: "download-osm", + phase: "osm", + recipient: "ops-agent" as const, + targets: ["osrm"], + permittedFurtherEdges: [], + depth: 1, + }; + await expect( + runLeaseOperation(installation, store(), "host-cli", { + kind: "lease.update", + leaseId, + action: { action: "delegate", child }, + }), + ).rejects.toThrow(); + expect(store().get(leaseId)?.children).toEqual([]); + }); +}); diff --git a/apps/ops-agent/src/lease-operations.ts b/apps/ops-agent/src/lease-operations.ts new file mode 100644 index 000000000..b4c97765e --- /dev/null +++ b/apps/ops-agent/src/lease-operations.ts @@ -0,0 +1,121 @@ +import { + type ControlInstallation, + controlRegisteredServiceIds, + type LeaseRecord, + type LeaseStore, + leaseRules, + TargetBusyError, + verifyFiniteControlEdge, +} from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + type OpsOperation, + type OpsResultFor, + type OpsRole, +} from "@openmapx/core/ops"; +import { childPhaseKinds } from "../../../packages/core/src/build-artifacts/control-child-records"; + +type LeaseOperation = Extract< + OpsOperation, + { kind: "lease.acquire" | "lease.read" | "lease.update" } +>; +const recipients: Partial> = { + // Data-manager's children are the agent's own operations; the host CLI's are + // data-manager executions. + "data-manager": "ops-agent", + "host-cli": "data-manager", +}; + +/** + * Leases data-manager and the host CLI hold through the agent. Each caller + * sees and changes only leases its own executions own, and the same rules + * apply as to the agent's leases: one admission path, children recorded + * before they start, release only once everything has a known outcome. + */ +export async function runLeaseOperation( + installation: ControlInstallation, + store: LeaseStore, + role: OpsRole, + operation: LeaseOperation, +): Promise> { + if (role !== "data-manager" && role !== "host-cli") throw new LeaseRefused(); + const owned = (lease: LeaseRecord | undefined) => + lease && lease.owner.executor === role ? lease : undefined; + const now = Date.now(); + if (operation.kind === "lease.acquire") { + const registered = new Set(controlRegisteredServiceIds(installation)); + if ( + operation.owner.executor !== role || + operation.targets.some((target) => !registered.has(target)) + ) + throw new LeaseRefused(); + try { + const lease = await store.update((leases) => leaseRules.acquire(leases, operation, now)); + return { outcome: "acquired", lease }; + } catch (error) { + if (!(error instanceof TargetBusyError)) throw error; + const { leaseId, owner, ownerKind, state } = error.holder; + return { + outcome: "busy", + target: error.target, + holder: { leaseId, owner, ownerKind, state }, + }; + } + } + if (operation.kind === "lease.read") + return { lease: owned(store.get(operation.leaseId)) ?? null }; + const lease = await store.update((leases) => { + const current = owned(leases.find((value) => value.leaseId === operation.leaseId)); + if (!current) { + if (operation.action.action === "release" || operation.action.action === "cancel") + return undefined; + throw new LeaseRefused(); + } + const action = operation.action; + switch (action.action) { + case "delegate": { + const child = action.child; + const edges = verifyFiniteControlEdge( + role, + current.ownerKind, + child.phase as never, + child.recipient, + ); + if ( + child.parentKey !== null || + child.depth !== 1 || + child.recipient !== recipients[role] || + canonicalJson(child.parent) !== canonicalJson(current.owner) || + canonicalJson(child.permittedFurtherEdges) !== canonicalJson(edges) || + !(childPhaseKinds as Record)[child.phase]?.includes( + child.kind, + ) || + child.execution.deadlineAt > current.owner.deadlineAt + ) + throw new LeaseRefused(); + return leaseRules.delegate(leases, { leaseId: current.leaseId, child }, now); + } + case "settle": { + if (action.key !== null) { + const child = current.children.find((value) => value.key === action.key); + // A caller settles only a child it issued whose recipient cannot reach the store. + if (child?.parentKey !== null || child.recipient !== "data-manager") + throw new LeaseRefused(); + } + return leaseRules.settle(leases, { leaseId: current.leaseId, ...action }); + } + case "cancel": + return leaseRules.cancel(leases, { leaseId: current.leaseId, key: action.key }); + case "release": + leaseRules.release(leases, { leaseId: current.leaseId }); + return undefined; + } + }); + return { lease: lease ?? null }; +} + +export class LeaseRefused extends Error { + constructor() { + super("Lease operation is not permitted"); + } +} diff --git a/apps/ops-agent/src/lifecycle-exclusion.test.ts b/apps/ops-agent/src/lifecycle-exclusion.test.ts deleted file mode 100644 index 79617a5f0..000000000 --- a/apps/ops-agent/src/lifecycle-exclusion.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { - chmodSync, - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { services } from "@openmapx/core/server"; -import { afterEach, expect, it } from "vitest"; -import { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; - -const roots: string[] = []; -afterEach(() => { - for (const root of roots.splice(0)) { - chmodSync(root, 0o700); - rmSync(root, { recursive: true, force: true }); - } -}); -function fixture() { - const infra = mkdtempSync(join(tmpdir(), "ops-lifecycle-state-")); - roots.push(infra); - const state = services.agentStatePaths(infra).directory; - mkdirSync(state, { recursive: true, mode: 0o700 }); - return { infra, state }; -} -it("retains an unresolved lifecycle fence through restart below writable state while infrastructure is read-only", async () => { - const { infra, state } = fixture(); - chmodSync(infra, 0o555); - const lifecycle = new ServiceLifecycleExclusion(infra); - await expect( - lifecycle.creating(["alpha"], new AbortController().signal, async (begin) => { - begin(); - throw Error("unknown external completion"); - }), - ).rejects.toThrow("unknown external completion"); - const pending = join(state, "service-lifecycle/pending.json"); - expect(readFileSync(pending, "utf8")).toBe('["alpha"]'); - expect(statSync(pending).mode & 0o777).toBe(0o600); - expect(existsSync(join(infra, ".service-lifecycle"))).toBe(false); - const restored = new ServiceLifecycleExclusion(infra); - await expect( - restored.run(["alpha"], new AbortController().signal, async () => "effect"), - ).rejects.toThrow("settlement is unresolved"); - await expect( - restored.run(["beta"], new AbortController().signal, async () => "unrelated"), - ).resolves.toBe("unrelated"); - chmodSync(pending, 0o644); - expect(() => new ServiceLifecycleExclusion(infra)).toThrow("settlement is unresolved"); -}); -it("refuses legacy lifecycle state intact instead of resetting an older unresolved fence", () => { - const { infra } = fixture(); - const legacy = join(infra, ".service-lifecycle"); - mkdirSync(legacy, { mode: 0o700 }); - const pending = join(legacy, "pending.json"); - writeFileSync(pending, '["alpha"]', { mode: 0o600 }); - const before = statSync(pending); - expect(() => new ServiceLifecycleExclusion(infra)).toThrow("settlement is unresolved"); - expect(readFileSync(pending, "utf8")).toBe('["alpha"]'); - expect(statSync(pending).ino).toBe(before.ino); -}); diff --git a/apps/ops-agent/src/lifecycle-exclusion.ts b/apps/ops-agent/src/lifecycle-exclusion.ts deleted file mode 100644 index 7b41e97d9..000000000 --- a/apps/ops-agent/src/lifecycle-exclusion.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { - closeSync, - constants, - fstatSync, - fsyncSync, - lstatSync, - mkdirSync, - openSync, - readFileSync, - renameSync, - rmSync, - writeFileSync, -} from "node:fs"; -import { join } from "node:path"; -import { services } from "@openmapx/core/server"; -import { isContainedProcessFailure } from "./docker-runtime"; - -interface Waiter { - ids: string[]; - signal: AbortSignal; - grant(release: () => void): void; - abort(): void; -} -const FAILED = "Service lifecycle settlement is unresolved; host reconciliation required"; -const ID = /^[a-z0-9][a-z0-9-]{0,63}$/; - -/** Auxiliary serialization/denial, never a substitute for existing root/target authority. - * One instance is shared by the actual production lifecycle and repository adapters. - * Pending effects are durable denials across restart; no TTL or PID can clear them. */ -export class ServiceLifecycleExclusion { - private active = new Set(); - private waiting: Waiter[] = []; - private pending: Set; - private directory: string; - private file: string; - constructor(infraDir: string) { - // An older agent may have retained an unresolved effect here. A new, - // empty canonical store must never silently discard that denial. - try { - lstatSync(join(infraDir, ".service-lifecycle")); - throw Error(FAILED); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - const state = services.agentStatePaths(infraDir).directory; - mkdirSync(state, { recursive: true, mode: 0o700 }); - this.directory = join(state, "service-lifecycle"); - try { - mkdirSync(this.directory, { mode: 0o700 }); - const parent = openSync(state, constants.O_RDONLY | constants.O_DIRECTORY); - try { - fsyncSync(parent); - } finally { - closeSync(parent); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - } - const dir = lstatSync(this.directory); - if ( - !dir.isDirectory() || - dir.isSymbolicLink() || - dir.uid !== (process.geteuid?.() ?? 0) || - (dir.mode & 0o777) !== 0o700 - ) - throw Error(FAILED); - this.file = join(this.directory, "pending.json"); - let fd: number | undefined; - try { - fd = openSync(this.file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); - const stat = fstatSync(fd); - if ( - !stat.isFile() || - stat.uid !== dir.uid || - stat.nlink !== 1 || - (stat.mode & 0o777) !== 0o600 || - stat.size > 64 * 1024 - ) - throw Error(FAILED); - const ids: unknown = JSON.parse(readFileSync(fd, "utf8")); - if ( - !Array.isArray(ids) || - ids.length > 256 || - ids.some((id) => typeof id !== "string" || !ID.test(id)) || - new Set(ids).size !== ids.length - ) - throw Error(FAILED); - this.pending = new Set(ids); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - this.pending = new Set(); - } finally { - if (fd !== undefined) closeSync(fd); - } - } - private persist(next: Set): void { - if (next.size > 256) throw Error(FAILED); - const temporary = `${this.file}.tmp`; - try { - const stat = lstatSync(temporary); - if ( - !stat.isFile() || - stat.isSymbolicLink() || - stat.uid !== (process.geteuid?.() ?? 0) || - stat.nlink !== 1 || - (stat.mode & 0o777) !== 0o600 || - stat.size > 64 * 1024 - ) - throw Error(FAILED); - rmSync(temporary); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - const fd = openSync( - temporary, - constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, - 0o600, - ); - try { - writeFileSync(fd, JSON.stringify([...next].sort())); - fsyncSync(fd); - } finally { - closeSync(fd); - } - renameSync(temporary, this.file); - const parent = openSync(this.directory, constants.O_RDONLY | constants.O_DIRECTORY); - try { - fsyncSync(parent); - } finally { - closeSync(parent); - } - this.pending = next; - } - private drain(): void { - const earlier = new Set(); - for (const waiter of [...this.waiting]) { - if (waiter.ids.some((id) => this.active.has(id) || earlier.has(id))) { - for (const id of waiter.ids) earlier.add(id); - continue; - } - this.waiting.splice(this.waiting.indexOf(waiter), 1); - waiter.signal.removeEventListener("abort", waiter.abort); - for (const id of waiter.ids) this.active.add(id); - waiter.grant(() => { - for (const id of waiter.ids) this.active.delete(id); - this.drain(); - }); - } - } - private async acquire(ids: string[], signal: AbortSignal): Promise<() => void> { - signal.throwIfAborted(); - return new Promise((grant, reject) => { - const waiter: Waiter = { - ids, - signal, - grant, - abort: () => { - const index = this.waiting.indexOf(waiter); - if (index < 0) return; - this.waiting.splice(index, 1); - reject(signal.reason); - this.drain(); - }, - }; - this.waiting.push(waiter); - signal.addEventListener("abort", waiter.abort, { once: true }); - if (signal.aborted) waiter.abort(); - else this.drain(); - }); - } - async run( - serviceIds: readonly string[], - signal: AbortSignal, - work: () => Promise, - ): Promise { - if (serviceIds.length > 256 || serviceIds.some((id) => !ID.test(id))) throw Error(FAILED); - const ids = [...new Set(serviceIds)].sort(); - const release = await this.acquire(ids, signal); - try { - signal.throwIfAborted(); - if (ids.some((id) => this.pending.has(id))) throw Error(FAILED); - return await work(); - } finally { - release(); - } - } - async creating( - serviceIds: readonly string[], - signal: AbortSignal, - work: (begin: () => void) => Promise, - ): Promise { - return this.run(serviceIds, signal, async () => { - let begun = false; - const begin = () => { - signal.throwIfAborted(); - if (!begun) { - this.persist(new Set([...this.pending, ...serviceIds])); - begun = true; - } - }; - const clear = () => - this.persist(new Set([...this.pending].filter((id) => !serviceIds.includes(id)))); - try { - const result = await work(begin); - if (begun) clear(); - return result; - } catch (error) { - if (begun && isContainedProcessFailure(error)) clear(); - throw error; - } - }); - } -} diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index ff12820f8..4a48eda6c 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -187,7 +187,11 @@ describe("assembled production service authority", () => { }); const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal, @@ -305,7 +309,11 @@ describe("assembled production service authority", () => { serviceIds: host.loaded.map((service) => service.manifest.id), }), app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal: await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal), @@ -418,7 +426,11 @@ describe("assembled production service authority", () => { }); const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url") }, + tokens: { + api: host.token, + "data-manager": Buffer.alloc(32, 23).toString("base64url"), + "host-cli": "h".repeat(43), + }, runtime: host.runtime, resourceClaimer: host.claimer, journal, @@ -606,7 +618,7 @@ describe("assembled production service authority", () => { const api = Buffer.alloc(32, 31).toString("base64url"), dm = Buffer.alloc(32, 32).toString("base64url"); const app = buildOpsAgentServer({ - tokens: { api, "data-manager": dm }, + tokens: { api, "data-manager": dm, "host-cli": "h".repeat(43) }, runtime: host.runtime, resourceClaimer: host.claimer, }); @@ -718,139 +730,6 @@ async function installedCommunity() { return { host, start, remove, checkout, record, create }; } const drain = () => new Promise((resolve) => setImmediate(resolve)); -describe("assembled lifecycle/removal exclusion", () => { - it("fences selected dependencies created by another service's Compose Start", async () => { - const { host, remove, checkout, create } = await installedCommunity(); - await host.select(["motis", "community-live"]); - writeFileSync( - host.stack.composeReleasePath, - "services:\n motis:\n depends_on:\n community-live:\n condition: service_started\n", - { mode: 0o644 }, - ); - const start = { - kind: "service.start", - serviceId: "motis", - expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, - } as const; - const entered = Promise.withResolvers(), - finish = Promise.withResolvers(); - host.setCreatingProcess(async () => { - entered.resolve(); - await finish.promise; - create(); - }); - const starting = dispatchOpsOperation(host.runtime, start, await host.context(start)); - await entered.promise; - const removing = dispatchOpsOperation(host.runtime, remove, await host.context(remove)).then( - (value) => ({ value }), - (error) => ({ error }), - ); - try { - await drain(); - expect(existsSync(checkout), "Compose can also create the selected dependency").toBe(true); - finish.resolve(); - await starting; - expect("error" in (await removing)).toBe(true); - } finally { - finish.resolve(); - await starting.catch(() => undefined); - await removing; - } - }); - - it("retains authority throughout a community Start process and renewed removal inventory", async () => { - const { host, start, remove, checkout, record, create } = await installedCommunity(); - const entered = Promise.withResolvers(), - finish = Promise.withResolvers(); - host.setCreatingProcess(async () => { - entered.resolve(); - await finish.promise; - create(); - }); - const starting = dispatchOpsOperation(host.runtime, start, await host.context(start)); - await entered.promise; - let removed = false; - const removing = dispatchOpsOperation(host.runtime, remove, await host.context(remove)) - .then( - (value) => ({ value }), - (error) => ({ error }), - ) - .finally(() => { - removed = true; - }); - try { - await drain(); - expect( - existsSync(checkout), - "checkout cannot disappear while Start's real process is pending", - ).toBe(true); - expect(existsSync(record)).toBe(true); - expect(removed).toBe(false); - finish.resolve(); - await starting; - const outcome = await removing; - expect( - "error" in outcome && - outcome.error instanceof Error && - outcome.error.message.includes("require cleanup"), - ).toBe(true); - const stop = { kind: "service.stop", serviceId: "community-live" } as const; - await dispatchOpsOperation(host.runtime, stop, await host.context(stop)); - const logs = { kind: "service.logs", serviceId: "community-live", tail: 20 } as const; - await expect( - dispatchOpsOperation(host.runtime, logs, await host.context(logs)), - ).resolves.toMatchObject({ lines: ["retained log"] }); - } finally { - finish.resolve(); - await starting.catch(() => undefined); - await removing; - } - }); - - it("revalidates a previously admitted Start after removal wins the same boundary", async () => { - const { host, start, remove, checkout, create } = await installedCommunity(); - const startContext = await host.context(start), - removeContext = await host.context(remove); - const stackStart = { - kind: "stack.start", - expectedConfigurationRevisionId: services.currentConfigurationGeneration(host.infraDir)!, - } as const; - const stackContext = await host.context(stackStart); - const entered = Promise.withResolvers(), - finish = Promise.withResolvers(); - host.setInventoryProcess(async () => { - entered.resolve(); - await finish.promise; - }); - let starts = 0; - host.setCreatingProcess(async () => { - starts++; - create(); - }); - const removing = dispatchOpsOperation(host.runtime, remove, removeContext); - await entered.promise; - const starting = dispatchOpsOperation(host.runtime, start, startContext).then( - (value) => ({ value }), - (error) => ({ error }), - ); - try { - await drain(); - expect(starts).toBe(0); - finish.resolve(); - await expect(removing).resolves.toMatchObject({ removed: true }); - expect(existsSync(checkout)).toBe(false); - const outcome = await starting; - expect("error" in outcome && outcome.error instanceof Error).toBe(true); - await expect(dispatchOpsOperation(host.runtime, stackStart, stackContext)).rejects.toThrow(); - expect(starts).toBe(0); - } finally { - finish.resolve(); - await removing.catch(() => undefined); - await starting; - } - }); -}); - it.runIf(installedCompose)( "executes assembled retained-generation cleanup after real Compose path qualification", async () => { @@ -870,71 +749,6 @@ it.runIf(installedCompose)( ); describe("settlement and cancellation at the installed lifecycle boundary", () => { - it("cancels a waiting removal without stealing the still-executing Start guard", async () => { - const { host, start, remove, checkout, create } = await installedCommunity(); - const entered = Promise.withResolvers(), - finish = Promise.withResolvers(); - host.setCreatingProcess(async () => { - entered.resolve(); - await finish.promise; - create(); - }); - const starting = dispatchOpsOperation(host.runtime, start, await host.context(start)); - await entered.promise; - const controller = new AbortController(); - const removeContext = await host.context(remove, controller.signal); - const removing = dispatchOpsOperation(host.runtime, remove, removeContext).then( - (value) => ({ value }), - (error) => ({ error }), - ); - await drain(); - controller.abort(new Error("synthetic waiting cancellation")); - try { - const result = await removing; - expect("error" in result).toBe(true); - expect(existsSync(checkout)).toBe(true); - expect(host.row.State).toBe("running"); - } finally { - finish.resolve(); - await starting; - } - await expect( - dispatchOpsOperation(host.runtime, remove, await host.context(remove)), - ).rejects.toThrow("require cleanup"); - }); - - it("retains an unknown aborted process fence across production runtime reconstruction", async () => { - const { host, start, remove, checkout } = await installedCommunity(); - const entered = Promise.withResolvers(), - finish = Promise.withResolvers(); - host.setCreatingProcess(async () => { - entered.resolve(); - await finish.promise; - throw Error("unknown external process settlement"); - }); - const controller = new AbortController(), - ctx = await host.context(start, controller.signal); - const starting = dispatchOpsOperation(host.runtime, start, ctx).then( - (value) => ({ value }), - (error) => ({ error }), - ); - await entered.promise; - controller.abort(); - finish.resolve(); - expect("error" in (await starting)).toBe(true); - const restarted = await createProductionOpsRuntime({ - rootDir: host.root, - stack: host.stack, - loadAuthority: host.loadAuthority, - communityRepositories: host.repositories, - docker: { execFile: host.execFile, fileExists: existsSync }, - }); - await expect( - dispatchOpsOperation(restarted, remove, await host.context(remove)), - ).rejects.toThrow("settlement is unresolved"); - expect(existsSync(checkout)).toBe(true); - }); - it("releases a known-contained failing process and orders multi-service Start without deadlock", async () => { const { host, start, remove, checkout } = await installedCommunity(); let first = true; diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 929e5b623..884a4c62b 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -34,7 +34,6 @@ import { } from "./docker-runtime"; import { installImageStageRuntime } from "./image-stage-runtime"; import { type IntegrationStore, installIntegrationRuntime } from "./integration-store"; -import { ServiceLifecycleExclusion } from "./lifecycle-exclusion"; import { createOwnedServiceResolver, type OwnedServiceResolver } from "./owned-services"; import { installPlatformConfigurationRuntime } from "./platform-configuration-runtime"; import { serviceActionForOperation } from "./policy"; @@ -199,7 +198,6 @@ export async function createProductionOpsRuntime( options: ProductionOpsRuntimeOptions, ): Promise { const initial = await options.loadAuthority(); - const lifecycle = new ServiceLifecycleExclusion(options.stack.infraDir); const owned = createOwnedServiceResolver({ stack: options.stack, execFile: options.docker?.execFile ?? runContainedProcess, @@ -240,7 +238,7 @@ export async function createProductionOpsRuntime( ), }); if (options.integrationStore) installIntegrationRuntime(runtime, options.integrationStore); - installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned, lifecycle); + installCommunityRepositoryRuntime(runtime, options.communityRepositories, owned); installConfigurationHardlinksRuntime(runtime, { infraDir: options.stack.infraDir }); registerConfigurationMaintenanceSource(runtime, { rootDir: options.rootDir, @@ -410,54 +408,14 @@ export async function createProductionOpsRuntime( } return handler(operation, context); }; - const guarded = async ( - operation: Extract, - context: OpsExecutionContext, - ) => { - if ( - ![ - "service.start", - "service.restart", - "service.recreate", - "service.recreateIsolated", - "service.update", - ].includes(kind) - ) - return execute(operation, context); - // Compose up may also create dependencies, including release-overlay - // dependencies. Conservatively exclude every registered service for those - // commands; this local boundary does not acquire or replace root authority. - const ids = - kind === "service.start" || kind === "service.recreate" || kind === "service.update" - ? (await options.loadAuthority()).services.map((service) => service.manifest.id).sort() - : [operation.serviceId]; - return lifecycle.creating(ids, context.signal, async (begin) => { - if (kind === "service.start" || kind === "service.recreate" || kind === "service.update") { - const current = (await options.loadAuthority()).services - .map((service) => service.manifest.id) - .sort(); - if (JSON.stringify(current) !== JSON.stringify(ids)) - throw new Error("Service authority changed while waiting"); - } - return execute(operation, { ...context, beginServiceCreation: begin }); - }); - }; // The finite loop preserves each handler's operation/result correspondence. - Object.assign(runtime, { [kind]: guarded }); + // Exclusion between writers is the lease the agent holds while this runs. + Object.assign(runtime, { [kind]: execute }); } const stackStart = runtime["stack.start"]; runtime["stack.start"] = async (operation, context) => { - const ids = (await options.loadAuthority()).services - .map((service) => service.manifest.id) - .sort(); - return lifecycle.creating(ids, context.signal, async (begin) => { - const authority = await options.loadAuthority(); - const current = authority.services.map((service) => service.manifest.id).sort(); - if (JSON.stringify(current) !== JSON.stringify(ids)) - throw new Error("Service authority changed while waiting"); - serviceActionAuthority(authority, options.stack.infraDir); - return stackStart(operation, { ...context, beginServiceCreation: begin }); - }); + serviceActionAuthority(await options.loadAuthority(), options.stack.infraDir); + return stackStart(operation, context); }; resolvers.set(runtime, owned); registeredServiceSources.set(runtime, options.loadAuthority); diff --git a/apps/ops-agent/src/runtime.ts b/apps/ops-agent/src/runtime.ts index de15ae27f..bfdfd352f 100644 --- a/apps/ops-agent/src/runtime.ts +++ b/apps/ops-agent/src/runtime.ts @@ -61,8 +61,6 @@ export interface OpsExecutionContext { completionDigest: string; }): Promise; markFeedProxyReloadAttempted?(): Promise; - /** Installed only by the shared production exclusion; records denial before external creation. */ - beginServiceCreation?(): void; signal: AbortSignal; emitLog(stream: "stdout" | "stderr", message: string): void; claim: OpsTrustedClaim; diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 26e87ac47..bb594fb22 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -84,7 +84,7 @@ describe("ops-agent async foundation", () => { return operation.kind === "service.pull" ? pullResult : { reachable: true }; }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 23), @@ -225,7 +225,7 @@ describe("ops-agent async foundation", () => { it("rejects resource and option mismatches for an exact retained terminal key", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 29), @@ -308,7 +308,7 @@ describe("ops-agent async foundation", () => { }, ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, idByte++), @@ -448,7 +448,7 @@ describe("ops-agent async foundation", () => { }, ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, journal, maxConcurrency: 1, @@ -514,7 +514,7 @@ describe("ops-agent async foundation", () => { let calls = 0; const audits: OpsAuditEvent[] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), operationTimeoutMs: 10, @@ -571,7 +571,7 @@ describe("ops-agent async foundation", () => { it("accepts only bounded UTF-8 log events from the typed execution context", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 24), @@ -629,7 +629,7 @@ describe("ops-agent async foundation", () => { it("reports a host update as its safe error code through the async protocol", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), dispatch: async () => { @@ -665,7 +665,7 @@ describe("ops-agent async foundation", () => { it("rejects oversized sync output and retains the maximum build-all receipt", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 25), @@ -722,7 +722,7 @@ describe("ops-agent async foundation", () => { let nowMs = baseTime; let randomValue = 30; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(nowMs), randomBytes: () => Buffer.alloc(18, randomValue++), @@ -799,7 +799,7 @@ describe("ops-agent async foundation", () => { it("bounds entries that need recovery separately from capacity for new work", async () => { let randomValue = 60; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, randomValue++), @@ -810,12 +810,17 @@ describe("ops-agent async foundation", () => { return { changed: true }; }, }); + // A stop that needs recovery keeps its target leased; new work uses another one. const submit = (kind: "service.stop" | "service.restart", name: string) => app.inject({ method: "POST", url: "/v1/operations", headers: auth(), - payload: request({ kind, serviceId: "redis" }, `ops1_${name}`, key(`opk1_${name}`)), + payload: request( + { kind, serviceId: kind === "service.stop" ? "redis" : "osrm" }, + `ops1_${name}`, + key(`opk1_${name}`), + ), }); const finish = async (id: string) => eventually( @@ -837,3 +842,96 @@ describe("ops-agent async foundation", () => { await app.close(); }); }); + +describe("one admission path for target effects", () => { + it("answers busy for a restart or removal of a held target and audits the holder", async () => { + const control = await import("@openmapx/core/build-artifacts"); + const { mkdtempSync, mkdirSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const root = mkdtempSync(join(tmpdir(), "omx-leased-admission-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["redis", "osrm"], + }); + const leaseStore = control.openLeaseStore({ + directory: control.leaseStoreDirectory(root), + purpose: "test", + }); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + const audit: OpsAuditEvent[] = []; + const app = await buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + resourcePolicy: allowAllResources, + controlInstallation: installation, + leaseStore, + audit: (event) => audit.push(event), + dispatch: async (operation) => { + if (operation.kind === "service.stop") await blocked; + return { changed: true }; + }, + }); + try { + const submit = (operation: OpsOperation, name: string) => + app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(), + // Issued now: this test may run long after the module's base time. + payload: { + ...request(operation, `ops1_${name}Lease`, key(`opk1_${name}Lease`)), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + }, + }); + const stop = await submit({ kind: "service.stop", serviceId: "redis" }, "leasedStop00000"); + expect(stop.statusCode).toBe(202); + await eventually( + async () => leaseStore.list(), + (leases) => leases.some((lease) => lease.targets.includes("redis")), + ); + const holder = leaseStore.list()[0]!.owner.operationKey; + for (const [operation, name] of [ + [{ kind: "service.restart", serviceId: "redis" }, "leasedRestart00"], + [{ kind: "service.remove", serviceId: "redis" }, "leasedRemove000"], + ] as const) { + const refused = await submit(operation, name); + expect(refused.statusCode).toBe(409); + expect(refused.json().error.class).toBe("busy"); + } + expect( + audit + .filter((event) => event.heldBy?.includes(holder)) + .map((event) => [event.kind, event.result]), + ).toEqual([ + ["service.restart", "denied"], + ["service.remove", "denied"], + ]); + // Another target is free. + expect( + (await submit({ kind: "service.restart", serviceId: "osrm" }, "leasedOther0000")) + .statusCode, + ).toBe(202); + release(); + await eventually( + async () => leaseStore.list(), + (leases) => !leases.some((lease) => lease.targets.includes("redis")), + ); + expect( + (await submit({ kind: "service.restart", serviceId: "redis" }, "leasedAfter0000")) + .statusCode, + ).toBe(202); + } finally { + release(); + await app.close(); + control.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/ops-agent/src/server-review-2.test.ts b/apps/ops-agent/src/server-review-2.test.ts index 8fd172b88..45cfac20e 100644 --- a/apps/ops-agent/src/server-review-2.test.ts +++ b/apps/ops-agent/src/server-review-2.test.ts @@ -86,7 +86,7 @@ describe("reviewed durable admission boundary", () => { revisionId: `cfg1_${"t".repeat(43)}`, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, claimTimeoutMs: 5, resourceClaimer: { claim: async (_operation, fingerprint) => { @@ -144,7 +144,7 @@ describe("reviewed durable admission boundary", () => { revisionId: `cfg1_${"r".repeat(43)}`, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, journal: { records: () => records, replace: async (next) => { @@ -242,7 +242,7 @@ describe("reviewed durable admission boundary", () => { }); const dispatch = vi.fn(async () => pullResult); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal: { records: () => [tombstone], replace: async () => undefined }, maxJobEntries: 1, @@ -294,7 +294,7 @@ describe("reviewed durable admission boundary", () => { }; const dispatched: string[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: { claim: async (operation, fingerprint) => { sourceReference = operation; @@ -351,7 +351,7 @@ describe("reviewed durable admission boundary", () => { }; const dispatch = vi.fn(async () => pullResult); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal, now: () => new Date(baseTime), @@ -391,7 +391,7 @@ describe("reviewed durable admission boundary", () => { }, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal, now: () => new Date(baseTime), @@ -464,7 +464,7 @@ describe("reviewed durable admission boundary", () => { let releaseFirst!: () => void; const firstBlocked = new Promise((resolve) => (releaseFirst = resolve)); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal, now: () => new Date(baseTime), @@ -560,7 +560,7 @@ describe("reviewed durable admission boundary", () => { return { changed: true }; }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, now: () => new Date(baseTime), dispatch, @@ -599,7 +599,7 @@ describe("reviewed durable admission boundary", () => { result.lines[255] = "x".repeat(budget - baseBytes); expect(Buffer.byteLength(JSON.stringify(result), "utf8")).toBe(budget); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, now: () => new Date(baseTime), dispatch: async () => result, @@ -651,7 +651,7 @@ describe("reviewed durable admission boundary", () => { }), ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal: { records: () => records, replace: async () => undefined }, now: () => new Date(baseTime), @@ -690,7 +690,7 @@ describe("reviewed durable admission boundary", () => { }); const dispatch = vi.fn(async () => pullResult); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal, now: () => new Date(baseTime + 1_000), @@ -747,7 +747,7 @@ describe("reviewed durable admission boundary", () => { terminalAt: new Date(baseTime).toISOString(), }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal: { records: () => [existing], replace: async () => undefined }, now: () => new Date(baseTime), @@ -783,7 +783,7 @@ describe("reviewed durable admission boundary", () => { }; const seen: string[] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: claimer, claimTimeoutMs: 5, maxConcurrency: 1, @@ -856,7 +856,7 @@ describe("reviewed durable admission boundary", () => { }, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: claimer, maxConcurrency: 1, now: () => new Date(baseTime), @@ -881,7 +881,7 @@ describe("reviewed durable admission boundary", () => { it("contains a throwing audit sink on success and failure", async () => { let fail = false; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, now: () => new Date(baseTime), audit: () => { @@ -923,7 +923,7 @@ describe("reviewed durable admission boundary", () => { const blocked = new Promise((resolve) => (release = resolve)); let calls = 0; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, now: () => new Date(baseTime), operationTimeoutMs: 5, @@ -1004,7 +1004,7 @@ describe("asynchronous operation slots", () => { it("a throw during terminal bookkeeping releases the concurrency slot", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: releasingClaimer(() => { throw new Error("release failed synchronously"); }), @@ -1031,7 +1031,7 @@ describe("asynchronous operation slots", () => { const blocked = new Promise((resolve) => (unblock = resolve)); let calls = 0; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, maxConcurrency: 2, now: () => new Date(baseTime), @@ -1061,7 +1061,7 @@ describe("operation context for configuration helper drain", () => { const counts: number[] = []; const journal: OpsJobJournal = { records: () => [], replace: async () => {} }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: allow, journal, now: () => new Date(baseTime), diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index 09ab7b02e..f4cc75f62 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -77,7 +77,7 @@ function auth(token: string) { describe("ops-agent request boundary", () => { it("reports health without privacy backup collector evidence", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, }); expect((await app.inject({ method: "GET", url: "/health" })).json()).toEqual({ ok: true }); await app.close(); @@ -86,7 +86,7 @@ describe("ops-agent request boundary", () => { it("authenticates and resolves role before operation validation or dispatch", async () => { const dispatch = vi.fn(); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch, @@ -113,7 +113,7 @@ describe("ops-agent request boundary", () => { it("rejects unknown fields, stale/future/duplicate requests, and excessive bodies", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), bodyLimit: 512, @@ -186,7 +186,7 @@ describe("ops-agent request boundary", () => { const events: OpsAuditEvent[] = []; const secret = `${apiToken} docker argv http://ops-agent:4300`; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), audit: (event) => events.push(event), @@ -221,11 +221,11 @@ describe("ops-agent request boundary", () => { it("requires distinct role credentials and fails closed on resource authorization", async () => { expect(() => buildActualOpsAgentServer({ - tokens: { api: apiToken, "data-manager": apiToken }, + tokens: { api: apiToken, "data-manager": apiToken, "host-cli": "h".repeat(43) }, }), ).toThrow("distinct"); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: {}, now: () => new Date(nowIso), dispatch: vi.fn(), @@ -297,7 +297,7 @@ describe("ops-agent lifecycle deadlines", () => { }, }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourceClaimer: host.claimer, controlInstallation: installation, now: () => new Date(nowIso), @@ -359,7 +359,7 @@ describe("ops-agent lifecycle deadlines", () => { }), ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch, @@ -423,7 +423,7 @@ describe("retained execution deadlines", () => { let calls = 0; const records: import("./journal").PersistedOpsJob[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, journal: { records: () => [], @@ -509,7 +509,7 @@ describe("retained execution deadlines", () => { let signal: AbortSignal | undefined; let finish!: (x: unknown) => void; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, dispatch: async (_o, _r, c) => { signal = c.signal; @@ -550,7 +550,7 @@ describe("retained execution deadlines", () => { it("rejects unsigned wire parent and delegation headers before effects", async () => { const dispatch = vi.fn(); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch, @@ -606,7 +606,7 @@ describe("execution identity at the agent boundary", () => { let finish!: (x: unknown) => void; let execution: import("@openmapx/core/ops").ExecutionRef | undefined; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, dispatch: async (_o, _r, c) => { execution = c.execution; @@ -660,7 +660,7 @@ describe("execution identity at the agent boundary", () => { let fail!: (x: unknown) => void; const snapshots: import("./journal").PersistedOpsJob[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), journal: { @@ -719,7 +719,7 @@ it("observer abort leaves the admitted backend running until its real completion let backendSignal: AbortSignal | undefined; let finish!: (x: unknown) => void; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch: async (_o, _r, c) => { @@ -839,7 +839,7 @@ it.each([0, 1])( }); const retained: import("./journal").PersistedOpsJob[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, runtime, journal: { @@ -925,7 +925,7 @@ describe("ops protocol version", () => { maxDurationSeconds: 300, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch: async () => @@ -1015,7 +1015,7 @@ describe("ops protocol version", () => { it("refuses cancellation without the execution fingerprint", async () => { let finish!: (x: unknown) => void; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch: async () => diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 59d326df7..bba12d9fc 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -20,6 +20,7 @@ import { inspectControlAdmission, inspectFeedProxyValidationFailure, isFixedSingleDataCommand, + type LeaseStore, markFeedProxyReloadAttempted, planFixedSingleDataCommand, controlHeaders as readControlHeaders, @@ -45,10 +46,11 @@ import { retainGenerationHardlinksOwnerPlan, settleArtifactTargetLeaseChild, startControlPrune, + TargetBusyError, type TargetLeaseAuthority, type VerifiedExecutionAuthority, verifyControlEffectSettlement, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { @@ -156,6 +158,7 @@ import { } from "./dawarich-subject-export"; import { registerOpsIngressRateLimit } from "./ingress-rate-limit"; import type { OpsJobJournal, PersistedOpsJob } from "./journal"; +import { LeaseRefused, runLeaseOperation } from "./lease-operations"; import { type CoordinatorPullOwnershipObservation, consumeCoordinatorPullOwnershipCapture, @@ -202,6 +205,32 @@ export const OPS_AGENT_RETENTION_LIMITS = { maxAggregateEventBytes: 2 * DEFAULT_JOB_ENTRIES * MAX_RETAINED_EVENT_BYTES_PER_JOB, } as const; // Conservative read-only surface while the release store cannot be read safely. +/** + * Operations that change containers claim their targets in the lease store + * before any effect. Pulls, builds, execs and probes leave running containers + * alone and claim nothing. + */ +const LEASED_KINDS = new Set([ + "configurationMaintenance.prepare", + "artifactTransfer.import", + "artifactTransfer.export", + "stack.start", + "stack.stop", + "coreService.restart", + "service.start", + "service.stop", + "service.restart", + "service.remove", + "service.recreate", + "service.recreateIsolated", + "service.update", + "serviceRepository.remove", + "backup.create", + "backup.restore", + "configuration.hardlinks.apply", + "valhalla.traffic.maintain", + "valhalla.traffic.refreshWaysToEdges", +]); const RELEASE_RECOVERY_READ_KINDS = new Set([ "configurationHandoff.inspect", "configurationMaintenance.inspect", @@ -239,11 +268,15 @@ export interface OpsAuditEvent { result: "success" | "denied" | "error" | "timeout" | "terminated" | "late_completion"; durationMs: number; errorClass?: OpsErrorClass; + /** The lease that refused a busy operation. */ + heldBy?: string; } export interface BuildOpsAgentServerOptions { artifactTransfers?: ArtifactTransferStore; controlInstallation?: ControlInstallation; + /** The agent's lease store, for the leases data-manager and the host CLI hold. */ + leaseStore?: LeaseStore; terminalClaimCleanup?: VerifiedTerminalClaimCleanup; tokens: Record; releaseRecovery?: { ready: boolean; reason?: "recovery_required" }; @@ -271,7 +304,6 @@ export interface BuildOpsAgentServerOptions { } interface JobRecord extends Omit { - role: OpsRole; operation: OpsOperation; operationId: string; operationKey: string; @@ -302,9 +334,11 @@ function resolveRole( const candidate = digest(match?.[1] ?? ""); const apiMatches = timingSafeEqual(candidate, digest(tokens.api)); const dataManagerMatches = timingSafeEqual(candidate, digest(tokens["data-manager"])); + const hostCliMatches = timingSafeEqual(candidate, digest(tokens["host-cli"])); if (!match) return null; if (apiMatches) return "api"; if (dataManagerMatches) return "data-manager"; + if (hostCliMatches) return "host-cli"; return null; } @@ -428,7 +462,7 @@ function classify(error: unknown): { errorClass: OpsErrorClass; statusCode: numb } export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): FastifyInstance { - if (options.tokens.api === options.tokens["data-manager"]) { + if (new Set(Object.values(options.tokens)).size !== Object.keys(options.tokens).length) { throw new Error("Ops agent caller credentials must be distinct"); } const app = Fastify({ @@ -991,7 +1025,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); if (lease.mode === "owner") { await releaseArtifactTargetLease(lease); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); // Classify only this freshly proved, fully released never-prepared // refusal before the immutable terminal is first committed. // An unknown/partial release never takes this branch. @@ -1056,7 +1090,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } if (releaseUnstartedRoot && lease?.mode === "owner" && authority) { await releaseArtifactTargetLease(lease); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); if (job.operation.kind === "artifactTransfer.import") await options.artifactTransfers?.reconcileReleasedPreparedImport(authority); } @@ -2200,6 +2234,31 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const nowMs = now().getTime(); const timing = validateTiming(parsed.issuedAt, parsed.expiresAt, nowMs); replay.accept(parsed.requestId, timing.expiresAtMs, nowMs); + if ( + operation.kind === "lease.acquire" || + operation.kind === "lease.read" || + operation.kind === "lease.update" + ) { + if (!options.controlInstallation || !options.leaseStore) + throw new RequestGuardError("recovery_required", 409); + let value: unknown; + try { + value = await runLeaseOperation( + options.controlInstallation, + options.leaseStore, + role, + operation, + ); + } catch (error) { + if (error instanceof LeaseRefused) throw new RequestGuardError("authorization", 403); + throw new RequestGuardError("conflict", 409); + } + return reply + .code(200) + .send(opsSuccess(parsed.requestId, { execution: "sync", kind, value })); + } + // Only lease operations are open to the host CLI's token. + if (role === "host-cli") throw new RequestGuardError("authorization", 403); const policy = OPS_KIND_POLICIES[operation.kind]; const operationFingerprint = opsOperationFingerprint(operation); if ( @@ -2698,6 +2757,43 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("authorization", 403); } } + // A held target refuses the operation as busy before it is recorded, so + // nothing is left to recover. The holder is named in the audit line. + if ( + !controlAdmission && + !configurationMaintenanceControl && + LEASED_KINDS.has(operation.kind) && + options.controlInstallation && + options.leaseStore + ) { + const wanted = operation.kind.startsWith("valhalla.") + ? ["valhalla"] + : "serviceId" in operation && + ![ + "service.start", + "service.recreate", + "service.recreateIsolated", + "service.update", + "artifactTransfer.export", + ].includes(operation.kind) + ? [operation.serviceId] + : controlRegisteredServiceIds(options.controlInstallation); + const holder = options.leaseStore + .list() + .find((lease) => lease.targets.some((target) => wanted.includes(target))); + if (holder) { + options.audit?.({ + role, + kind, + resourceId, + result: "denied", + durationMs: Date.now() - startedAt, + errorClass: "busy", + heldBy: `${holder.owner.executor} ${holder.owner.operationKey} (${holder.ownerKind}, ${holder.state})`, + }); + throw new RequestGuardError("busy", 409); + } + } await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); jobsById.set(job.operationId, job); jobsByKey.set(ownerKey, job); @@ -2813,38 +2909,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (lifecyclePlan) retainFixedServiceLifecyclePlan(authority, lifecyclePlan); reserveControlTerminal(authority); controlAuthorities.set(job.operationId, authority); - if ( - [ - "configurationMaintenance.prepare", - "artifactTransfer.import", - "artifactTransfer.export", - "stack.start", - "coreService.restart", - "service.start", - "service.recreate", - "service.recreateIsolated", - "service.update", - "configuration.hardlinks.apply", - "valhalla.traffic.maintain", - "valhalla.traffic.refreshWaysToEdges", - ].includes(job.operation.kind) - ) { + if (LEASED_KINDS.has(job.operation.kind)) { const lease = await acquireArtifactTargetLease({ - dataRoot: controlInstallationDataRoot(options.controlInstallation), - serviceIds: - maintenancePlan || - hardlinksPlan || - lifecyclePlan || - job.operation.kind === "artifactTransfer.export" || - [ - "stack.start", - "service.start", - "coreService.restart", - "service.recreate", - "service.recreateIsolated", - ].includes(job.operation.kind) - ? targets - : ["valhalla"], + serviceIds: job.operation.kind.startsWith("valhalla.") ? ["valhalla"] : targets, owner: authority, }); controlLeases.set(job.operationId, lease); @@ -2919,7 +2986,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif active -= 1; throw error instanceof RequestGuardError ? error - : new RequestGuardError("runtime", 500); + : error instanceof TargetBusyError + ? new RequestGuardError("busy", 409) + : new RequestGuardError("runtime", 500); }); if (admission.type === "retained") { await claimAdmission?.rollback().catch(() => undefined); diff --git a/apps/ops-agent/src/service-lifecycle-runtime.ts b/apps/ops-agent/src/service-lifecycle-runtime.ts index 59bea71f8..b7b3c0dfa 100644 --- a/apps/ops-agent/src/service-lifecycle-runtime.ts +++ b/apps/ops-agent/src/service-lifecycle-runtime.ts @@ -6,7 +6,7 @@ import { consumeControlChildDispatch, type FixedServiceLifecyclePlan, fixedCliArguments, - inspectArtifactTargetLeaseChildSettlement, + inspectLocalLeaseChildSettlement, issueControlChildPhase, openFixedChildInstallation, planFixedServiceUpdate, @@ -77,7 +77,6 @@ export function installServiceLifecycleRuntime( consumeControlChildDispatch(issued.dispatchTicket); let failure: unknown; try { - context.beginServiceCreation?.(); await options.runFixedCli(args, { ...context, execution: child.execution, @@ -121,7 +120,7 @@ export function installServiceLifecycleRuntime( { serviceLifecycle: evidence }, ); verifyControlEffectSettlement(installation, child.execution); - const settled = inspectArtifactTargetLeaseChildSettlement(lease, child.execution); + const settled = inspectLocalLeaseChildSettlement(lease, child.execution); if (settled?.outcome !== "committed" || settled.publication !== "committed") throw new OpsExecutionFailure("recovery_required", { serviceLifecycle: evidence }); return { changed: true, revisionId: receipt.finalRevisionId }; diff --git a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts index 25ea6b4eb..d273e1c7f 100644 --- a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts +++ b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts @@ -105,7 +105,7 @@ const app = buildOpsAgentServer({ runtime, journal, controlInstallation: installation, - tokens: { api: "a".repeat(43), "data-manager": token }, + tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, resourceClaimer: createProductionRegistryResourceClaimer({ services: ["motis", "motis-feed-proxy"].map((serviceId) => ({ serviceId, diff --git a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs index 8a0f9dbb0..89feb2d59 100644 --- a/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs +++ b/apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs @@ -21,42 +21,6 @@ let raw = ""; for await (const chunk of process.stdin) raw += chunk; const input = JSON.parse(raw); syncBuiltinESMExports(); -if (["held", "unwound"].includes(input.stopAfterFirstTargetRelease)) { - const remove = fs.rmdirSync, - locks = join(input.infraDir, "data/.admin-target-control/locks"); - let stopped = false, - acknowledgementLost = false; - fs.rmdirSync = (path, ...options) => { - let target = false; - if (String(path).endsWith("/app-api")) { - try { - const actual = fs.statSync(dirname(String(path))), - expected = fs.statSync(locks); - target = actual.dev === expected.dev && actual.ino === expected.ino; - } catch { - /* Only this original physical lock is the named fault edge. */ - } - } - remove(path, ...options); - if (target && !stopped && input.stopAfterFirstTargetRelease === "unwound") { - acknowledgementLost = true; - throw Error("Named target release acknowledgement loss"); - } - if ( - !stopped && - ((target && input.stopAfterFirstTargetRelease === "held") || - (acknowledgementLost && String(path).endsWith("/.journal-writer"))) - ) { - stopped = true; - fs.writeSync( - 1, - `${JSON.stringify({ event: "pre-effect-release-unlinked", guard: input.stopAfterFirstTargetRelease })}\n`, - ); - process.kill(process.pid, "SIGSTOP"); - } - }; - syncBuiltinESMExports(); -} if ( !Array.isArray(input.serviceIds) || input.serviceIds.length > 256 || diff --git a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts index 4d578b52c..778bb4a04 100644 --- a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts +++ b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts @@ -202,7 +202,7 @@ journal.replace = async (records) => { if (input.boundary === "terminal") stop(terminal); }; const app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": "b".repeat(32) }, + tokens: { api: token, "data-manager": "b".repeat(32), "host-cli": "h".repeat(43) }, controlInstallation: installation, journal, resourcePolicy: { allowBackup: () => true, allowGlobal: () => true, allowService: () => true }, diff --git a/apps/ops-agent/src/traffic-target-ownership.test.ts b/apps/ops-agent/src/traffic-target-ownership.test.ts index 43ee6400a..a1c7d40dc 100644 --- a/apps/ops-agent/src/traffic-target-ownership.test.ts +++ b/apps/ops-agent/src/traffic-target-ownership.test.ts @@ -1,7 +1,7 @@ import { randomBytes } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey, @@ -66,7 +66,7 @@ it.each(["succeeded", "canceled"] as const)( runtime, journal, controlInstallation: installation, - tokens: { api: "a".repeat(43), "data-manager": token }, + tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, resourceClaimer: createProductionRegistryResourceClaimer({ services: [{ serviceId: "valhalla", enabled: true, isBuiltIn: true }], integrationIds: [], @@ -92,7 +92,7 @@ it.each(["succeeded", "canceled"] as const)( if (admitted.execution !== "async") throw Error("Async traffic root required"); await started; expect(effects).toBe(1); - expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(true); if (mode === "canceled") { const status = await client.status(operation.kind, admitted.operationId), lossy = new OpsClient({ @@ -114,7 +114,7 @@ it.each(["succeeded", "canceled"] as const)( expect((await client.status(operation.kind, admitted.operationId)).state).toBe( "termination_pending", ); - expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(true); if (!status.fingerprint || !status.deadlineAt) throw Error("Original dated traffic identity required"); const authority = control.restoreAdmittedControlExecution(installation, { @@ -133,7 +133,7 @@ it.each(["succeeded", "canceled"] as const)( timeout: 10000, }) .toBe("failed"); - expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(true); } else { release(); await expect @@ -141,7 +141,7 @@ it.each(["succeeded", "canceled"] as const)( timeout: 10000, }) .toBe("succeeded"); - expect(existsSync(join(data, ".admin-target-control/locks/valhalla"))).toBe(false); + expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(false); } } finally { release(); diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index ae7aa3f72..595925379 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -303,12 +303,14 @@ describe("trusted configuration ownership and outcomes", () => { expect(f.label()).toBe("b"); expect(f.followUp()).toBe(routes); }); - it("retains a committed revision when real writer cleanup fails and leaves the foreign contents intact", async () => { + it("retains a committed revision when real writer cleanup fails and leaves the foreign lock intact", async () => { const f = fixture(); - const foreign = join(f.infraDir, coreServices.CONFIGURATION_DIRECTORY, ".writer", "foreign"); + const foreign = coreServices.configurationWriterLockPath(f.infraDir); const result = await f.apply("a", undefined, { afterCommit: async () => { - writeFileSync(foreign, "preserve", { mode: 0o644 }); + // Another file took the lock's place; release must not remove it. + rmSync(foreign); + writeFileSync(foreign, "preserve", { mode: 0o600 }); }, }); expect(result).toMatchObject({ @@ -529,7 +531,11 @@ describe("authenticated conditional configuration admission", () => { }, }); const app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url") }, + tokens: { + api: token, + "data-manager": Buffer.alloc(32, 8).toString("base64url"), + "host-cli": "h".repeat(43), + }, resourceClaimer: claimer, controlInstallation: installation, // This deliberately holds one authenticated claim across the other real diff --git a/packages/cli/__tests__/cli-build-artifacts.test.ts b/packages/cli/__tests__/cli-build-artifacts.test.ts index 2265e4375..d083bb347 100644 --- a/packages/cli/__tests__/cli-build-artifacts.test.ts +++ b/packages/cli/__tests__/cli-build-artifacts.test.ts @@ -214,7 +214,6 @@ describe("build bundles", () => { serviceIds: ["osrm"], }); lease = await control.acquireArtifactTargetLease({ - dataRoot: dataDir(targetHost), serviceIds: ["osrm"], owner: authority, }); diff --git a/packages/cli/__tests__/cli-control-context.test.ts b/packages/cli/__tests__/cli-control-context.test.ts index 40374159f..f59dfbf19 100644 --- a/packages/cli/__tests__/cli-control-context.test.ts +++ b/packages/cli/__tests__/cli-control-context.test.ts @@ -98,7 +98,6 @@ it.each([0, 1, 130])( plan: [plan], }); const lease = await c.acquireArtifactTargetLease({ - dataRoot: join(infra, "data"), serviceIds: ["motis"], owner: authority, }); diff --git a/packages/cli/__tests__/cli-control.test.ts b/packages/cli/__tests__/cli-control.test.ts new file mode 100644 index 000000000..8ad19ebde --- /dev/null +++ b/packages/cli/__tests__/cli-control.test.ts @@ -0,0 +1,128 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import type { ExecutionRef } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { Command } from "commander"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { registerControlCommands } from "../src/commands/control"; +import { dockerCompose } from "../src/lib/docker"; + +vi.mock("../src/lib/docker", async (original) => ({ + ...(await original()), + dockerCompose: vi.fn(), +})); + +let root: string; +let infraDir: string; +let printed: string[]; +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "openmapx-control-")); + infraDir = join(root, "infra/docker"); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(infraDir, { recursive: true }); + vi.stubEnv("OPENMAPX_ROOT_DIR", root); + printed = []; + vi.spyOn(console, "log").mockImplementation((line: string) => { + printed.push(String(line)); + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(process, "exit").mockImplementation((code) => { + throw Error(`exit:${code}`); + }); + vi.mocked(dockerCompose) + .mockReset() + .mockResolvedValue({ + exitCode: 0, + stdout: "alpha running\n", + stderr: "", + } as Awaited>); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); +}); +async function run(...args: string[]) { + const program = new Command().exitOverride(); + registerControlCommands(program); + await expect(program.parseAsync(["node", "openmapx", "control", ...args])).rejects.toThrow( + "exit:0", + ); +} +function owner(): ExecutionRef { + return { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 60_000, + }; +} +const store = () => + control.openLeaseStore({ directory: control.leaseStoreDirectory(infraDir), purpose: "test" }); +async function held(ownerKind = "service.stop") { + const execution = owner(); + await control + .localLeaseBackend(store()) + .acquire({ owner: execution, ownerKind, targets: ["alpha"] }); + return execution; +} +const auditLog = () => + readFileSync(join(control.leaseStoreDirectory(infraDir), "control-audit.jsonl"), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + +describe("openmapx control", () => { + it("shows held leases with their owner and targets", async () => { + const execution = await held(); + await run("status"); + const text = printed.join("\n"); + expect(text).toContain(execution.operationKey); + expect(text).toContain("targets: alpha"); + }); + + it("clears the lease of an operation, prints what runs and audits it", async () => { + const execution = await held(); + await run("clear", "--op", execution.operationKey); + expect(store().list()).toEqual([]); + expect(printed.join("\n")).toContain("alpha: running"); + expect(auditLog()).toMatchObject([ + { action: "clear", operationKey: execution.operationKey, lease: { targets: ["alpha"] } }, + ]); + }); + + it("resolves an operation's unknown outcome and releases its targets", async () => { + const execution = await held("service.restart"); + await run("resolve", "--op", execution.operationKey, "--as", "failed"); + expect(store().list()).toEqual([]); + expect(auditLog()).toMatchObject([ + { action: "resolve", operationKey: execution.operationKey, as: "failed" }, + ]); + }); + + it("clears a lock by its file name and records the holder it removed", async () => { + const path = join(control.leaseStoreDirectory(infraDir), "leases.lock"); + mkdirSync(control.leaseStoreDirectory(infraDir), { recursive: true }); + const holder = { + version: 1, + purpose: "gone writer", + pid: 1, + startTime: null, + bootId: null, + pidNamespace: "elsewhere", + containerId: null, + acquiredAt: Date.now(), + }; + writeFileSync(path, JSON.stringify(holder)); + expect(services.readLockHolder(path)).toEqual(holder); + await run("clear", "--lock", "leases.lock"); + expect(services.readLockHolder(path)).toBeUndefined(); + expect(auditLog()).toMatchObject([{ action: "clear-lock", lock: "leases.lock", holder }]); + }); +}); diff --git a/packages/cli/__tests__/cli-coordinator-ready.test.ts b/packages/cli/__tests__/cli-coordinator-ready.test.ts index f8da5112c..4230aee54 100644 --- a/packages/cli/__tests__/cli-coordinator-ready.test.ts +++ b/packages/cli/__tests__/cli-coordinator-ready.test.ts @@ -53,7 +53,6 @@ it("starts an unavailable coordinator without recreation under actual existing o serviceIds: ["data-manager", "motis"], }); lease = await control.acquireArtifactTargetLease({ - dataRoot: join(host.infraDir, "data"), owner: authority, serviceIds: ["data-manager", "motis"], }); diff --git a/packages/cli/__tests__/cli-host-effect.test.ts b/packages/cli/__tests__/cli-host-effect.test.ts new file mode 100644 index 000000000..347661ba9 --- /dev/null +++ b/packages/cli/__tests__/cli-host-effect.test.ts @@ -0,0 +1,100 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as control from "@openmapx/core/build-artifacts"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { dockerCompose } from "../src/lib/docker"; +import { leasedHostEffect, withHostTargets } from "../src/lib/host-effect"; + +vi.mock("../src/lib/docker", async (original) => ({ + ...(await original()), + dockerCompose: vi.fn(), +})); + +let root: string; +let infraDir: string; +let errors: string[]; +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "openmapx-host-effect-")); + infraDir = join(root, "infra/docker"); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(infraDir, { recursive: true }); + vi.stubEnv("OPENMAPX_ROOT_DIR", root); + errors = []; + vi.spyOn(console, "error").mockImplementation((line: string) => { + errors.push(String(line)); + }); + // The agent's container is not running, so the CLI writes the store itself. + vi.mocked(dockerCompose) + .mockReset() + .mockResolvedValue({ + exitCode: 0, + stdout: "", + stderr: "", + } as Awaited>); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); +}); +const held = () => control.heldTargets(infraDir); + +describe("host effects under a lease", () => { + it("holds the targets while the effect runs and releases them after", async () => { + let during: string[] = []; + const code = await leasedHostEffect( + "services.restart", + ["community-live", "alpha"], + async () => { + during = held(); + return 0; + }, + ); + expect(code).toBe(0); + expect(during).toEqual(["alpha", "community-live"]); + expect(held()).toEqual([]); + }); + + it("refuses a held target before the effect and names the holder", async () => { + const holder = { + executor: "ops-agent" as const, + principal: "api" as const, + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 60_000, + }; + await control + .localLeaseBackend( + control.openLeaseStore({ + directory: control.leaseStoreDirectory(infraDir), + purpose: "test", + }), + ) + .acquire({ owner: holder, ownerKind: "service.update", targets: ["alpha"] }); + const effect = vi.fn(async () => 0); + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + try { + const pending = leasedHostEffect("services.stop", ["alpha"], effect); + await vi.advanceTimersByTimeAsync(6000); + expect(await pending).toBe(1); + } finally { + vi.useRealTimers(); + } + expect(effect).not.toHaveBeenCalled(); + expect(errors.join("\n")).toContain(holder.operationKey); + expect(held()).toEqual(["alpha"]); + }); + + it("releases the targets of an effect that failed and passes the failure on", async () => { + await expect( + withHostTargets({ kind: "backup.restore", targets: ["postgis"] }, async () => { + throw Error("restore failed"); + }), + ).rejects.toThrow("restore failed"); + expect(held()).toEqual([]); + }); +}); diff --git a/packages/cli/__tests__/cli-service-lifecycle.test.ts b/packages/cli/__tests__/cli-service-lifecycle.test.ts index 9d95cb849..7c307dfeb 100644 --- a/packages/cli/__tests__/cli-service-lifecycle.test.ts +++ b/packages/cli/__tests__/cli-service-lifecycle.test.ts @@ -86,7 +86,9 @@ afterEach(() => { if (root) rmSync(root, { recursive: true, force: true }); }); function expectConfigurationWriterFree() { - expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/.writer"))).toBe(false); + expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/writer.lock"))).toBe( + false, + ); expect( existsSync( join( @@ -114,11 +116,11 @@ function command(action: "start" | "update", args: string[] = [], target = "alph } function lifecycleEnvelope(name: string) { return JSON.parse( - readFileSync(join(root, "infra/docker/data/ops-agent/lifecycle", name), "utf8"), + readFileSync(join(root, "infra/docker/state/ops-agent/lifecycle", name), "utf8"), ); } function lifecycleOperationKeys() { - return readdirSync(join(root, "infra/docker/data/ops-agent/lifecycle")).map((name) => + return readdirSync(join(root, "infra/docker/state/ops-agent/lifecycle")).map((name) => name.replace(/\.json$/, ""), ); } @@ -139,8 +141,7 @@ async function retainedReceipt() { return receipts[0]!; } function targetLocks() { - const directory = join(root, "infra/docker/data/.admin-target-control/locks"); - return existsSync(directory) ? readdirSync(directory).sort() : []; + return control.heldTargets(join(root, "infra/docker")); } /** A selection whose rendered hardlink plan has one entry, so a Start issues a child to data-manager. */ async function renderWithHardlinkPlan() { @@ -313,9 +314,7 @@ describe("actual CLI service lifecycle preparation", () => { it("holds physical target ownership through target-only Start after full-selection preparation", async () => { vi.mocked(dockerComposeStream).mockImplementation(async (args) => { expect(args).toEqual(["up", "-d", "alpha"]); - expect(existsSync(join(root, "infra/docker/data/.admin-target-control/locks/alpha"))).toBe( - true, - ); + expect(targetLocks()).toContain("alpha"); expect( JSON.parse( readFileSync( @@ -331,7 +330,7 @@ describe("actual CLI service lifecycle preparation", () => { }); await expect(command("start")).rejects.toThrow("exit:0"); expect(dockerComposeStream).toHaveBeenCalledTimes(1); - expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/.writer"))).toBe( + expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/writer.lock"))).toBe( false, ); }); @@ -339,9 +338,7 @@ describe("actual CLI service lifecycle preparation", () => { const calls: string[][] = []; vi.mocked(dockerComposeStream).mockImplementation(async (args) => { calls.push([...args]); - expect(existsSync(join(root, "infra/docker/data/.admin-target-control/locks/alpha"))).toBe( - true, - ); + expect(targetLocks()).toContain("alpha"); if (args[0] === "pull") expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).toBe(seed); else { @@ -478,13 +475,13 @@ describe("actual CLI service lifecycle preparation", () => { failure: { reason: "pull", failedServiceIds: ["alpha"] }, }); expect(targetLocks()).toEqual([]); - expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/.writer"))).toBe( + expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/writer.lock"))).toBe( false, ); }); it("records failure when the execution deadline passes during pull", async () => { vi.mocked(dockerComposeStream).mockImplementation(async () => { - const directory = join(root, "infra/docker/data/ops-agent/lifecycle"), + const directory = join(root, "infra/docker/state/ops-agent/lifecycle"), envelope = JSON.parse(readFileSync(join(directory, readdirSync(directory)[0]!), "utf8")); vi.spyOn(Date, "now").mockReturnValue(envelope.seed.payload.execution.deadlineAt + 1); return 0; @@ -503,7 +500,9 @@ describe("actual CLI service lifecycle preparation", () => { }); it("keeps the original failure and warns when its receipt cannot be recorded", async () => { vi.mocked(dockerComposeStream).mockImplementation(async () => { - mkdirSync(join(root, "infra/docker/state/ops-agent/configuration/.writer"), { mode: 0o700 }); + mkdirSync(join(root, "infra/docker/state/ops-agent/configuration/writer.lock"), { + mode: 0o700, + }); return 7; }); await expect(command("update")).rejects.toThrow("exit:7"); @@ -532,7 +531,7 @@ describe("actual CLI service lifecycle preparation", () => { ); expect(receipt.finalRevisionId).not.toBe(seed); expect(targetLocks()).toEqual([]); - expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/.writer"))).toBe( + expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/writer.lock"))).toBe( false, ); }); @@ -647,7 +646,7 @@ describe("actual CLI service lifecycle preparation", () => { await expect(command("start")).rejects.toThrow("exit:0"); expect(String(refusal)).toContain("exit:1"); expect(String(refusal)).toContain(`operation ${holder}`); - expect(String(refusal)).toContain(`ops-agent/lifecycle/${holder}.json`); + expect(String(refusal)).toContain("openmapx control status"); expect(dockerComposeStream).toHaveBeenCalledTimes(1); expect(targetLocks()).toEqual([]); }); diff --git a/packages/cli/__tests__/cli-service-recovery.test.ts b/packages/cli/__tests__/cli-service-recovery.test.ts deleted file mode 100644 index 72e6f4857..000000000 --- a/packages/cli/__tests__/cli-service-recovery.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import * as c from "@openmapx/core/build-artifacts"; -import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { failedServiceLifecycle } from "../../core/src/build-artifacts/test-fixtures/service-lifecycle-recovery"; -import { recoverServiceUpdate } from "../src/lib/service-recovery"; - -let root: string; -const installations: c.ControlInstallation[] = []; -vi.setConfig({ testTimeout: 30000 }); -beforeEach(async () => { - root = mkdtempSync(join(tmpdir(), "omx-cli-recovery-")); - writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); - mkdirSync(join(root, "infra/docker/data"), { recursive: true, mode: 0o700 }); - mkdirSync(join(root, "services/alpha"), { recursive: true }); - writeFileSync( - join(root, "services/alpha/service.json"), - JSON.stringify({ - id: "alpha", - name: "Alpha", - version: "1.0.0", - quality: "built-in", - container: { image: "fixture/alpha", tag: "1" }, - }), - ); - await c.provisionControlInstallation(join(root, "infra/docker")); -}); -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllGlobals(); - vi.unstubAllEnvs(); - for (const installation of installations.splice(0)) c.closeControlInstallation(installation); - rmSync(root, { recursive: true, force: true }); -}); -it("restores only the caller-named original signed update and leaves its failed receipt unchanged", async () => { - const fixture = await failedServiceLifecycle(join(root, "infra/docker")); - installations.push(...fixture.installations); - vi.spyOn(Date, "now").mockReturnValue(fixture.execution.deadlineAt + 1); - vi.stubGlobal("fetch", fixture.recoveryFetch); - vi.stubEnv("DATA_MANAGER_URL", "http://127.0.0.1:4301"); - const result = await recoverServiceUpdate({ - rootDir: root, - operationKey: fixture.execution.operationKey, - }); - expect(result).toEqual({ - operationKey: fixture.execution.operationKey, - revisionId: fixture.revisionId, - }); - expect(readFileSync(fixture.path)).toEqual(fixture.receiptBytes); - expect(existsSync(join(root, "infra/docker/data/.admin-target-control/locks/alpha"))).toBe(false); -}); -it("rejects paths instead of looking up a different private record", async () => { - await expect( - recoverServiceUpdate({ rootDir: root, operationKey: "../../unrelated" }), - ).rejects.toThrow("Unsupported control root key"); -}); diff --git a/packages/cli/__tests__/cli-services-update.test.ts b/packages/cli/__tests__/cli-services-update.test.ts index a65f086b7..80ca6ac8d 100644 --- a/packages/cli/__tests__/cli-services-update.test.ts +++ b/packages/cli/__tests__/cli-services-update.test.ts @@ -181,7 +181,7 @@ describe("services update pull failures", () => { events.push("pull"); } else { expect(current).not.toBe(seed); - const directory = join(root, "infra/docker/data/ops-agent/lifecycle"); + const directory = join(root, "infra/docker/state/ops-agent/lifecycle"); const raw = JSON.parse(readFileSync(join(directory, readdirSync(directory)[0]!), "utf8")); const receipt = await services.readServiceLifecycleReceipt({ dataRoot: join(root, "infra/docker/data"), diff --git a/packages/cli/src/commands/backup.ts b/packages/cli/src/commands/backup.ts index beecfee82..59b96efd8 100644 --- a/packages/cli/src/commands/backup.ts +++ b/packages/cli/src/commands/backup.ts @@ -35,6 +35,7 @@ import { execa } from "execa"; import kleur from "kleur"; import { completeCliCommand } from "../lib/control-context"; import { assertCliExecutionAvailable, cliProcessOptions } from "../lib/execution-deadline"; +import { withHostTargets } from "../lib/host-effect"; import { log, table } from "../lib/output"; import { repoPaths } from "../lib/paths"; import { failRestoredJobsSql } from "../lib/restored-job-cleanup"; @@ -1742,7 +1743,10 @@ export function registerBackupCommands(program: Command): void { .option("--name ", "Backup name (default: ISO timestamp)") .action(async (options: { name?: string }) => { try { - await createBackup({ name: options.name }); + // Backups stop the services they copy, so every service is claimed. + await withHostTargets({ kind: "backup.create", targets: "all" }, () => + createBackup({ name: options.name }), + ); } catch (err) { log.err((err as Error).message); completeCliCommand(err instanceof BackupInterruptedError ? 130 : 1); @@ -1764,11 +1768,13 @@ export function registerBackupCommands(program: Command): void { .option("--stop-running", "Stop currently-running target services before restoring") .action(async (name: string, options: { services?: string[]; stopRunning?: boolean }) => { try { - await restoreBackup({ - name, - serviceIds: options.services, - stopRunning: options.stopRunning, - }); + await withHostTargets({ kind: "backup.restore", targets: "all" }, () => + restoreBackup({ + name, + serviceIds: options.services, + stopRunning: options.stopRunning, + }), + ); } catch (err) { log.err((err as Error).message); process.exit(1); diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index c473891a6..2c5121724 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -11,7 +11,7 @@ import { assertCliDeploymentSecrets } from "../lib/deployment-secret-policy"; import { dockerComposeStream } from "../lib/docker"; import { assertCliExecutionAvailable } from "../lib/execution-deadline"; import { applyGeneratedHardlinksBeforeStart } from "../lib/hardlinks"; - +import { leasedHostEffect } from "../lib/host-effect"; import { log } from "../lib/output"; import { repoPaths } from "../lib/paths"; import { @@ -526,8 +526,10 @@ export function registerComposeCommands(program: Command): void { log.dim("No enabled services to start"); process.exit(0); } - const code = await dockerComposeStream(["up", "-d", ...enabledServiceIds]); - + // Compose may also start dependencies, so every service is claimed. + const code = await leasedHostEffect("compose.up", "all", () => + dockerComposeStream(["up", "-d", ...enabledServiceIds]), + ); process.exit(code); }); @@ -598,7 +600,7 @@ export function registerComposeCommands(program: Command): void { .action(async (options: { volumes?: boolean }) => { const args = ["down"]; if (options.volumes) args.push("-v"); - const code = await dockerComposeStream(args); + const code = await leasedHostEffect("compose.down", "all", () => dockerComposeStream(args)); process.exit(code); }); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts new file mode 100644 index 000000000..7cf787f1e --- /dev/null +++ b/packages/cli/src/commands/control.ts @@ -0,0 +1,240 @@ +import { createHash } from "node:crypto"; +import { closeSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs"; +import { basename, join } from "node:path"; +import { + type LeaseOperation, + type LeaseRecord, + leaseStoreDirectory, + openLeaseStore, +} from "@openmapx/core/build-artifacts"; +import { canonicalJson, OpsClient, opsOperationSchema } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import type { Command } from "commander"; +import { runsInAgent } from "../lib/agent-integrations"; +import { completeCliCommand } from "../lib/control-context"; +import { settleMaintenanceFence } from "../lib/control-resolve"; +import { dockerCompose } from "../lib/docker"; +import { log } from "../lib/output"; +import { repoPaths } from "../lib/paths"; + +/** + * The operator exit for the update system: what the agent's store believes is + * held, and the two host-only ways to end an operation whose outcome is + * unknown. Every clear and resolve is appended to an audit file in the + * agent's state and printed. + */ +const LOCKS = { + leases: () => join(leaseStoreDirectory(repoPaths().infraDir), "leases.lock"), + writer: () => services.configurationWriterLockPath(repoPaths().infraDir), + backups: () => join(repoPaths().infraDir, "backups", ".backup-store.lock"), +} as const; + +function store() { + return openLeaseStore({ + directory: leaseStoreDirectory(repoPaths().infraDir), + purpose: "host-cli", + }); +} + +function audit(entry: Record): void { + const line = JSON.stringify({ at: new Date().toISOString(), ...entry }); + const path = join(leaseStoreDirectory(repoPaths().infraDir), "control-audit.jsonl"); + const fd = openSync(path, "a", 0o600); + try { + writeSync(fd, `${line}\n`); + fsyncSync(fd); + } finally { + closeSync(fd); + } + log.info(`control audit: ${line}`); +} + +function describe(lease: LeaseRecord): string[] { + const lines = [ + `${lease.owner.executor} ${lease.ownerKind} ${lease.owner.operationKey} (${lease.state})`, + ` lease ${lease.leaseId}, since ${new Date(lease.createdAt).toISOString()}`, + ` targets: ${lease.targets.join(", ")}`, + ]; + for (const child of lease.children) + lines.push( + ` child ${child.kind} ${child.execution.operationKey} → ${child.recipient} (${child.state}${child.settlement ? `, ${child.settlement.outcome}` : ""})`, + ); + return lines; +} + +/** What Docker says runs for the targets, printed before anything is changed. */ +async function printLive(targets: readonly string[]): Promise { + const result = await dockerCompose(["ps", "--all", "--format", "{{.Service}} {{.State}}"]); + const rows = result.exitCode === 0 ? result.stdout.split("\n").filter(Boolean) : []; + log.info("Containers of the held targets, as Docker reports them now:"); + for (const target of targets) { + const row = rows.find((value) => value.split(" ")[0] === target); + log.info(` ${target}: ${row ? row.split(" ").slice(1).join(" ") : "no container"}`); + } +} + +function findLease(operationKey: string): LeaseRecord { + const lease = store() + .list() + .find( + (value) => + value.owner.operationKey === operationKey || + value.children.some((child) => child.execution.operationKey === operationKey), + ); + if (!lease) throw new Error(`No lease names operation ${operationKey}`); + return lease; +} + +function readJournal(): { + operationKey: string; + kind: string; + state: string; + errorClass?: string; +}[] { + try { + const journal = JSON.parse( + readFileSync(services.agentStatePaths(repoPaths().infraDir).journal, "utf8"), + ) as { + jobs?: { + operationKey: string; + operation: { kind: string }; + state: string; + errorClass?: string; + }[]; + }; + return (journal.jobs ?? []).map((job) => ({ + operationKey: job.operationKey, + kind: job.operation.kind, + state: job.state, + errorClass: job.errorClass, + })); + } catch { + return []; + } +} + +export function registerControlCommands(program: Command): void { + const control = program + .command("control") + .description("Inspect and end held update-system operations (host only)"); + + control + .command("status") + .description("Show held leases, unfinished operations and lock holders") + .action(() => { + const leases = store().list(); + log.info(leases.length ? "Held leases:" : "No leases are held."); + for (const lease of leases) for (const line of describe(lease)) log.info(line); + const open = readJournal().filter( + (job) => + ["queued", "running", "termination_pending"].includes(job.state) || + job.errorClass === "recovery_required", + ); + if (open.length) { + log.info("Agent operations without a final outcome:"); + for (const job of open) + log.info(` ${job.kind} ${job.operationKey} (${job.errorClass ?? job.state})`); + } + for (const [name, path] of Object.entries(LOCKS)) { + const holder = services.readLockHolder(path()); + if (holder) + log.info( + `Lock ${name} held by ${holder.purpose} (pid ${holder.pid}, since ${new Date(holder.acquiredAt).toISOString()})`, + ); + } + completeCliCommand(0); + }); + + control + .command("clear") + .description("Drop the lease of an operation, or a lock, whose holder is gone") + .option("--op ", "Operation key that owns or appears in the lease") + .option("--lock ", `Lock to clear: ${Object.keys(LOCKS).join(", ")}`) + .action(async (options: { op?: string; lock?: string }) => { + if (options.lock) { + // A busy message names the lock by its file; the short name works too. + const path = Object.entries(LOCKS) + .map(([name, path]) => ({ name, path: path() })) + .find(({ name, path }) => options.lock === name || options.lock === basename(path))?.path; + if (!path) throw new Error(`Unknown lock ${options.lock}`); + const holder = services.clearRecoverableLock(path); + audit({ action: "clear-lock", lock: options.lock, holder: holder ?? null }); + completeCliCommand(0); + } + if (!options.op) throw new Error("Pass --op or --lock "); + const lease = findLease(options.op); + for (const line of describe(lease)) log.info(line); + await printLive(lease.targets); + const removed = await store().update((leases) => { + const index = leases.findIndex((value) => value.leaseId === lease.leaseId); + return index < 0 ? undefined : leases.splice(index, 1)[0]; + }); + audit({ action: "clear", operationKey: options.op, lease: removed ?? null }); + completeCliCommand(0); + }); + + control + .command("resolve") + .description("Record the outcome of an operation whose outcome is unknown, and release it") + .requiredOption("--op ", "Operation key that owns the lease") + .requiredOption("--as ", "committed or failed") + .action(async (options: { op: string; as: string }) => { + if (options.as !== "committed" && options.as !== "failed") + throw new Error("--as must be committed or failed"); + const lease = findLease(options.op); + if (lease.owner.operationKey !== options.op) + throw new Error("Resolve the operation that owns the lease"); + for (const line of describe(lease)) log.info(line); + await printLive(lease.targets); + // A maintenance root also holds the data-manager fence; the same idempotent + // settle the agent uses releases it. + if (lease.ownerKind === "configurationMaintenance.prepare") + await settleMaintenanceFence(repoPaths().infraDir, lease, options.as); + const outcome: "committed" | "contained" = + options.as === "committed" ? "committed" : "contained"; + const evidence = (identity: LeaseRecord["owner"]) => ({ + identity, + outcome, + publication: "committed" as const, + terminalReceiptDigest: createHash("sha256") + .update( + canonicalJson({ + kind: "operator-resolution", + operationKey: options.op, + as: options.as, + }), + ) + .digest("hex"), + }); + await store().update((leases) => { + const current = leases.find((value) => value.leaseId === lease.leaseId); + if (!current) return; + for (const child of current.children) + if (child.state !== "settled") { + child.state = "settled"; + child.settlement = evidence(child.execution); + } + current.state = "settled"; + current.settlement = evidence(current.owner); + leases.splice(leases.indexOf(current), 1); + }); + audit({ action: "resolve", operationKey: options.op, as: options.as, lease }); + completeCliCommand(0); + }); + + // The host CLI's transport to the agent: run as the fixed CLI in the agent's + // container, with the host-CLI token the agent created. + control.command("lease ", { hidden: true }).action(async (request: string) => { + if (!runsInAgent()) throw new Error("Runs only in the operations agent's container"); + const operation = opsOperationSchema.parse(JSON.parse(request)) as LeaseOperation; + if (!operation.kind.startsWith("lease.")) throw new Error("Lease operations only"); + const client = new OpsClient({ + baseUrl: `http://127.0.0.1:${process.env.PORT?.trim() || "4300"}`, + tokenFile: services.hostCliTokenPath(repoPaths().infraDir), + }); + const submitted = await client.execute(operation); + if (submitted.execution !== "sync") throw new Error("Lease operations answer at once"); + process.stdout.write(`${JSON.stringify(submitted.value)}\n`); + completeCliCommand(0); + }); +} diff --git a/packages/cli/src/commands/services.ts b/packages/cli/src/commands/services.ts index bb4d6180b..9fc1f30f9 100644 --- a/packages/cli/src/commands/services.ts +++ b/packages/cli/src/commands/services.ts @@ -5,12 +5,12 @@ import { cliControlContext, completeCliCommand } from "../lib/control-context"; import { dockerComposeStream } from "../lib/docker"; import { applyGeneratedHardlinks, logAppliedHardlinks } from "../lib/hardlinks"; +import { leasedHostEffect } from "../lib/host-effect"; import { log, table } from "../lib/output"; import { repoPaths } from "../lib/paths"; import { expandPresets, UnknownPresetError } from "../lib/presets"; import { buildServices } from "../lib/service-builds"; import { runCliServiceLifecycle, ServiceLifecycleCommandError } from "../lib/service-lifecycle"; -import { recoverServiceUpdate } from "../lib/service-recovery"; import { applyServiceSelection, disableSelectedServices, @@ -188,25 +188,6 @@ function mergeIdsWithPresets(ids: string[], preset: string | undefined): string[ export function registerServicesCommands(program: Command): void { const services = program.command("services").description("Manage services"); - services - .command("recover-update") - .description( - "Settle an expired original failed update only when its hardlink child was never adopted", - ) - .requiredOption("--operation-key ", "Original host service-update operation key") - .action(async (options: { operationKey: string }) => { - try { - const result = await recoverServiceUpdate(options); - log.info( - `Recovered original failed lifecycle ${result.operationKey}; configuration ${result.revisionId} is preserved. No update was replayed.`, - ); - } catch (error) { - log.err(error instanceof Error ? error.message : "Original service update remains fenced"); - completeCliCommand(1); - } - completeCliCommand(0); - }); - services .command("selected") .description("Show requested and effective selected services") @@ -424,8 +405,11 @@ export function registerServicesCommands(program: Command): void { log.err("No services selected. Pass or --preset."); completeCliCommand(1); } - const code = await dockerComposeStream(["stop", ...allIds]); - completeCliCommand(code); + completeCliCommand( + await leasedHostEffect("services.stop", allIds, () => + dockerComposeStream(["stop", ...allIds]), + ), + ); }); services @@ -440,8 +424,11 @@ export function registerServicesCommands(program: Command): void { log.err("No services selected. Pass or --preset."); completeCliCommand(1); } - const code = await dockerComposeStream(["restart", ...allIds]); - completeCliCommand(code); + completeCliCommand( + await leasedHostEffect("services.restart", allIds, () => + dockerComposeStream(["restart", ...allIds]), + ), + ); }); services diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 4b055110b..cbabd9931 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -5,6 +5,7 @@ import { registerBackupCommands } from "./commands/backup"; import { registerCacheCommands } from "./commands/cache"; import { registerCheckCommand } from "./commands/check"; import { registerComposeCommands } from "./commands/compose"; +import { registerControlCommands } from "./commands/control"; import { registerDataCommands } from "./commands/data"; import { registerExtCommands } from "./commands/ext"; import { registerIntegrationsCommands } from "./commands/integrations"; @@ -34,6 +35,7 @@ registerServicesCommands(program); registerSystemCommands(program); registerIntegrationsCommands(program); registerComposeCommands(program); +registerControlCommands(program); registerDataCommands(program); registerExtCommands(program); registerBackupCommands(program); diff --git a/packages/cli/src/lib/artifact-ownership.ts b/packages/cli/src/lib/artifact-ownership.ts index 8b2667e63..a39329491 100644 --- a/packages/cli/src/lib/artifact-ownership.ts +++ b/packages/cli/src/lib/artifact-ownership.ts @@ -13,6 +13,7 @@ import { import { services } from "@openmapx/core/server"; import { cliControlContext, pruneHostControlRecords } from "./control-context"; import { cliAcquisitionOptions, cliProcessOptions } from "./execution-deadline"; +import { hostCliLeases } from "./host-leases"; import { repoPaths } from "./paths"; type ArtifactOperation = Extract< @@ -35,6 +36,7 @@ export async function openCliArtifactOwner(operation: ArtifactOperation, rootDir infraDir: paths.infraDir, role: "host-cli", serviceIds: targets, + leases: hostCliLeases(paths.infraDir), }); let lease: control.TargetLeaseAuthority | undefined; try { @@ -86,7 +88,6 @@ export async function openCliArtifactOwner(operation: ArtifactOperation, rootDir }); if (hardlinks) control.retainGenerationHardlinksOwnerPlan(authority, hardlinks); lease = await control.acquireArtifactTargetLease({ - dataRoot: join(paths.infraDir, "data"), serviceIds: targets, owner: authority, signal, diff --git a/packages/cli/src/lib/control-context.ts b/packages/cli/src/lib/control-context.ts index 05477e423..2fdca6908 100644 --- a/packages/cli/src/lib/control-context.ts +++ b/packages/cli/src/lib/control-context.ts @@ -94,7 +94,6 @@ export async function withCliControlContext( if (delegation) { parent = verifiedParentForArtifactDelegation(installation, assertion, execution); lease = await adoptArtifactTargetLease({ - dataRoot: join(infraDir, "data"), assertion, peer, execution, diff --git a/packages/cli/src/lib/control-resolve.ts b/packages/cli/src/lib/control-resolve.ts new file mode 100644 index 000000000..e680be014 --- /dev/null +++ b/packages/cli/src/lib/control-resolve.ts @@ -0,0 +1,84 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { LeaseRecord } from "@openmapx/core/build-artifacts"; +import { + createDataManagerTransport, + DataManagerHttpError, + settleDataConfigurationFence, +} from "@openmapx/core/data-manager-sdk"; +import { z } from "zod"; +import { dataManagerUrl } from "./data-manager-url"; + +/** One of the agent's own records, as it keeps them in its private state. */ +function agentRecord(infraDir: string, namespace: string, name: string): unknown { + const raw = JSON.parse( + readFileSync(join(infraDir, "control-private", "ops-agent", namespace, name), "utf8"), + ) as { payload?: unknown }; + return raw.payload ?? raw; +} + +/** + * Settles the data-manager fence of a maintenance the operator resolves, with + * the agent's own data-manager token and the original preparation delegation, + * as the agent itself would. A fence that was never prepared needs nothing. + */ +export async function settleMaintenanceFence( + infraDir: string, + lease: LeaseRecord, + as: "committed" | "failed", +): Promise { + const fence = lease.children.find((child) => child.phase === "configuration-fence"); + if (!fence) return; + const admission = z + .object({ operation: z.object({ maintenanceId: z.uuid() }).passthrough() }) + .passthrough() + .parse(agentRecord(infraDir, "executions", `${lease.owner.operationKey}.json`)); + const owner = z + .object({ + plan: z + .object({ + operation: z.object({ maintenanceId: z.uuid(), candidateId: z.uuid() }).passthrough(), + planDigest: z.string(), + }) + .passthrough(), + }) + .passthrough() + .parse( + agentRecord( + infraDir, + "configuration-maintenance", + `${admission.operation.maintenanceId}.json`, + ), + ); + const attempt = z + .object({ assertion: z.string() }) + .passthrough() + .parse(agentRecord(infraDir, "attempts", `${fence.key}.json`)); + const token = readFileSync( + join(infraDir, "secrets", "data-manager-ops-agent-token"), + "utf8", + ).trim(); + try { + await settleDataConfigurationFence( + { + maintenanceId: owner.plan.operation.maintenanceId, + candidateId: owner.plan.operation.candidateId, + planDigest: owner.plan.planDigest, + outcome: as === "committed" ? "applied" : "contained", + }, + { + transport: createDataManagerTransport({ + baseUrl: dataManagerUrl(), + authToken: token, + requestTimeoutMs: 15000, + maxJsonBytes: 65536, + }), + controlAssertion: attempt.assertion, + }, + ); + } catch (error) { + // No fence row: the preparation never reached data-manager. + if (error instanceof DataManagerHttpError && error.status === 404) return; + throw error; + } +} diff --git a/packages/cli/src/lib/data-execution.ts b/packages/cli/src/lib/data-execution.ts index 0771da29a..f8988137f 100644 --- a/packages/cli/src/lib/data-execution.ts +++ b/packages/cli/src/lib/data-execution.ts @@ -46,6 +46,7 @@ import { withHostDataControlContext, } from "./control-context"; import { cliAcquisitionOptions } from "./execution-deadline"; +import { hostCliLeases } from "./host-leases"; import { repoPaths } from "./paths"; export async function executeDurableDataStep( input: DataExecutionInput, @@ -258,6 +259,7 @@ export async function runCliDataStep( infraDir: repoPaths().infraDir, role: "host-cli", serviceIds: ["data-manager"], + leases: hostCliLeases(repoPaths().infraDir), }); await pruneHostControlRecords(installation); const normalized = normalizeExecutionAdmission({ diff --git a/packages/cli/src/lib/data-update-command.ts b/packages/cli/src/lib/data-update-command.ts index 6c0403d75..625a3d2c4 100644 --- a/packages/cli/src/lib/data-update-command.ts +++ b/packages/cli/src/lib/data-update-command.ts @@ -17,6 +17,7 @@ import { runOwnedDataStep } from "./data-execution"; import { DataUpdateFailure, runDataUpdate } from "./data-update"; import { cliAcquisitionOptions } from "./execution-deadline"; import { readGeneratedHardlinkPlan } from "./hardlinks"; +import { hostCliLeases } from "./host-leases"; import { log } from "./output"; import { repoPaths } from "./paths"; import { buildServices, ServiceBuildExecutionError } from "./service-builds"; @@ -44,6 +45,7 @@ export async function runCliDataUpdate(options: { infraDir: paths.infraDir, role: "host-cli", serviceIds: targets, + leases: hostCliLeases(paths.infraDir), }); await pruneHostControlRecords(installation); const operation = { @@ -140,7 +142,6 @@ export async function runCliDataUpdate(options: { control.assertDataUpdateSources(authority); if (!lease) lease = await control.acquireArtifactTargetLease({ - dataRoot: join(paths.infraDir, "data"), serviceIds: control.controlRegisteredServiceIds(context.installation), owner: authority, signal: cliAcquisitionOptions().signal, diff --git a/packages/cli/src/lib/host-effect.ts b/packages/cli/src/lib/host-effect.ts new file mode 100644 index 000000000..ec1c03314 --- /dev/null +++ b/packages/cli/src/lib/host-effect.ts @@ -0,0 +1,100 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { TargetBusyError } from "@openmapx/core/build-artifacts"; +import { + canonicalJson, + createControlRootOperationKey, + type ExecutionRef, + executionFingerprint, +} from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; +import { runsInAgent } from "./agent-integrations"; +import { hostCliLeases } from "./host-leases"; +import { log } from "./output"; +import { repoPaths } from "./paths"; + +export class HostTargetsBusyError extends Error {} + +/** + * Runs one host Docker effect that changes containers (stop, restart, compose + * up or down, a backup that stops services) under a lease on its targets, the + * same admission path the agent uses. A held target refuses it before the + * effect, naming the holder. A finished effect releases the lease whether it + * succeeded or failed; a CLI killed during the effect leaves it held for + * `openmapx control`. + */ +export async function withHostTargets( + input: { kind: string; targets: readonly string[] | "all"; budgetMs?: number }, + run: () => Promise, +): Promise { + // A fixed command the agent started runs under the lease the agent holds. + if (runsInAgent() && process.env.OPENMAPX_CONTROL_EXECUTION) return run(); + const targets = input.targets === "all" ? await registeredServiceIds() : input.targets; + const leases = hostCliLeases(repoPaths().infraDir); + const deadlineAt = Date.now() + (input.budgetMs ?? 2 * 3600_000); + const body = { kind: input.kind, targets: [...new Set(targets)].sort() }; + const operationFingerprint = createHash("sha256").update(canonicalJson(body)).digest("hex"); + const owner: ExecutionRef = { + executor: "host-cli", + principal: "host-cli", + executionId: randomUUID(), + operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), + operationFingerprint, + deadlineAt, + fingerprint: executionFingerprint({ kind: input.kind, operationFingerprint, deadlineAt }), + }; + let lease: Awaited>; + try { + lease = await leases.acquire({ owner, ownerKind: input.kind, targets: body.targets }); + } catch (error) { + if (error instanceof TargetBusyError) throw new HostTargetsBusyError(error.message); + throw error; + } + const finish = async (outcome: "committed" | "contained", receipt: unknown) => { + await leases.settle({ + leaseId: lease.leaseId, + key: null, + settlement: { + identity: owner, + outcome, + publication: "committed", + terminalReceiptDigest: createHash("sha256").update(canonicalJson(receipt)).digest("hex"), + }, + }); + await leases.release({ leaseId: lease.leaseId }); + }; + let result: T; + try { + result = await run(); + } catch (error) { + // The effect ran to its end and failed: its outcome is known. + await finish("contained", { kind: input.kind, failed: true }); + throw error; + } + await finish("committed", { kind: input.kind, failed: false }); + return result; +} + +/** Every registered service, for effects that may touch any container. */ +export async function registeredServiceIds(): Promise { + const registry = new services.ServiceRegistry({ rootDir: repoPaths().root }); + await registry.load(); + return registry + .list() + .map((service) => service.manifest.id) + .sort(); +} + +/** A host command's Docker effect under a lease; a held target exits 1 with the holder named. */ +export async function leasedHostEffect( + kind: string, + targets: readonly string[] | "all", + run: () => Promise, +): Promise { + try { + return await withHostTargets({ kind, targets }, run); + } catch (error) { + if (!(error instanceof HostTargetsBusyError)) throw error; + log.err(error.message); + return 1; + } +} diff --git a/packages/cli/src/lib/host-leases.ts b/packages/cli/src/lib/host-leases.ts new file mode 100644 index 000000000..3c57feac2 --- /dev/null +++ b/packages/cli/src/lib/host-leases.ts @@ -0,0 +1,54 @@ +import { + agentStoreLeases, + type LeaseBackend, + type LeaseOperation, + remoteLeaseBackend, +} from "@openmapx/core/build-artifacts"; +import { agentContainerRunning, runsInAgent } from "./agent-integrations"; +import { dockerCompose } from "./docker"; + +/** Runs one lease operation through the fixed CLI in the agent's container. */ +async function throughAgentContainer(operation: LeaseOperation): Promise { + const result = await dockerCompose([ + "exec", + "-T", + "ops-agent", + "node", + "dist/cli.mjs", + "control", + "lease", + JSON.stringify(operation), + ]); + if (result.exitCode !== 0) + throw new Error( + result.stderr.trim() || "The operations agent could not answer the lease operation", + ); + return JSON.parse(result.stdout); +} + +/** + * The host CLI's leases. While the agent runs they go through it, as the + * fixed CLI in its container with the host-CLI token, so the agent stays the + * only writer of its store. While it is stopped the CLI writes the store under + * its recoverable lock. Inside the agent's container the store is used directly. + */ +export function hostCliLeases(infraDir: string): LeaseBackend { + let chosen: Promise | undefined; + const backend = () => { + chosen ??= (async () => + runsInAgent() || !(await agentContainerRunning()) + ? agentStoreLeases(infraDir, "host-cli") + : remoteLeaseBackend(throughAgentContainer))(); + return chosen; + }; + return { + acquire: async (input) => (await backend()).acquire(input), + read: async (leaseId) => (await backend()).read(leaseId), + list: async () => (await backend()).list(), + delegate: async (input) => (await backend()).delegate(input), + adopt: async (input) => (await backend()).adopt(input), + settle: async (input) => (await backend()).settle(input), + cancel: async (input) => (await backend()).cancel(input), + release: async (input) => (await backend()).release(input), + }; +} diff --git a/packages/cli/src/lib/service-lifecycle.ts b/packages/cli/src/lib/service-lifecycle.ts index 8df15c4a4..54810b408 100644 --- a/packages/cli/src/lib/service-lifecycle.ts +++ b/packages/cli/src/lib/service-lifecycle.ts @@ -176,6 +176,7 @@ export async function runCliServiceLifecycle(options: { infraDir: paths.infraDir, role: "host-cli", serviceIds: targets, + leases: (await import("./host-leases")).hostCliLeases(paths.infraDir), }); if (!inherited) await pruneHostControlRecords(installation); const admittedAt = Date.now(), @@ -400,7 +401,6 @@ export async function runCliServiceLifecycle(options: { lease = inherited?.lease ?? (await control.acquireArtifactTargetLease({ - dataRoot: join(paths.infraDir, "data"), owner: authority, serviceIds: targets, signal: inner.signal, diff --git a/packages/cli/src/lib/service-recovery.ts b/packages/cli/src/lib/service-recovery.ts deleted file mode 100644 index 144ec460e..000000000 --- a/packages/cli/src/lib/service-recovery.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { - closeControlInstallation, - openControlInstallation, - recoverServiceLifecycleByKey, -} from "@openmapx/core/build-artifacts"; -import { createDataManagerTransport } from "@openmapx/core/data-manager-sdk"; -import { services } from "@openmapx/core/server"; -import { cliControlContext } from "./control-context"; -import { cliDataManagerToken } from "./data-manager-token"; -import { dataManagerUrl } from "./data-manager-url"; -import { repoPaths } from "./paths"; - -export async function recoverServiceUpdate(options: { rootDir?: string; operationKey: string }) { - if (cliControlContext()) throw Error("Recovery requires the installation host"); - const paths = repoPaths(options.rootDir), - registry = new services.ServiceRegistry({ rootDir: paths.root }); - await registry.load(); - const installation = openControlInstallation({ - infraDir: paths.infraDir, - role: "host-cli", - serviceIds: registry - .list() - .map((service) => service.manifest.id) - .sort(), - }); - try { - return await recoverServiceLifecycleByKey({ - infraDir: paths.infraDir, - installation, - operationKey: options.operationKey, - transport: createDataManagerTransport({ - baseUrl: dataManagerUrl(), - authToken: cliDataManagerToken(), - requestTimeoutMs: 15000, - maxJsonBytes: 65536, - }), - }); - } finally { - closeControlInstallation(installation); - } -} diff --git a/packages/cli/src/lib/transitous-api-keys.ts b/packages/cli/src/lib/transitous-api-keys.ts index 89b36493e..60f37df11 100644 --- a/packages/cli/src/lib/transitous-api-keys.ts +++ b/packages/cli/src/lib/transitous-api-keys.ts @@ -32,6 +32,7 @@ import { import { execa } from "execa"; import { cliControlContext, pruneHostControlRecords } from "./control-context"; import { cliProcessOptions } from "./execution-deadline"; +import { hostCliLeases } from "./host-leases"; import { repoPaths } from "./paths"; import { ensurePlatformPrivateDirectory, @@ -331,6 +332,7 @@ export async function generateTransitousApiKeys( infraDir: paths.infraDir, role: "host-cli", serviceIds: ["motis"], + leases: hostCliLeases(paths.infraDir), }); let lease: control.ArtifactTargetLease | undefined; let authority: control.VerifiedExecutionAuthority | undefined; @@ -366,7 +368,6 @@ export async function generateTransitousApiKeys( serviceIds: ["motis"], }); lease = await control.acquireArtifactTargetLease({ - dataRoot: join(paths.infraDir, "data"), serviceIds: ["motis"], owner: authority, }); diff --git a/packages/core/src/build-artifacts/child-phase.test.ts b/packages/core/src/build-artifacts/child-phase.test.ts index ee23b2d6f..04bcaf170 100644 --- a/packages/core/src/build-artifacts/child-phase.test.ts +++ b/packages/core/src/build-artifacts/child-phase.test.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes, randomUUID } from "node:crypto"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; @@ -11,9 +11,16 @@ import { } from "../ops/execution-identity"; import { canonicalJson, opsOperationFingerprint } from "../ops/fingerprint"; import { inspectControlChildPublication } from "./child-phase"; +import { installationState } from "./control-authority"; import * as control from "./index"; afterEach(() => vi.restoreAllMocks()); +/** The leases the agent's store holds for the installation at `root`. */ +function heldLeases(root: string) { + return control + .openLeaseStore({ directory: control.leaseStoreDirectory(root), purpose: "test" }) + .list(); +} it("admits a120s fixed phase after hours of original-root work without a new root", async () => { const root = mkdtempSync(join(tmpdir(), "omx-late-phase-")); mkdirSync(join(root, "data"), { mode: 0o700 }); @@ -81,7 +88,6 @@ it("admits a120s fixed phase after hours of original-root work without a new roo reservations: [reservation], }); const lease = await control.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -89,7 +95,7 @@ it("admits a120s fixed phase after hours of original-root work without a new roo const original = readFileSync( join(root, "control-private/data-manager/executions", `${owner.operationKey}.json`), ); - const held = statSync(join(root, "data/.admin-target-control/locks/motis")).ino; + const held = heldLeases(root); const late = admittedAt + 4 * 3600000; vi.spyOn(Date, "now").mockReturnValue(late); const normalized = normalizeExecutionAdmission({ @@ -123,7 +129,9 @@ it("admits a120s fixed phase after hours of original-root work without a new roo join(root, "control-private/data-manager/executions", `${owner.operationKey}.json`), ), ).toEqual(original); - expect(statSync(join(root, "data/.admin-target-control/locks/motis")).ino).toBe(held); + expect(heldLeases(root).map((value) => value.leaseId)).toEqual( + held.map((value) => value.leaseId), + ); expect(control.cancelControlExecution(authority)).toEqual([bound.execution]); await expect( control.issueControlChildPhase(authority, { @@ -280,14 +288,7 @@ it("keeps an old exact admission's signed bytes when a duplicate supplies empty import { AnchoredDirectory } from "./filesystem"; -it.each([ - "reference", - "cancellation-index", - "lease-outbox", - "grant-outbox", - "shared-grant", - "parent-linkage", -] as const)( +it.each(["reference", "cancellation-index", "lease-outbox", "grant-outbox"] as const)( "retains the exact phase and refuses another dispatch after a crash at %s", async (point) => { const f = await fixture(), @@ -297,7 +298,6 @@ it.each([ serviceIds: ["motis", "motis-staging"], }), lease = await control.acquireArtifactTargetLease({ - dataRoot: join(f.root, "data"), serviceIds: ["motis", "motis-staging"], owner: f.authority, }); @@ -325,13 +325,7 @@ it.each([ (point === "lease-outbox" && path.endsWith("/parent-attempts") && text.includes('"kind":"lease"')) || - (point === "grant-outbox" && path.endsWith("/attempts")) || - (point === "shared-grant" && path.endsWith("/grants")) || - (point === "parent-linkage" && - path.endsWith(`/${lease.leaseId}`) && - name === "owner.json" && - text.includes('"grantDigests":{') && - !text.includes('"grantDigests":{}'))); + (point === "grant-outbox" && path.endsWith("/attempts"))); const value = Reflect.apply(original, this, args); if (selected) { fired = true; @@ -370,9 +364,7 @@ it.each([ /* Torn private/shared publication is itself a closed recovery fence. */ } expect(() => control.assertControlExecutionForward(f.authority)).toThrow(); - expect(statSync(join(f.root, "data/.admin-target-control/locks/motis")).isDirectory()).toBe( - true, - ); + expect(heldLeases(f.root).map((value) => value.leaseId)).toEqual([lease.leaseId]); await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); } finally { vi.restoreAllMocks(); @@ -436,7 +428,6 @@ it("rechecks durable cancellation inside publication after the outside identity serviceIds: ["motis", "motis-staging"], }), lease = await control.acquireArtifactTargetLease({ - dataRoot: join(f.root, "data"), serviceIds: ["motis", "motis-staging"], owner: f.authority, }); @@ -444,16 +435,16 @@ it("rechecks durable cancellation inside publication after the outside identity try { const phase = { phase: "stop-primary" as const, ordinal: 0 }; await control.bindControlChildPhase(f.authority, phase); - const original = AnchoredDirectory.prototype.name; - vi.spyOn(AnchoredDirectory.prototype, "name").mockImplementation(function ( - this: AnchoredDirectory, - name: string, - ) { - if (!fired && name === ".journal-writer" && this.path.endsWith(`/${lease.leaseId}`)) { + // Cancel right after delegation read the lease from the store. + const leases = installationState(f.dm).leases, + read = leases.read.bind(leases); + vi.spyOn(leases, "read").mockImplementation(async (leaseId) => { + const value = await read(leaseId); + if (!fired) { fired = true; control.cancelControlExecution(f.authority); } - return original.call(this, name); + return value; }); await expect( control.issueControlChildPhase(f.authority, { @@ -722,11 +713,7 @@ it.each(["cancel-first", "grant-first"] as const)( let fired = false; try { const bound = await control.bindControlChildPhase(f.cliAuthority, f.slot); - expect( - (await import("node:fs")).existsSync( - join(f.root, "data/.admin-target-control/locks/data-manager"), - ), - ).toBe(false); + expect(heldLeases(f.root)).toEqual([]); if (order === "cancel-first") { const original = AnchoredDirectory.prototype.name; vi.spyOn(AnchoredDirectory.prototype, "name").mockImplementation(function ( diff --git a/packages/core/src/build-artifacts/child-phase.ts b/packages/core/src/build-artifacts/child-phase.ts index 4aa7e29d8..df26bbc5d 100644 --- a/packages/core/src/build-artifacts/child-phase.ts +++ b/packages/core/src/build-artifacts/child-phase.ts @@ -383,9 +383,10 @@ export function lookupControlChildPhaseAttempt( ); if (controlDigest(saved.grant) !== reference.grantDigest) throw Error("Original phase outbox digest changed"); - if (channel === "delegation") { + // A role that reaches the lease store only through the agent relies on its + // own outbox; the agent checked the row when it recorded it. + if (channel === "delegation" && s.installation.leases.readSync) { const observed = inspectArtifactDelegation({ - dataRoot: s.installation.dataRoot, assertion: saved.assertion, installation: s.installation.installation, }); @@ -473,6 +474,8 @@ export async function issueControlChildPhase( ordinal: number; childPublicKey: string; lease?: TargetLeaseAuthority; + /** What the child acts on, recorded with its grant for the recipient. */ + evidence?: Record; }, ) { const child = lookupControlChildPhase(authority, input); @@ -489,6 +492,7 @@ export async function issueControlChildPhase( ? await delegateArtifactTargetLease(input.lease, { ...child, childPublicKey: input.childPublicKey, + ...(input.evidence ? { evidence: input.evidence } : {}), }) : await attestParentExecution(authority, { ...child, childPublicKey: input.childPublicKey }); const saved = inspectControlChildPublication(authority, input); diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts index 921916918..f55260adc 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -98,7 +98,6 @@ export const configurationHandoffIntentRecordSchema = z .refine((names) => Buffer.byteLength(JSON.stringify(names)) <= 8192) .refine((names) => names.every((name, index) => index === 0 || names[index - 1]! < name)), preparerAdmissionDigest: digest, - ownerCertificateDigest: digest, leaseId: z.uuid(), delegationId: z.uuid(), assertion: z.string().min(1).max(4096), diff --git a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts index fdd54d911..f44f9e1e4 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts @@ -38,7 +38,7 @@ export const configurationOwnerIdentitySchema = z origin: controlOriginSchema, ownerPrepare: executionRefSchema, admissionDigest: digest, - rootCertificateDigest: digest, + leaseId: id, maintenanceId: id, candidateId: id, planDigest: digest, @@ -317,7 +317,7 @@ export function assertUnusedConfigurationMaintenanceFinalState( !same(original.identity.origin, owner.origin) || !same(original.identity.ownerPrepare, owner.plan.prepare) || original.identity.admissionDigest !== owner.admissionDigest || - original.identity.rootCertificateDigest !== owner.rootCertificateDigest || + original.identity.leaseId !== owner.leaseId || original.identity.maintenanceId !== owner.plan.operation.maintenanceId || original.identity.candidateId !== owner.plan.operation.candidateId || original.identity.planDigest !== owner.plan.planDigest || @@ -536,7 +536,7 @@ function assertExecutedConfigurationMaintenanceFinalState( !same(original.identity.origin, owner.origin) || !same(original.identity.ownerPrepare, owner.plan.prepare) || original.identity.admissionDigest !== owner.admissionDigest || - original.identity.rootCertificateDigest !== owner.rootCertificateDigest || + original.identity.leaseId !== owner.leaseId || original.identity.maintenanceId !== owner.plan.operation.maintenanceId || original.identity.candidateId !== owner.plan.operation.candidateId || original.identity.planDigest !== owner.plan.planDigest || diff --git a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts deleted file mode 100644 index aba655844..000000000 --- a/packages/core/src/build-artifacts/configuration-maintenance-forward-records.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** Shared non-secret original-source copies. Decoding grants no authority. */ -import { z } from "zod"; -import { configurationMaintenanceReceiptSchema } from "../ops/configuration-maintenance"; -import { type OpsJobStatus, parseOpsJobStatus } from "../ops/contract"; -import { executionRefSchema } from "../ops/execution-ref"; -import { configurationMaintenanceDecisionCommitmentSchema } from "./configuration-maintenance-owner-records"; -import { fixedConfigurationMaintenancePlanSchema } from "./fixed-configuration-maintenance"; - -export const configurationMaintenanceForwardDomain = - "openmapx-configuration-maintenance-forward-v1"; -const signed = z.strictObject({ - payload: z.unknown(), - signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), -}); -const digest = z.string().regex(/^[a-f0-9]{64}$/); -export const configurationMaintenanceForwardGraphSchema = z.strictObject({ - version: z.literal(1), - binding: z.lazy(() => configurationMaintenanceReceiptSchema.shape.binding), - prepare: executionRefSchema, - apiJobId: z.uuid(), - parentOperationKey: z.string(), - parentFingerprint: digest, - parentDeadlineAt: z.number().int().positive().safe(), - planDigest: digest, - coordinatorImage: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.coordinatorImage), - handoff: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.handoff), - release: z.lazy(() => fixedConfigurationMaintenancePlanSchema.shape.release), - serviceIds: z.lazy(() => configurationMaintenanceReceiptSchema.shape.serviceIds), - steps: z.lazy(() => configurationMaintenanceReceiptSchema.shape.steps), -}); -export const configurationMaintenanceForwardSchema = z.strictObject({ - version: z.literal(1), - graph: configurationMaintenanceForwardGraphSchema, - rootAdmission: signed, - decision: z.lazy(() => configurationMaintenanceDecisionCommitmentSchema).nullable(), - generation: z - .strictObject({ - admission: signed, - terminal: signed, - effect: signed, - status: z - .unknown() - .transform((value) => parseOpsJobStatus(value) as OpsJobStatus) - .refine((value) => value.diagnostics === undefined), - }) - .nullable(), -}); diff --git a/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts index b6a81a8a2..def68eb47 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-owner-records.ts @@ -138,7 +138,7 @@ export const configurationMaintenanceOwnerRecordSchema = z.strictObject({ installationId: z.uuid(), origin: controlOriginSchema, admissionDigest: digest, - rootCertificateDigest: digest, + leaseId: z.uuid(), apiParentAttestationDigest: digest, plan: z.lazy(() => fixedConfigurationMaintenancePlanSchema), originalContainers: z.array(ownedServiceSnapshotSchema).max(256), @@ -269,7 +269,7 @@ export function configurationMaintenanceDecisionCommitmentPayload( origin: owner.origin, ownerPrepare: owner.plan.prepare, admissionDigest: owner.admissionDigest, - rootCertificateDigest: owner.rootCertificateDigest, + leaseId: owner.leaseId, maintenanceId: owner.plan.operation.maintenanceId, candidateId: owner.plan.operation.candidateId, planDigest: owner.plan.planDigest, diff --git a/packages/core/src/build-artifacts/configuration-pre-effect.test.ts b/packages/core/src/build-artifacts/configuration-pre-effect.test.ts index f58945a1d..fd53f864d 100644 --- a/packages/core/src/build-artifacts/configuration-pre-effect.test.ts +++ b/packages/core/src/build-artifacts/configuration-pre-effect.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it } from "vitest"; @@ -27,6 +27,7 @@ import { retainControlTerminalCommitment, verifyControlEffectSettlement, } from "./control-authority"; +import { leaseStoreDirectory, openLeaseStore } from "./lease-store"; import { acquireArtifactTargetLease, releaseArtifactTargetLease, @@ -104,7 +105,6 @@ function start() { it("settles the actual unissued root and consumes only its indexed not-started effect", async () => { const owner = start(), lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: targets, owner, }); @@ -115,7 +115,7 @@ it("settles the actual unissued root and consumes only its indexed not-started e verifyControlEffectSettlement(installation, owner.execution), ); await releaseArtifactTargetLease(lease); - verifyRetainedArtifactTargetRootRelease(owner); + await verifyRetainedArtifactTargetRootRelease(owner); expect(() => beginControlEffect(owner)).toThrow(); }); it.each(["begun", "bound", "issued"])( @@ -133,7 +133,6 @@ it.each(["begun", "bound", "issued"])( true, ), lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: targets, owner, }); @@ -150,8 +149,12 @@ it.each(["begun", "bound", "issued"])( } expect(() => retainConfigurationPreEffectRefusal(owner)).toThrow(); expect(() => inspectConfigurationPreEffectRefusal(owner)).toThrow(); - expect(() => verifyRetainedArtifactTargetRootRelease(owner)).toThrow(); - expect(existsSync(join(root, "data/.admin-target-control/locks/data-manager"))).toBe(true); + await expect(verifyRetainedArtifactTargetRootRelease(owner)).rejects.toThrow(); + expect( + openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }) + .list() + .map((value) => value.leaseId), + ).toEqual([lease.leaseId]); }, ); it.each(["exact", "different error", "missing"] as const)( diff --git a/packages/core/src/build-artifacts/control-authority.test.ts b/packages/core/src/build-artifacts/control-authority.test.ts index 9adfec4c6..02f47bc19 100644 --- a/packages/core/src/build-artifacts/control-authority.test.ts +++ b/packages/core/src/build-artifacts/control-authority.test.ts @@ -193,19 +193,24 @@ it("retains deadline-only attempts separately from execution witnesses", async ( ...plan, childPublicKey: receiver.publicKey, }); - const peer = a.verifyControlPeer(receiver, { - rawHeaders: ["x-openmapx-parent-execution", assertion], + // data-manager trusts the agent's token and reads the self-contained grant. + const verified = a.verifiedRemoteParentExecution(receiver, assertion, { transportPrincipal: "ops-agent", - execution: child, + operationKey: child.operationKey, + operationFingerprint: hash, kind: body.kind, + requestedDeadlineAt: deadlineAt, }); - expect( - a.verifyParentExecution(assertion, peer, { - execution: child, + expect(verified.parent).toMatchObject({ operationKey: parent.operationKey }); + expect(() => + a.verifiedRemoteParentExecution(receiver, assertion, { + transportPrincipal: "host-cli", + operationKey: child.operationKey, + operationFingerprint: hash, kind: body.kind, requestedDeadlineAt: deadlineAt, }), - ).toMatchObject({ operationKey: parent.operationKey }); + ).toThrow(/mismatch/); await expect( a.attestParentExecution(authority, { ...plan, childPublicKey: receiver.publicKey }), ).rejects.toThrow(/replay/); @@ -393,28 +398,19 @@ it("verifies the certified ephemeral fixed-CLI deadline chain without exporting ...leaf, childPublicKey: dm.publicKey, }); - const dmPeer = c.verifyControlPeer(dm, { - rawHeaders: ["x-openmapx-parent-execution", next], - transportPrincipal: "ops-agent", - execution: child, - kind: body.kind, - }); - expect( - c.verifyParentExecution(next, dmPeer, { - execution: child, + const remoteParent = () => + c.verifiedRemoteParentExecution(dm, next, { + transportPrincipal: "ops-agent", + operationKey: child.operationKey, + operationFingerprint: childHash, kind: body.kind, requestedDeadlineAt: deadlineAt, - }), - ).toMatchObject({ operationKey: cli.operationKey }); + }).parent; + expect(remoteParent()).toMatchObject({ operationKey: cli.operationKey }); c.cancelControlExecution(authority); expect(() => c.assertControlExecutionForward(cliAuthority)).toThrow(/ancestor|observation/); - expect( - c.verifyParentExecution(next, dmPeer, { - execution: child, - kind: body.kind, - requestedDeadlineAt: deadlineAt, - }), - ).toMatchObject({ operationKey: cli.operationKey }); + // An exact retry of the admitted child stays readable after the cancellation. + expect(remoteParent()).toMatchObject({ operationKey: cli.operationKey }); c.closeControlInstallation(fixed); c.closeControlInstallation(dm); c.closeControlInstallation(agent); @@ -482,16 +478,13 @@ async function deadlineFamily() { }); const grant = () => c.attestParentExecution(authority, { ...plan, childPublicKey: dm.publicKey }); const admit = (assertion: string) => - c.verifyParentExecution( - assertion, - c.verifyControlPeer(dm, { - rawHeaders: ["x-openmapx-parent-execution", assertion], - transportPrincipal: "ops-agent", - execution: child, - kind: plan.kind, - }), - { execution: child, kind: plan.kind, requestedDeadlineAt: deadlineAt }, - ); + c.verifiedRemoteParentExecution(dm, assertion, { + transportPrincipal: "ops-agent", + operationKey: child.operationKey, + operationFingerprint: hash, + kind: plan.kind, + requestedDeadlineAt: deadlineAt, + }).parent; return { c, agent, dm, authority, execution, child, grant, admit }; } it("terminal deadline-only parent cannot mint an unattempted child", async () => { @@ -508,32 +501,16 @@ it("terminal deadline-only parent cannot mint an unattempted child", async () => f.c.closeControlInstallation(f.agent); } }); -it("terminal deadline-only parent fences a previously attempted unadopted grant", async () => { - const f = await deadlineFamily(); - try { - const assertion = await f.grant(); - f.c.retainControlEffectReceipt(f.authority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - expect(() => f.admit(assertion)).toThrow(/terminal|observation/); - expect(existsSync(join(root, "data/.admin-target-control/locks"))).toBe(false); - } finally { - f.c.closeControlInstallation(f.dm); - f.c.closeControlInstallation(f.agent); - } -}); it.each([false, true])( - "canceled deadline-only parent fences unadopted attempts (attempted: %s)", + "canceled deadline-only parent returns its attempts and mints no more (attempted: %s)", async (attempted) => { const f = await deadlineFamily(); try { - const assertion = attempted ? await f.grant() : undefined; + if (attempted) await f.grant(); + // The parent cancels the returned children over data-manager's route. const children = f.c.cancelControlExecution(f.authority); expect(children).toHaveLength(attempted ? 1 : 0); - if (assertion) expect(() => f.admit(assertion)).toThrow(/cancel|forward|observation/); - else await expect(f.grant()).rejects.toThrow(/cancel|forward/); + await expect(f.grant()).rejects.toThrow(/cancel|forward|replay/); expect(() => f.c.beginControlEffect(f.authority)).toThrow(/cancel|forward/); } finally { f.c.closeControlInstallation(f.dm); @@ -541,37 +518,6 @@ it.each([false, true])( } }, ); -it("retained deadline-only duplicates observe after cancellation but cannot start work", async () => { - const f = await deadlineFamily(); - try { - const assertion = await f.grant(); - const parent = f.admit(assertion); - f.c.cancelControlExecution(f.authority); - expect(f.admit(assertion)).toEqual(parent); - const body = { kind: "download-osm", regionId: "europe/germany" }; - const authority = f.c.retainAdmittedControlExecution(f.dm, { - requestedDeadlineAt: f.child.deadlineAt, - execution: f.child, - operation: body, - kind: body.kind, - admittedAt: f.execution.deadlineAt - 60000, - parent, - serviceIds: ["osrm"], - }); - expect(() => f.c.beginControlEffect(authority)).toThrow(/cancel|forward|observation/); - const { acquireArtifactTargetLease } = await import("./target-lease"); - await expect( - acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["osrm"], - owner: authority, - }), - ).rejects.toThrow(/cancel|ancestor|observation/); - } finally { - f.c.closeControlInstallation(f.dm); - f.c.closeControlInstallation(f.agent); - } -}); it("expired exact deadline-only adoption stays observable without renewing one-use work", async () => { const f = await deadlineFamily(); diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 3cd750815..46fd8bab0 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -28,6 +28,8 @@ export { controlChildOperationKey, } from "./control-child-records"; +// biome-ignore lint/suspicious/noImportCycles: The failed receipt is read after initialization, never during module evaluation. +import { readFailedServiceLifecycleReceipt } from "../services/service-lifecycle-receipt"; import { type ControlInstallation, controlDigest, @@ -41,13 +43,14 @@ import { verifyControl, } from "./control-records"; // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. +import { leaseStoreDirectory, openLeaseStore } from "./lease-store"; import { closeArtifactTargetLeaseHandle, - inspectArtifactTargetLeaseChildSettlement, + inspectLocalLeaseChildSettlement, + type LeaseBackend, + localLeaseBackend, restoreArtifactTargetLeaseForSettlement, - verifyFailedServiceLifecycleContainment, - verifyRetainedArtifactTargetRootRelease, - verifyRetainedGenerationHardlinksSettlement, + verifyLocalArtifactTargetRootRelease, } from "./target-lease"; // biome-ignore-end lint/suspicious/noImportCycles: Same exact deferred guard import. @@ -122,11 +125,6 @@ import { resolveControlChildPlan, validateControlChildReservations, } from "./child-phase"; -import { - configurationMaintenanceForwardDomain, - configurationMaintenanceForwardGraphSchema, - configurationMaintenanceForwardSchema, -} from "./configuration-maintenance-forward-records"; import { assertConfigurationMaintenanceDecisionSource, configurationMaintenanceDecisionCommitmentDomain, @@ -147,7 +145,6 @@ import { // biome-ignore-start lint/suspicious/noImportCycles: Fixed proof verification uses this module's original nominal authority and effect writer after initialization. import { inspectFeedProxyValidationFailure, - publishFeedProxyCanonicalTerminal, recordFeedProxyReloadAttempted, } from "./data-execution-candidate"; // biome-ignore-end lint/suspicious/noImportCycles: Fixed live proof dependency described above. @@ -236,8 +233,6 @@ export async function provisionControlInstallation(infraDir: string): Promise [role, newControlKey()])); for (const role of CONTROL_ROLES) { @@ -293,6 +288,12 @@ export async function provisionControlInstallation(infraDir: string): Promise(); +/** The agent's own lease store, as its processes and the host CLI see it. */ +export function agentStoreLeases(infraDir: string, role: ControlIssuer): LeaseBackend { + return localLeaseBackend( + openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: role }), + ); +} export function installationState(value: ControlInstallation): InstallationState { const state = installations.get(value); if (!state) throw Error("Unverified control installation"); @@ -305,6 +306,8 @@ export function openControlInstallation(options: { role: ControlIssuer; serviceIds?: readonly string[]; container?: boolean; + /** Data-manager and the host CLI pass the agent's routes; others use its store. */ + leases?: LeaseBackend; }): ControlInstallation { if (options.serviceIds?.length) targetSchema.parse([...new Set(options.serviceIds)].sort()); const infra = options.container ? undefined : new AnchoredDirectory(options.infraDir); @@ -377,6 +380,7 @@ export function openControlInstallation(options: { privateRoot, dataRoot, registeredTargets: new Set(options.serviceIds ?? []), + leases: options.leases ?? agentStoreLeases(options.infraDir, options.role), }); privateRoot = undefined; return installation; @@ -639,9 +643,7 @@ export function retainControlTerminalStatus( const authority = restoreAdmittedControlExecution(installation, execution), s = executionAuthorityState(authority); withPrivateEffectWriter(s, () => { - const proof = verifyAndSealTerminalCommitment(s, status, terminalAt); - if (s.kind === "feedProxy.validateAndReload") - publishFeedProxyCanonicalTerminal(authority, status, proof.terminalAt); + verifyAndSealTerminalCommitment(s, status, terminalAt); }); } /** Rejoin a terminal worker from its original signed admission and known effect @@ -716,9 +718,7 @@ export function repairControlJournalTerminal( canonicalJson(proof.receipt); if (!succeeded && !failedWithoutEffect && !validationFailed && !generationNotCommitted) throw Error("Terminal worker containment is not proved"); - const committed = verifyAndSealTerminalCommitment(s, terminal, terminalAt); - if (s.kind === "feedProxy.validateAndReload") - publishFeedProxyCanonicalTerminal(authority, terminal, committed.terminalAt); + verifyAndSealTerminalCommitment(s, terminal, terminalAt); }); return authority; } @@ -1092,16 +1092,89 @@ export function controlPeerState(peer: VerifiedControlPeer) { if (!s) throw Error("Unverified control peer"); return s; } -export const assertionSchema = z.strictObject({ - version: z.literal(1), - installationId: uuid, - leaseId: uuid, - ownerCertificateDigest: digest, - delegationId: uuid, - grantDigest: digest, - ancestorDelegationIds: z.array(uuid).max(2), - signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +const delegatedRowSchema = z.strictObject({ + key: uuid, + parentKey: uuid.nullable(), + parent: executionRefSchema, + execution: executionRefSchema, + kind: z.string().max(128), + phase: z.string().max(64), + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), + targets: targetSchema, + permittedFurtherEdges: z.array(z.string().max(64)).max(1), + depth: z.number().int().min(1).max(2), }); +/** + * A child row of the agent's lease store, sent to the child's recipient. A + * recipient that reads the store checks the row there; data-manager cannot and + * trusts the caller it authenticated by token instead. + */ +const assertionSchema = z + .strictObject({ + version: z.literal(2), + installationId: uuid, + leaseId: uuid, + delegationId: uuid, + grantDigest: digest, + owner: executionRefSchema, + origin: controlOriginSchema, + grant: delegatedRowSchema, + }) + .refine( + (value) => + value.grant.key === value.delegationId && controlDigest(value.grant) === value.grantDigest, + ); +export function encodeControlAssertion( + fields: Omit, "version" | "grantDigest">, +): ArtifactDelegationAssertion { + return Buffer.from( + canonicalJson( + assertionSchema.parse({ version: 2, ...fields, grantDigest: controlDigest(fields.grant) }), + ), + ).toString("base64url") as ArtifactDelegationAssertion; +} +/** + * The delegation data-manager received from the agent or the host CLI. The + * caller's token says who sent it; data-manager never reads the lease store. + */ +export function inspectRemoteDelegation( + installation: ControlInstallation, + rawHeaders: readonly string[], + transportPrincipal: ControlTransportPrincipal, +) { + if (installation.role !== "data-manager") throw Error("Data-manager receiver required"); + const { delegation, parent } = controlHeaders(rawHeaders); + if (!delegation || parent) throw Error("Delegation required"); + const parsed = parseControlAssertion(delegation), + issuer = parsed.grant.parent.executor; + if ( + parsed.grant.recipient !== "data-manager" || + parsed.grant.execution.executor !== "data-manager" || + !( + (transportPrincipal === "ops-agent" && (issuer === "ops-agent" || issuer === "host-cli")) || + (transportPrincipal === "host-cli" && issuer === "host-cli") + ) + ) + throw Error("Delegation caller mismatch"); + const verifiedParent = Object.freeze( + retainedControlParent(parsed.grant.parent), + ) as import("../ops/execution-identity").VerifiedParentExecution; + verifiedParents.add(verifiedParent); + verifiedParentOrigins.set(verifiedParent, parsed.origin); + return Object.freeze({ + assertion: delegation, + leaseId: parsed.leaseId, + delegationId: parsed.delegationId, + owner: parsed.owner, + origin: parsed.origin, + parent: parsed.grant.parent, + verifiedParent, + execution: parsed.grant.execution, + kind: parsed.grant.kind, + phase: parsed.grant.phase, + serviceIds: Object.freeze([...parsed.grant.targets]), + }); +} export function parseControlAssertion(assertion: string) { if (Buffer.byteLength(assertion) > 4096 || !/^[A-Za-z0-9_-]+$/.test(assertion)) throw Error("Invalid control assertion encoding/limit"); @@ -1189,13 +1262,13 @@ export function verifyRetainedLocalControlPeer( const saved = z .object({ assertion: z.string(), - grant: z.object({ payload: z.object({ child: executionRefSchema, childKind: z.string() }) }), + grant: z.object({ execution: executionRefSchema, kind: z.string() }), }) .parse(readPrivateRecord(state, "attempts", `${assertion.delegationId}.json`)); if ( saved.assertion !== input.assertion || - saved.grant.payload.childKind !== input.kind || - controlDigest(saved.grant.payload.child) !== controlDigest(input.execution) + saved.grant.kind !== input.kind || + controlDigest(saved.grant.execution) !== controlDigest(input.execution) ) throw Error("Local continuation intent mismatch"); const peer = Object.freeze({}) as VerifiedControlPeer; @@ -1390,7 +1463,7 @@ function verifyConfigurationFenceNotPrepared(authority: VerifiedExecutionAuthori if (!binding) throw Error("Original fence preparation binding required"); const lease = restoreArtifactTargetLeaseForSettlement(authority); try { - const settled = inspectArtifactTargetLeaseChildSettlement(lease, binding.child.execution); + const settled = inspectLocalLeaseChildSettlement(lease, binding.child.execution); if (settled?.outcome !== "failed_without_effect") throw Error("Data-manager fence refusal required"); } finally { @@ -1770,8 +1843,11 @@ export function retainControlEffectReceipt( outcome = "committed"; publication = "committed"; } else if (receipt.kind === "service-lifecycle-contained-failure") { - const source = verifyFailedServiceLifecycleContainment(authority); - if (source.failedReceiptDigest !== receipt.failedReceiptDigest) + // The lease refuses release while a child it issued is unsettled, so the + // failed receipt alone names what this failure contained. + if ( + controlDigest(readFailedServiceLifecycleReceipt(authority)) !== receipt.failedReceiptDigest + ) throw Error("Original failed lifecycle source changed"); outcome = "contained"; publication = "committed"; @@ -1839,7 +1915,7 @@ export function retainControlEffectReceipt( ) throw Error("Bound data command identity required"); verifyDeadlineEdge(s.kind, "data-step", "data-manager", receipt.status.kind); - verifyDataControlTerminalWitness( + readDataControlTerminalObservation( s.installation.installation, bound.execution, receipt.status, @@ -1935,7 +2011,7 @@ export function retainControlEffectReceipt( const binding = lookupControlChildPhase(authority, slot); if ( !binding || - verifyDataControlTerminalWitness(s.installation.installation, binding.execution).status + readDataControlTerminalObservation(s.installation.installation, binding.execution).status .status !== "ok" ) throw Error("Actual backend data containment required before fixed command success"); @@ -1945,7 +2021,7 @@ export function retainControlEffectReceipt( const bound = lookupControlChildPhase(authority, slot); if ( !bound || - verifyDataControlTerminalWitness(s.installation.installation, bound.execution).status + readDataControlTerminalObservation(s.installation.installation, bound.execution).status .status !== "ok" ) throw Error("Actual Full Update hardlink containment required"); @@ -2135,7 +2211,7 @@ export function verifyMaintenanceCoordinatorPrerequisite( .changed !== true ) throw Error("Conclusive successful coordinator prerequisite required"); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); return Object.freeze({ execution: Object.freeze(admitted.execution), terminalDigest: canonical.terminalDigest, @@ -2351,7 +2427,7 @@ export function ownedStagedCoordinatorBaseline( verified = configurationMaintenanceChildState(child, authority), lease = restoreArtifactTargetLeaseForSettlement(authority); try { - const physical = inspectArtifactTargetLeaseChildSettlement(lease, generation.execution); + const physical = inspectLocalLeaseChildSettlement(lease, generation.execution); if (!physical || canonicalJson(physical) !== canonicalJson(verified.settlement)) throw Error("Actual staged generation settlement required"); } finally { @@ -2614,414 +2690,6 @@ export function observeCoordinatorPullForMaintenance( observedBaselineProofs.set(witness, proof); return witness; } -/** Copies actual retained source packets into the original owner record. - * The initial empty projection is allocated before the DM guard. No caller - * supplies a decision, generation result, signature or authority digest. */ -export function publishConfigurationMaintenanceForwardWitness( - authority: VerifiedExecutionAuthority, -): void { - const root = executionAuthorityState(authority); - if ( - root.kind !== "configurationMaintenance.prepare" || - root.installation.installation.role !== "ops-agent" || - root.installation.fixedChildId - ) - throw Error("Original maintenance forward publisher required"); - const raw = readPrivateRecord( - root.installation, - "executions", - `${root.execution.operationKey}.json`, - 65536, - ) as { payload?: { operation?: { maintenanceId?: string } } }, - name = `${uuid.parse(raw.payload?.operation?.maintenanceId)}.json`, - ownerRaw = readPrivateRecord(root.installation, "configuration-maintenance", name), - owner = verifyControl( - root.installation.installation.publicKey, - "openmapx-configuration-maintenance-owner-v2", - ownerRaw, - configurationMaintenanceOwnerRecordSchema, - ); - assertConfigurationMaintenanceDecisionSource(owner); - if ( - owner.admissionDigest !== root.receiptDigest || - controlDigest(raw) !== root.receiptDigest || - canonicalJson(owner.plan.prepare) !== canonicalJson(root.execution) || - canonicalJson(owner.origin) !== canonicalJson(root.origin) - ) - throw Error("Original maintenance forward source changed"); - const receipt = owner.receipt, - graph = configurationMaintenanceForwardGraphSchema.parse({ - version: 1, - binding: receipt.binding, - prepare: receipt.prepare, - apiJobId: receipt.apiJobId, - parentOperationKey: receipt.parentOperationKey, - parentFingerprint: receipt.parentFingerprint, - parentDeadlineAt: receipt.parentDeadlineAt, - planDigest: receipt.planDigest, - ...(owner.plan.coordinatorImage ? { coordinatorImage: owner.plan.coordinatorImage } : {}), - ...(owner.plan.handoff ? { handoff: owner.plan.handoff } : {}), - ...(owner.plan.release ? { release: owner.plan.release } : {}), - serviceIds: receipt.serviceIds, - steps: receipt.steps, - }); - const decision = owner.apiControls.decide?.commitment ?? null; - if (decision) - verifyControl( - root.installation.installation.publicKey, - configurationMaintenanceDecisionCommitmentDomain, - decision, - configurationMaintenanceDecisionCommitmentPayloadSchema, - ); - const source = owner.apiControls.steps.find((value) => value.ordinal === 0); - let generation = null; - if (source?.canonical && decision) { - verifyConfigurationMaintenanceChildTerminal( - authority, - 0, - source.canonical.status, - source.canonical.terminalAt, - ); - generation = { - admission: readPrivateRecord( - root.installation, - "executions", - `${source.execution.operationKey}.json`, - 65536, - ), - terminal: readPrivateRecord( - root.installation, - "observations", - `terminal-${source.execution.operationKey}.json`, - 8192, - ), - effect: readPrivateRecord( - root.installation, - "effects", - `${source.execution.operationKey}.json`, - ), - status: source.canonical.status, - }; - } - const packet = signControl( - root.installation.privateKey, - configurationMaintenanceForwardDomain, - configurationMaintenanceForwardSchema.parse({ - version: 1, - graph, - rootAdmission: raw, - decision, - generation, - }), - ), - bytes = canonicalJson(packet); - if (Buffer.byteLength(bytes) > 65536) - throw Error("Configuration maintenance forward witness capacity"); - const directory = executionWitnessDirectory(root.installation, "ops-agent", true); - try { - directory.role.write(`${root.execution.operationKey}.configuration-maintenance.json`, bytes); - } finally { - directory.close(); - } -} -/** DM reads ONLY shared original signed packets and its pinned role keys. - * This does not inspect the sibling agent's private directory. The actual - * verified transport parent and anchored depth-two grant must match the - * exact original root graph, continue decision and committed generation. */ -export function verifyConfigurationMaintenanceGenerationWitness( - installation: ControlInstallation, - value: unknown, - parent: import("../ops/execution-identity").VerifiedParentExecution, - grant: ReturnType, - expected: { - maintenanceId: string; - candidateId: string; - ownerPrepare: ExecutionRef; - revisionId: string; - hardlinkPlanDigest: string; - coordinatorMode: "ensure-running" | "require-running"; - serviceIds: readonly string[]; - rootCertificateDigest: string; - }, -) { - const local = installationState(installation), - request = startGenerationHardlinksExecutionSchema.parse(value); - if ( - installation.role !== "data-manager" || - local.fixedChildId || - !verifiedParents.has(parent) || - parent.issuer !== "ops-agent" || - !parent.execution || - canonicalJson(parent.execution) !== canonicalJson(grant.parent) || - canonicalJson(grant.owner) !== canonicalJson(expected.ownerPrepare) || - grant.ownerCertificateDigest !== expected.rootCertificateDigest - ) - throw Error("Actual original maintenance middle parent required"); - const directory = executionWitnessDirectory(local, "ops-agent"); - try { - const packet = verifyControl( - local.publicKeys["ops-agent"], - configurationMaintenanceForwardDomain, - JSON.parse( - directory.role - .read(`${expected.ownerPrepare.operationKey}.configuration-maintenance.json`, 65536) - .toString("utf8"), - ), - configurationMaintenanceForwardSchema, - ), - graph = packet.graph; - const admissionSchema = z - .object({ - version: z.literal(2), - installationId: uuid, - origin: controlOriginSchema, - execution: executionRefSchema, - kind: z.string(), - operation: opsOperationSchema, - parent: retainedParentExecutionSchema, - admittedAt: z.number().int().nonnegative().safe(), - requestedDeadlineAt: z.number().int().positive().safe(), - serviceIds: targetSchema, - plan: z.array(z.unknown()), - reservations: z.array(z.lazy(() => controlChildReservationSchema)).optional(), - }) - .passthrough(); - const root = verifyControl( - local.publicKeys["ops-agent"], - "openmapx-control-execution-v2", - packet.rootAdmission, - admissionSchema, - ), - original = readControlExecutionWitness( - local, - expected.ownerPrepare, - local.publicKeys["ops-agent"], - ), - operation = root.operation, - first = graph.steps[0]; - if ( - !packet.decision || - !packet.generation || - operation.kind !== "configurationMaintenance.prepare" || - !first || - root.installationId !== installation.installationId || - root.kind !== operation.kind || - root.parent.issuer !== "api" || - original.authorityDigest !== controlDigest(packet.rootAdmission) || - canonicalJson(root.origin) !== canonicalJson(original.origin) || - canonicalJson(root.execution) !== canonicalJson(expected.ownerPrepare) || - canonicalJson(graph.prepare) !== canonicalJson(root.execution) || - opsOperationFingerprint(operation) !== root.execution.operationFingerprint || - root.execution.fingerprint !== - executionFingerprint({ - kind: operation.kind, - operationFingerprint: root.execution.operationFingerprint, - deadlineAt: root.execution.deadlineAt, - parent: root.parent, - }) || - root.parent.apiJobId !== graph.apiJobId || - root.parent.operationKey !== graph.parentOperationKey || - root.parent.fingerprint !== graph.parentFingerprint || - root.parent.deadlineAt !== graph.parentDeadlineAt || - operation.maintenanceId !== expected.maintenanceId || - operation.candidateId !== expected.candidateId || - operation.revisionId !== expected.revisionId || - operation.coordinatorMode !== expected.coordinatorMode || - request.input.planDigest !== expected.hardlinkPlanDigest || - request.input.revisionId !== expected.revisionId || - request.input.coordinatorMode !== expected.coordinatorMode || - canonicalJson(root.serviceIds) !== canonicalJson(expected.serviceIds) || - canonicalJson(graph.serviceIds) !== canonicalJson(root.serviceIds) - ) - throw Error("Original maintenance forward graph identity conflict"); - const { kind: _kind, ...binding } = operation; - if (canonicalJson(binding) !== canonicalJson(graph.binding)) - throw Error("Original maintenance graph binding conflict"); - // Private candidate digests/source metadata are deliberately NOT shared. - // These placeholders do not enter the graph digest or reservations. - const rebuilt = buildFixedConfigurationMaintenancePlan({ - installationId: installation.installationId, - prepare: root.execution, - operation, - apiParent: root.parent, - apply: first.operation as Parameters< - typeof buildFixedConfigurationMaintenancePlan - >[0]["apply"], - candidateRecordDigest: "0".repeat(64), - sourceAuthorityRevision: "not-shared", - previewDigest: "0".repeat(64), - ...(graph.coordinatorImage ? { coordinatorImage: graph.coordinatorImage } : {}), - ...(graph.release ? { release: graph.release } : {}), - ...(graph.handoff ? { handoff: graph.handoff } : {}), - hardlinkPlan: request.plan, - serviceIds: root.serviceIds, - recreateServiceIds: graph.steps.flatMap((step) => - step.operation.kind === "service.recreate" || - step.operation.kind === "service.recreateIsolated" - ? [step.operation.serviceId] - : [], - ), - }); - const middle = rebuilt.steps.find((step) => step.phase === "maintenance-hardlinks"), - middleReservation = rebuilt.reservations.find( - (step) => step.phase === "maintenance-hardlinks", - ); - if ( - !middle || - !middleReservation || - !rebuilt.nestedHardlinks || - graph.planDigest !== rebuilt.planDigest || - canonicalJson(graph.steps) !== canonicalJson(rebuilt.steps) || - canonicalJson(root.reservations) !== canonicalJson(rebuilt.reservations) || - parent.execution.operationKey !== middle.operationKey || - parent.execution.operationFingerprint !== middle.operationFingerprint || - parent.execution.deadlineAt > root.execution.deadlineAt || - parent.fingerprint !== parent.execution.fingerprint || - parent.execution.fingerprint !== - executionFingerprint({ - kind: middle.operation.kind, - operationFingerprint: middle.operationFingerprint, - deadlineAt: parent.execution.deadlineAt, - parent: retainedControlParent(root.execution), - }) || - grant.phase !== "hardlinks" || - grant.kind !== "generation-hardlinks" || - grant.recipient !== "data-manager" || - canonicalJson(grant.execution) !== - canonicalJson({ - ...rebuilt.nestedHardlinks.execution, - deadlineAt: grant.execution.deadlineAt, - operationFingerprint: rebuilt.nestedHardlinks.operationFingerprint, - fingerprint: grant.execution.fingerprint, - }) || - grant.execution.operationKey !== request.operationKey || - grant.execution.operationFingerprint !== controlDigest(request.input) || - grant.execution.deadlineAt !== request.requestedDeadlineAt || - grant.execution.deadlineAt > parent.deadlineAt || - grant.execution.fingerprint !== - executionFingerprint({ - kind: "generation-hardlinks", - operationFingerprint: controlDigest(request.input), - deadlineAt: grant.execution.deadlineAt, - parent, - }) || - canonicalJson(grant.serviceIds) !== canonicalJson(root.serviceIds) - ) - throw Error("Original frozen middle/sole leaf conflict"); - const decision = verifyControl( - local.publicKeys["ops-agent"], - configurationMaintenanceDecisionCommitmentDomain, - packet.decision, - configurationMaintenanceDecisionCommitmentPayloadSchema, - ), - identity = decision.identity; - if ( - identity.installationId !== installation.installationId || - canonicalJson(identity.origin) !== canonicalJson(root.origin) || - canonicalJson(identity.ownerPrepare) !== canonicalJson(root.execution) || - identity.admissionDigest !== controlDigest(packet.rootAdmission) || - identity.rootCertificateDigest !== expected.rootCertificateDigest || - identity.maintenanceId !== expected.maintenanceId || - identity.candidateId !== expected.candidateId || - identity.planDigest !== graph.planDigest || - decision.operation.decision !== "continue" || - decision.operation.maintenanceId !== expected.maintenanceId || - decision.operation.candidateId !== expected.candidateId || - decision.operation.planDigest !== graph.planDigest || - decision.operation.prepareOperationKey !== root.execution.operationKey || - decision.operation.prepareFingerprint !== root.execution.operationFingerprint || - decision.apiParent.apiJobId !== graph.apiJobId || - decision.apiParent.operationKey !== graph.parentOperationKey || - decision.apiParent.fingerprint !== graph.parentFingerprint || - decision.apiParent.deadlineAt !== graph.parentDeadlineAt || - decision.execution.operationFingerprint !== opsOperationFingerprint(decision.operation) || - decision.execution.fingerprint !== - executionFingerprint({ - kind: decision.operation.kind, - operationFingerprint: decision.execution.operationFingerprint, - deadlineAt: decision.execution.deadlineAt, - parent: decision.apiParent, - }) || - decision.acceptedAt >= decision.execution.deadlineAt || - decision.result.decision?.decision !== "continue" || - decision.result.decision.decisionId !== decision.operation.decisionId || - decision.result.state !== "authorized" || - decision.result.dmFence?.state !== "prepared" - ) - throw Error("Actual original continue commitment required"); - const generation = packet.generation, - admitted = verifyControl( - local.publicKeys["ops-agent"], - "openmapx-control-execution-v2", - generation.admission, - admissionSchema, - ), - childWitness = readControlExecutionWitness( - local, - admitted.execution, - local.publicKeys["ops-agent"], - ), - terminal = verifyControl( - local.publicKeys["ops-agent"], - "openmapx-control-terminal-v1", - generation.terminal, - terminalCommitmentSchema, - ), - effect = verifyControl( - local.publicKeys["ops-agent"], - "openmapx-control-settlement-v1", - generation.effect, - settlementSchema.extend({ receipt: effectObservationSchema }), - ), - status = generation.status; - if ( - childWitness.authorityDigest !== controlDigest(generation.admission) || - admitted.installationId !== installation.installationId || - canonicalJson(admitted.origin) !== canonicalJson(root.origin) || - canonicalJson(admitted.operation) !== canonicalJson(first.operation) || - canonicalJson(admitted.parent) !== canonicalJson(retainedControlParent(root.execution)) || - admitted.execution.operationKey !== first.operationKey || - admitted.execution.operationFingerprint !== first.operationFingerprint || - admitted.execution.deadlineAt > root.execution.deadlineAt || - admitted.execution.fingerprint !== - executionFingerprint({ - kind: first.operation.kind, - operationFingerprint: first.operationFingerprint, - deadlineAt: admitted.execution.deadlineAt, - parent: admitted.parent, - }) || - terminal.phase !== "canonical" || - terminal.installationId !== installation.installationId || - canonicalJson(terminal.origin) !== canonicalJson(root.origin) || - canonicalJson(terminal.execution) !== canonicalJson(admitted.execution) || - terminal.admissionDigest !== controlDigest(generation.admission) || - terminal.terminalDigest !== controlTerminalReceiptDigest(status) || - canonicalJson(effect.identity) !== canonicalJson(admitted.execution) || - effect.outcome !== "committed" || - effect.publication !== "committed" || - effect.terminalReceiptDigest !== controlDigest(effect.receipt) || - effect.receipt.kind !== "ops-result" || - canonicalJson(parseBoundedOpsResult(first.operation.kind, effect.receipt.result)) !== - canonicalJson(status.result) || - status.state !== "succeeded" || - status.kind !== first.operation.kind || - status.operationId !== admitted.execution.executionId || - status.operationKey !== admitted.execution.operationKey || - status.fingerprint !== admitted.execution.fingerprint || - status.deadlineAt !== admitted.execution.deadlineAt || - (status.result as { outcome?: string; revisionId?: string })?.outcome !== "committed" || - (status.result as { revisionId?: string })?.revisionId !== expected.revisionId - ) - throw Error("Actual original committed generation required"); - return Object.freeze({ - witnessDigest: controlDigest(packet), - decisionId: decision.operation.decisionId, - generation: Object.freeze(admitted.execution), - }); - } finally { - directory.close(); - } -} declare const maintenanceChildBrand: unique symbol; export type VerifiedConfigurationMaintenanceChild = Readonly<{ [maintenanceChildBrand]: true }>; const maintenanceChildren = new WeakMap< @@ -3230,14 +2898,24 @@ export const controlAdoptionSchema = z.strictObject({ }); export type ControlAdoption = z.infer; +/** What a recipient in another role needs to know about the parent: it trusts + * the parent's authenticated caller, never a file of the parent's role. */ +const parentSummarySchema = z.strictObject({ + origin: controlOriginSchema, + parent: executionRefSchema, + child: executionRefSchema, + kind: z.string().max(128), + phase: z.enum(["data-workflow", "osm", "transit", "fonts", "data-step"]), + ordinal: z.number().int().min(0).max(63), +}); const parentAssertionSchema = z.strictObject({ - version: z.literal(1), + version: z.literal(2), installationId: uuid, issuer: z.enum(CONTROL_ROLES), operationKey: z.string().regex(/^opk1_[A-Za-z0-9_-]{16,64}$/), attemptId: uuid, grantDigest: digest, - signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), + grant: parentSummarySchema, }); const parentGrantSchema = z.strictObject({ version: z.literal(2), @@ -3253,6 +2931,79 @@ const parentGrantSchema = z.strictObject({ parentAssertion: z.string().max(4096).optional(), ordinal: z.number().int().min(0).max(63), }); +function parentSummary(grant: z.infer) { + return parentSummarySchema.parse({ + origin: grant.origin, + parent: grant.parent, + child: grant.child, + kind: grant.kind, + phase: grant.phase, + ordinal: grant.ordinal, + }); +} +/** Whether the parent's records are in this role's own private state. */ +function parentReadable(state: InstallationState, issuer: ControlIssuer): boolean { + return ( + issuer === state.installation.role || (Boolean(state.fixedChildId) && issuer === "ops-agent") + ); +} +/** + * The parent of a data-manager execution named by an agent or host CLI caller. + * Data-manager cannot read the caller's records; it trusts the caller it + * authenticated by token, and the parent cancels its children explicitly. + */ +export function verifiedRemoteParentExecution( + installation: ControlInstallation, + assertion: string, + input: { + transportPrincipal: ControlTransportPrincipal; + operationKey: string; + operationFingerprint: string; + kind: string; + requestedDeadlineAt?: number; + }, +): { + parent: import("../ops/execution-identity").VerifiedParentExecution; + execution: ExecutionRef; +} { + const parsed = parseParentAssertion(assertion), + grant = parsed.grant; + if ( + installation.role !== "data-manager" || + parsed.issuer !== grant.parent.executor || + !( + (input.transportPrincipal === "ops-agent" && + (grant.parent.executor === "ops-agent" || grant.parent.executor === "host-cli")) || + (input.transportPrincipal === "host-cli" && grant.parent.executor === "host-cli") + ) || + grant.child.operationKey !== input.operationKey || + grant.child.operationFingerprint !== input.operationFingerprint || + grant.kind !== input.kind || + grant.child.deadlineAt !== input.requestedDeadlineAt || + grant.child.deadlineAt > grant.parent.deadlineAt || + grant.child.operationKey !== + controlChildOperationKey(grant.parent.operationKey, grant.phase, grant.ordinal) || + !childPhaseKinds[grant.phase].includes(grant.kind) + ) + throw Error("Parent child identity mismatch"); + const parent = Object.freeze( + retainedControlParent(grant.parent), + ) as import("../ops/execution-identity").VerifiedParentExecution; + verifiedParents.add(parent); + verifiedParentOrigins.set(parent, grant.origin); + const local = installationState(installation), + name = `${grant.child.operationKey}.json`; + try { + const saved = z + .strictObject({ assertion: z.string() }) + .parse(readPrivateRecord(local, "execution-parents", name)); + if (saved.assertion !== assertion) throw Error("Parent attempt replay conflict"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + writePrivateRecord(local, "execution-parents", name, { assertion }, true); + } + return { parent, execution: grant.child }; +} function parseParentAssertion(assertion: string) { if (assertion.length > 4096 || !/^[A-Za-z0-9_-]+$/.test(assertion)) throw Error("Invalid parent assertion"); @@ -3263,133 +3014,157 @@ function parseParentAssertion(assertion: string) { throw Error("Noncanonical parent assertion"); return parsed; } -function executionWitnessDirectory( - state: InstallationState, - issuer: ControlIssuer, - create = false, -) { - const data = new AnchoredDirectory(state.dataRoot), - control = data.directory(".admin-target-control", create, true), - executions = control.directory("executions", create, true), - role = executions.directory(issuer, create, true); +function executionWitnessDirectory(state: InstallationState, issuer: ControlIssuer) { + // Execution witnesses stay in the role's own private state. Another role + // learns about an execution from the request that names it, never from here. + const executions = state.privateRoot.directory("execution-witnesses", true, true), + role = executions.directory(issuer, true, true); return { role, close() { role.close(); executions.close(); - control.close(); - data.close(); }, }; } -const dataContainmentWitnessSchema = z.strictObject({ - version: z.literal(1), - installationId: uuid, +const dataTerminalObservationSchema = z.strictObject({ + version: z.literal(2), execution: executionRefSchema, status: dataExecutionStatusSchema, - terminalAt: z.number().int().positive().safe(), + observedAt: z.number().int().positive().safe(), terminalDigest: digest, }); -/** Fixed DM journal calls only after verifying its own DB tuple, private - * original terminal commitment, all children and actual target release. */ -export function publishDataControlTerminalWitness( - authority: VerifiedExecutionAuthority, +const dataTerminalName = (execution: ExecutionRef) => + `data-terminal-${execution.operationKey}.json`; +/** + * Records the terminal status data-manager reported for a child this role + * issued, read over data-manager's authenticated route. Only a terminal status + * with a known publication is recorded, and a recorded status never changes. + */ +export function recordDataControlTerminalObservation( + installation: ControlInstallation, + execution: ExecutionRef, status: import("../services/data-manager/contracts/executions").DataExecutionStatus, - terminalAt: number, -): void { - const s = executionAuthorityState(authority); - if (s.installation.installation.role !== "data-manager" || s.installation.fixedChildId) - throw Error("Original data journal required"); - if (s.kind === "generation-hardlinks") verifyRetainedGenerationHardlinksSettlement(authority); - else verifyRetainedArtifactTargetRootRelease(authority); - const expected = controlDigest({ version: 1, status: dataExecutionStatusSchema.parse(status) }); - const original = verifyControl( - s.installation.installation.publicKey, - "openmapx-data-terminal-v1", - readPrivateRecord( - s.installation, - "observations", - `terminal-${s.execution.operationKey}.json`, - 8192, - ), - z - .object({ - phase: z.literal("canonical"), - execution: executionRefSchema, - terminalDigest: digest, - terminalAt: z.number(), - admissionDigest: digest, - }) - .passthrough(), - ); +) { + const local = installationState(installation), + parsed = dataExecutionStatusSchema.parse(status); if ( - original.terminalDigest !== expected || - original.terminalAt !== terminalAt || - original.admissionDigest !== s.receiptDigest || - controlDigest(original.execution) !== controlDigest(s.execution) + parsed.operationKey !== execution.operationKey || + parsed.fingerprint !== execution.fingerprint || + !["ok", "partial", "error", "canceled", "interrupted"].includes(parsed.status) || + parsed.publication === "unknown" ) - throw Error("Original canonical data commitment required"); - const payload = dataContainmentWitnessSchema.parse({ - version: 1, - installationId: s.installation.installation.installationId, - execution: s.execution, - status, - terminalAt, - terminalDigest: expected, - }), - directory = executionWitnessDirectory(s.installation, "data-manager"); + throw Error("Exact terminal data-manager status required"); + const terminalDigest = controlDigest({ version: 1, status: parsed }); try { - const name = `${s.execution.operationKey}.data-terminal.json`, - bytes = canonicalJson( - signControl(s.installation.privateKey, "openmapx-data-contained-terminal-v1", payload), - ); - let prior: Buffer | undefined; - try { - prior = directory.role.read(name, 65536); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - if (prior) { - if (prior.toString("utf8") !== bytes) throw Error("Canonical data witness changed"); - return; - } - directory.role.write(name, bytes, true); - } finally { - directory.close(); + const prior = dataTerminalObservationSchema.parse( + readPrivateRecord(local, "observations", dataTerminalName(execution)), + ); + if (prior.terminalDigest !== terminalDigest) throw Error("Recorded data terminal changed"); + return prior; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + const observation = dataTerminalObservationSchema.parse({ + version: 2, + execution, + status: parsed, + observedAt: Date.now(), + terminalDigest, + }); + writePrivateRecord(local, "observations", dataTerminalName(execution), observation, true); + return observation; } -/** A parent verifies the pinned DM signature and exact bound execution; a GET - * status or dead child process alone is never backend containment evidence. */ -export function verifyDataControlTerminalWitness( +/** The terminal status this role recorded for its data-manager child. A dead + * child process or an error alone is never containment evidence. */ +export function readDataControlTerminalObservation( installation: ControlInstallation, execution: ExecutionRef, status?: import("../services/data-manager/contracts/executions").DataExecutionStatus, ) { + const proof = dataTerminalObservationSchema.parse( + readPrivateRecord(installationState(installation), "observations", dataTerminalName(execution)), + ); + if ( + controlDigest(proof.execution) !== controlDigest(execution) || + (status && + canonicalJson(dataExecutionStatusSchema.parse(status)) !== canonicalJson(proof.status)) + ) + throw Error("Exact data-manager terminal observation required"); + return proof; +} +const opsTerminalObservationSchema = z.strictObject({ + version: z.literal(2), + execution: executionRefSchema, + status: z.unknown(), + observedAt: z.number().int().positive().safe(), + terminalDigest: digest, +}); +const opsTerminalName = (execution: ExecutionRef) => `ops-terminal-${execution.operationKey}.json`; +/** + * Records the terminal status the agent reported for a child this role issued, + * read over the agent's authenticated route. A recorded status never changes. + */ +export function recordOpsControlTerminalObservation( + installation: ControlInstallation, + execution: ExecutionRef, + input: OpsJobStatus, +): void { const local = installationState(installation), - directory = executionWitnessDirectory(local, "data-manager"); + status = parseOpsJobStatus(input); + if ( + status.operationId !== execution.executionId || + status.operationKey !== execution.operationKey || + status.fingerprint !== execution.fingerprint || + !["succeeded", "failed", "timed_out"].includes(status.state) || + // An outcome that still needs recovery is not known yet. + status.errorClass === "recovery_required" + ) + throw Error("Exact terminal agent status required"); + const terminalDigest = controlTerminalReceiptDigest(status); try { - const proof = verifyControl( - local.publicKeys["data-manager"], - "openmapx-data-contained-terminal-v1", - JSON.parse( - directory.role.read(`${execution.operationKey}.data-terminal.json`, 65536).toString("utf8"), - ), - dataContainmentWitnessSchema, + const prior = opsTerminalObservationSchema.parse( + readPrivateRecord(local, "observations", opsTerminalName(execution)), ); - if ( - proof.installationId !== installation.installationId || - controlDigest(proof.execution) !== controlDigest(execution) || - proof.terminalDigest !== controlDigest({ version: 1, status: proof.status }) || - (status && canonicalJson(status) !== canonicalJson(proof.status)) || - !["ok", "partial", "error", "canceled", "interrupted"].includes(proof.status.status) || - proof.status.publication === "unknown" - ) - throw Error("Exact canonical data containment witness required"); - readControlExecutionWitness(local, execution, local.publicKeys["data-manager"]); - return proof; - } finally { - directory.close(); + if (prior.terminalDigest !== terminalDigest) throw Error("Recorded agent terminal changed"); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + writePrivateRecord( + local, + "observations", + opsTerminalName(execution), + opsTerminalObservationSchema.parse({ + version: 2, + execution, + // Diagnostics are observational and excluded from the terminal digest. + status: { ...status, diagnostics: { tail: [], truncated: false } }, + observedAt: Date.now(), + terminalDigest, + }), + true, + ); +} +/** The terminal status this role recorded for its agent child. */ +export function readOpsControlTerminalObservation( + installation: ControlInstallation, + execution: ExecutionRef, +): OpsJobStatus { + const proof = opsTerminalObservationSchema.parse( + readPrivateRecord( + installationState(installation), + "observations", + opsTerminalName(execution), + ), + ), + status = parseOpsJobStatus(proof.status); + if ( + controlDigest(proof.execution) !== controlDigest(execution) || + controlTerminalReceiptDigest(status) !== proof.terminalDigest + ) + throw Error("Exact agent terminal observation required"); + return status; } const lifecycleSchema = z.strictObject({ version: z.literal(1), @@ -3416,7 +3191,7 @@ function withExecutionWriter( execution: ExecutionRef, run: (dir: AnchoredDirectory) => T, ): T { - const d = executionWitnessDirectory(state, execution.executor, true); + const d = executionWitnessDirectory(state, execution.executor); const name = `${execution.operationKey}.writer`; let guard: AnchoredDirectory | undefined; try { @@ -3545,6 +3320,8 @@ function withParentLifecycles( verifier: string; }> = []; function collect(value: string) { + // A parent in another role cancels its children through their route. + if (!parentReadable(state, parseParentAssertion(value).issuer)) return; const grant = readParentGrant(state, value); if (grant.parentAssertion) collect(grant.parentAssertion); const verifier = grant.parentAssertion @@ -3981,20 +3758,17 @@ export async function attestParentExecution( }); const signed = signControl(s.installation.privateKey, "openmapx-parent-grant-v2", grant); const fields = { - version: 1 as const, + version: 2 as const, installationId: grant.installationId, issuer: s.execution.executor, operationKey: s.execution.operationKey, attemptId, grantDigest: controlDigest(signed), + grant: parentSummary(grant), }; - const assertion = Buffer.from( - canonicalJson({ - ...fields, - signature: signControl(s.installation.privateKey, "openmapx-parent-assertion-v1", fields) - .signature, - }), - ).toString("base64url"); + const assertion = Buffer.from(canonicalJson(parentAssertionSchema.parse(fields))).toString( + "base64url", + ); parseParentAssertion(assertion); return withControlDelegationPublication(authority, child.phase, (dir) => { const lifecycle = readLifecycle( @@ -4084,16 +3858,10 @@ export function readParentGrant( throw Error("Invalid fixed parent certification"); verifier = ancestor.childPublicKey; } - const { signature, ...fields } = parsed; - verifyControl( - verifier, - "openmapx-parent-assertion-v1", - { payload: fields, signature }, - parentAssertionSchema.omit({ signature: true }), - ); const grant = verifyControl(verifier, "openmapx-parent-grant-v2", signed, parentGrantSchema); const parent = readControlExecutionWitness(state, grant.parent, verifier); if ( + canonicalJson(parentSummary(grant)) !== canonicalJson(parsed.grant) || !sameControlOrigin(grant.origin, parent.origin) || grant.parent.executor !== parsed.issuer || grant.parent.operationKey !== parsed.operationKey || @@ -4275,6 +4043,8 @@ export function openFixedChildInstallation(options: { privateRoot: recovery, dataRoot: join(options.infraDir, "data"), registeredTargets: new Set(), + // Fixed children run in the agent's container or its helper: both use the store. + leases: agentStoreLeases(options.infraDir, "ops-agent"), fixedChildId: id, ...("attemptId" in envelope ? { fixedParentAssertion: options.assertion } : {}), }); @@ -4630,7 +4400,7 @@ function publishControlExecutionWitness( }, authorityDigest: string, ) { - const d = executionWitnessDirectory(state, input.execution.executor, true); + const d = executionWitnessDirectory(state, input.execution.executor); try { const witness = signControl(state.privateKey, "openmapx-control-execution-witness-v2", { version: 2 as const, @@ -4808,7 +4578,7 @@ export function retainControlPreparedTerminal(authority: VerifiedExecutionAuthor signed, true, ); - const d = executionWitnessDirectory(s.installation, "ops-agent", true); + const d = executionWitnessDirectory(s.installation, "ops-agent"); try { d.role.write(`${s.execution.operationKey}.prepared.json`, canonicalJson(signed), true); } finally { @@ -4897,7 +4667,7 @@ export async function verifyDataUpdatePrerequisites(authority: VerifiedExecution const bound = lookupControlChildPhase(dataAuthority, slot); if (!bound) throw Error("Full Update prerequisite missing"); const child = receipt.children[index], - witness = verifyDataControlTerminalWitness( + witness = readDataControlTerminalObservation( dataState.installation.installation, bound.execution, ); @@ -4955,7 +4725,7 @@ function verifyDataUpdateCompletionSource( child.jobId !== bound.execution.executionId || child.fingerprint !== bound.execution.fingerprint || child.deadlineAt !== bound.execution.deadlineAt || - verifyDataControlTerminalWitness(dataState.installation.installation, bound.execution) + readDataControlTerminalObservation(dataState.installation.installation, bound.execution) .status.status !== "ok" ) throw Error("Actual Full Update data completion required"); @@ -4992,7 +4762,7 @@ function verifyDataUpdateCompletionSource( const bound = lookupControlChildPhase(authority, slot); if ( !bound || - verifyDataControlTerminalWitness(s.installation.installation, bound.execution).status + readDataControlTerminalObservation(s.installation.installation, bound.execution).status .status !== "ok" ) throw Error("Actual Full Update hardlink completion required"); diff --git a/packages/core/src/build-artifacts/control-prune.test.ts b/packages/core/src/build-artifacts/control-prune.test.ts index a0a3a74e2..65ec6d0cc 100644 --- a/packages/core/src/build-artifacts/control-prune.test.ts +++ b/packages/core/src/build-artifacts/control-prune.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, beforeEach, expect, it } from "vitest"; @@ -18,6 +18,7 @@ import { verifyControlEffectSettlement, } from "./control-authority"; import { CONTROL_PRUNE_AFTER_MS, pruneControlRecords } from "./control-prune"; +import { leaseStoreDirectory, openLeaseStore } from "./lease-store"; import { acquireArtifactTargetLease, releaseArtifactTargetLease, @@ -140,10 +141,14 @@ it("refuses an expired root whether or not its records were pruned", async () => expect(existsSync(record("executions", execution.operationKey))).toBe(false); }); -it("removes a released lease and frees its retained history slot", async () => { +const heldLeases = () => + openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }) + .list() + .map((value) => value.leaseId); + +it("prunes the root's lease record once its released lease left the store", async () => { const { execution, authority } = admit(); const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: authority, }); @@ -153,14 +158,10 @@ it("removes a released lease and frees its retained history slot", async () => { verifyControlEffectSettlement(installation, execution), ); await releaseArtifactTargetLease(lease); - const history = join(root, "data/.admin-target-control/admission-index-v1/history"), - leases = join(root, "data/.admin-target-control/leases"); - expect(readdirSync(history)).toHaveLength(1); - expect(readdirSync(leases)).toEqual([lease.leaseId]); + expect(heldLeases()).toEqual([]); + expect(existsSync(record("execution-leases", execution.operationKey))).toBe(true); const later = execution.deadlineAt + CONTROL_PRUNE_AFTER_MS + 60_000; expect(await pruneControlRecords(installation, { now: later })).toEqual([execution.operationKey]); - expect(readdirSync(history)).toEqual([]); - expect(readdirSync(leases)).toEqual([]); expect(existsSync(record("execution-leases", execution.operationKey))).toBe(false); }); @@ -234,7 +235,6 @@ it("refuses a pruned and expired execution to its readers", async () => { it("keeps an execution whose effect record is absent, with its lease held", async () => { const { execution } = admit(), lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: restoreAdmittedControlExecution(installation, execution), }); @@ -243,14 +243,13 @@ it("keeps an execution whose effect record is absent, with its lease held", asyn expect(await pruneControlRecords(installation, { now })).toEqual([]); expect(existsSync(record("executions", execution.operationKey))).toBe(true); await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(); - expect(readdirSync(join(root, "data/.admin-target-control/leases"))).toEqual([lease.leaseId]); + expect(heldLeases()).toEqual([lease.leaseId]); }); it("keeps a borrower while the lease it adopted is held, and while any adoption is unreadable", async () => { const owner = admit(), borrower = admit(), lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: owner.authority, }); diff --git a/packages/core/src/build-artifacts/control-prune.ts b/packages/core/src/build-artifacts/control-prune.ts index f7bfa3022..a63eb3912 100644 --- a/packages/core/src/build-artifacts/control-prune.ts +++ b/packages/core/src/build-artifacts/control-prune.ts @@ -14,25 +14,19 @@ * missing record as permission: a missing admission, effect or lease is * unknown, and an expired root key is refused at admission. An agent * execution also needs its sealed canonical terminal commitment. */ -import { readFileSync, rmSync, statSync } from "node:fs"; -import { join } from "node:path"; +import { rmSync, statSync } from "node:fs"; +import { dirname, join } from "node:path"; import { z } from "zod"; -import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; +import { executionRefSchema } from "../ops/execution-ref"; import { type ControlInstallation, childSettlementState, installationState, readPrivateRecord, - restoreAdmittedControlExecution, verifyControlEffectSettlement, } from "./control-authority"; import { controlDigest, controlOriginSchema, type InstallationState } from "./control-records"; import { AnchoredDirectory } from "./filesystem"; -import { - artifactTargetLeaseReleased, - removeReleasedArtifactTargetLease, - verifyRetainedArtifactTargetRootRelease, -} from "./target-lease"; export const CONTROL_PRUNE_AFTER_MS = 72 * 3600_000; /** Namespaces swept by name. `executions`, `effects` and the terminal @@ -108,9 +102,6 @@ const cursors = new WeakMap(); interface Prunable { key: string; - execution: ExecutionRef; - /** The owned lease still exists and is released; the prune removes it. */ - leased: boolean; /** Name fragments of this execution's own records, per namespace ("*" for * every swept namespace). */ identifiers: Map>; @@ -166,8 +157,9 @@ function mtime(dir: AnchoredDirectory, name: string): number | null { throw error; } } -function leaseDirectory(local: InstallationState, leaseId: string): string { - return join(local.dataRoot, ".admin-target-control", "leases", leaseId); +/** The role's execution witnesses, in its own private state. */ +function witnessDirectory(local: InstallationState): string { + return join(local.privateRoot.path, "execution-witnesses", local.installation.role); } function assertionLeaseId(assertion: string): string { return leaseReferenceSchema.parse( @@ -217,14 +209,14 @@ function borrowedLeases( ]; } /** The execution and the identifiers of its own records, when they may go. */ -function prunable( +async function prunable( installation: ControlInstallation, local: InstallationState, name: string, now: number, onFull: () => void, adoptions: () => Map | null, -): Prunable | null { +): Promise { const admitted = admissionSchema.parse( readPrivateRecord(local, "executions", name, 65536), ).payload; @@ -274,24 +266,16 @@ function prunable( add(["hardlinks"], admitted.execution.executionId); if (admitted.operation?.handoffId) add(["observations"], admitted.operation.handoffId); // Fixed children it prepared: their key, spawn and key index records. - const fixed = optional(() => - z - .array(z.uuid()) - .parse( - JSON.parse( - readFileSync( - join( - local.dataRoot, - ".admin-target-control", - "executions", - local.installation.role, - `${key}.keys.json`, - ), - "utf8", - ), - ), - ), - ); + const fixed = optional(() => { + const witnesses = new AnchoredDirectory(witnessDirectory(local)); + try { + return z + .array(z.uuid()) + .parse(JSON.parse(witnesses.read(`${key}.keys.json`, 4096).toString("utf8"))); + } finally { + witnesses.close(); + } + }); for (const id of fixed ?? []) { add(["children", "spawns", "spawn-started"], id); const child = optional(() => readPrivateRecord(local, "children", `${id}.key`, 2048)) as @@ -317,30 +301,17 @@ function prunable( // As a borrower: only its own adoption record, once the lease is released. const index = adoptions(); if (!index) return null; + // A lease is released once its row is gone from the store. for (const lease of borrowedLeases(local, admitted, name, index)) { - if (!artifactTargetLeaseReleased(local, lease.leaseId)) return null; + if (await local.leases.read(lease.leaseId)) return null; add(["adoptions"], lease.delegationId); } const owned = optional( () => leaseReferenceSchema.parse(readPrivateRecord(local, "execution-leases", name)).leaseId, ); - let leased = false; if (owned) { + if (await local.leases.read(owned)) return null; add(issued, owned); - const ledger = leaseDirectory(local, owned); - // A lease already removed by an earlier interrupted pass has nothing to hold. - if (optional(() => statSync(ledger))) { - verifyRetainedArtifactTargetRootRelease( - restoreAdmittedControlExecution(installation, admitted.execution), - ); - leased = true; - const grants = new AnchoredDirectory(join(ledger, "grants")); - try { - for (const entry of entries(grants, onFull)) add(issued, entry.replace(/\.json$/, "")); - } finally { - grants.close(); - } - } } if (ownerRecord && preparation) { const source = optional(() => @@ -356,8 +327,6 @@ function prunable( } return { key, - execution: admitted.execution, - leased, identifiers, ...(ownerRecord && preparation ? { ownerRecord } : {}), }; @@ -367,9 +336,14 @@ function prunable( function removeShared(local: InstallationState, keys: readonly string[], onFull: () => void): void { const role = local.installation.role, paths = [ - { path: join(local.dataRoot, ".admin-target-control", "executions", role), seed: false }, + { path: witnessDirectory(local), seed: false }, ...(role === "ops-agent" || role === "host-cli" - ? [{ path: join(local.dataRoot, "ops-agent", "lifecycle"), seed: true }] + ? [ + { + path: join(dirname(local.dataRoot), "state", "ops-agent", "lifecycle"), + seed: true, + }, + ] : []), ]; for (const { path, seed } of paths) { @@ -440,7 +414,7 @@ export async function pruneControlRecords( for (const name of batch) { if (options.keep?.(name.replace(/\.json$/, ""))) continue; try { - const value = prunable(installation, local, name, now, onFull, adoptions); + const value = await prunable(installation, local, name, now, onFull, adoptions); if (value) found.push(value); } catch { /* An unreadable or unverifiable record is kept. */ @@ -449,9 +423,6 @@ export async function pruneControlRecords( const keys = found.map((value) => value.key); if (keys.length) { await options.beforeRemove?.(keys); - for (const { execution, leased } of found) - if (leased) - removeReleasedArtifactTargetLease(restoreAdmittedControlExecution(installation, execution)); removeShared(local, keys, onFull); const terminals = new Set(keys.map((key) => `terminal-${key}.json`)); for (const namespace of NAMESPACES) { diff --git a/packages/core/src/build-artifacts/control-records.ts b/packages/core/src/build-artifacts/control-records.ts index c0090f5e6..af2436d93 100644 --- a/packages/core/src/build-artifacts/control-records.ts +++ b/packages/core/src/build-artifacts/control-records.ts @@ -5,6 +5,7 @@ import { canonicalJson } from "../ops/canonical-json"; import { type ControlExecutionOrigin, parseControlRootOperationKey } from "../ops/control-root-key"; import type { ControlIssuer } from "../ops/execution-identity"; import type { AnchoredDirectory } from "./filesystem"; +import type { LeaseBackend } from "./target-lease"; declare const installationBrand: unique symbol; export type ControlInstallation = Readonly<{ @@ -22,6 +23,8 @@ export interface InstallationState { privateRoot: AnchoredDirectory; dataRoot: string; registeredTargets: ReadonlySet; + /** Where this role claims targets: the agent's store, directly or through the agent. */ + leases: LeaseBackend; } export const controlOriginSchema = z .strictObject({ diff --git a/packages/core/src/build-artifacts/data-configuration-fence.ts b/packages/core/src/build-artifacts/data-configuration-fence.ts index acb864b1b..41a477c7f 100644 --- a/packages/core/src/build-artifacts/data-configuration-fence.ts +++ b/packages/core/src/build-artifacts/data-configuration-fence.ts @@ -1,11 +1,12 @@ /** The data-manager's configuration fence: one row in * `data_manager.configuration_fence`. While a row is `prepared`, no data job - * is admitted. The agent prepares and settles it with its signed delegation; - * both calls are idempotent on the same body. */ + * is admitted. Only the operations agent prepares and settles it, authenticated + * by its own data-manager token; both calls are idempotent on the same body. + * The agent records the fence leaf in its lease store and settles it from the + * receipt this returns. */ import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; import { executionFingerprint, type VerifiedParentExecution } from "../ops/execution-identity"; -import type { ExecutionRef } from "../ops/execution-ref"; import { type DataConfigurationFencePrepare, type DataConfigurationFenceReceipt, @@ -14,18 +15,15 @@ import { dataConfigurationFencePrepareSchema, dataConfigurationFenceSettleSchema, } from "../services/data-manager/contracts/configuration-maintenance"; -import { - GENERATION_HARDLINKS_NON_ADMISSION_DOMAIN, - startGenerationHardlinksExecutionSchema, -} from "../services/data-manager/contracts/executions"; +import { startGenerationHardlinksExecutionSchema } from "../services/data-manager/contracts/executions"; import { beginControlEffect, type ControlInstallation, type ControlTransportPrincipal, childSettlementState, controlDigest, - controlHeaders, executionAuthorityState, + inspectRemoteDelegation, installationState, readPrivateRecord, reserveControlTerminal, @@ -34,26 +32,11 @@ import { retainedControlParent, settleExecutionLifecycle, type VerifiedExecutionAuthority, - verifyConfigurationMaintenanceGenerationWitness, verifyControlEffectSettlement, - verifyControlPeer, withPrivateEffectWriter, writePrivateRecord, } from "./control-authority"; -import { signControl, verifyControl } from "./control-records"; -import { verifiedParentForArtifactDelegation } from "./peer-admission"; -import { - adoptArtifactTargetLease, - assertArtifactDelegationRootHeld, - assertArtifactTargetLeaseForward, - bindBorrowedControlExecution, - closeArtifactTargetLeaseHandle, - inspectArtifactDelegation, - observeUnadoptedServiceHardlinkGrant, - settleArtifactTargetLeaseChild, - settleUnadmittedConfigurationPrepare, - withConfigurationFenceReceiverAdmission, -} from "./target-lease"; +import { signControl } from "./control-records"; export interface DataFenceSql { unsafe( @@ -63,21 +46,13 @@ export interface DataFenceSql { transaction(run: (tx: Pick) => Promise): Promise; } type Tx = Pick; -const inspectWitnessSchema = z.strictObject({ - installationId: z.uuid(), - maintenanceId: z.uuid(), - candidateId: z.uuid(), - ownerCertificateDigest: z.string().regex(/^[a-f0-9]{64}$/), - issuedAt: z.number().int().nonnegative().safe(), - expiresAt: z.number().int().nonnegative().safe(), -}); const fenceRowSchema = z.object({ maintenance_id: z.uuid(), fence_id: z.uuid(), operation_key: z.string(), candidate_id: z.uuid(), plan_digest: z.string(), - owner_certificate_digest: z.string(), + owner_lease_id: z.uuid(), input: dataConfigurationFencePrepareSchema, state: z.enum(["prepared", "settled"]), outcome: dataConfigurationFenceOutcomeSchema.nullable(), @@ -116,9 +91,10 @@ const preparedGenerations = new WeakMap< installation: ControlInstallation; request: ReturnType; headers: readonly string[]; + principal: ControlTransportPrincipal; parent: VerifiedParentExecution; fenceDigest: string; - execution: ReturnType["execution"]; + execution: ReturnType["execution"]; } >(); export function createDataConfigurationFenceStore( @@ -144,65 +120,41 @@ export function createDataConfigurationFenceStore( ); return rows[0] ? fenceRowSchema.parse(rows[0]) : null; } - function inspectGrant( + function inspectDelegation( body: DataConfigurationFencePrepare, rawHeaders: readonly string[], - transportPrincipal: Exclude, + transportPrincipal: ControlTransportPrincipal, ) { const kind = "configuration-maintenance.prepare"; try { - const headers = controlHeaders(rawHeaders); - if (!headers.delegation || headers.parent) throw Error("Pinned agent fence grant required"); - const grant = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - assertion: headers.delegation, - installation, - }); + if (transportPrincipal !== "ops-agent") throw Error("Agent transport required"); + const delegation = inspectRemoteDelegation(installation, rawHeaders, transportPrincipal); const operation = { kind, ...body }; if ( - grant.recipient !== "data-manager" || - grant.parent.executor !== "ops-agent" || - grant.kind !== kind || - grant.phase !== "configuration-fence" || - controlDigest(grant.parent) !== controlDigest(body.ownerPrepare) || - grant.execution.operationFingerprint !== controlDigest(operation) || - grant.execution.executor !== "data-manager" || - grant.execution.fingerprint !== + delegation.parent.executor !== "ops-agent" || + delegation.kind !== kind || + delegation.phase !== "configuration-fence" || + controlDigest(delegation.parent) !== controlDigest(body.ownerPrepare) || + controlDigest(delegation.owner) !== controlDigest(body.ownerPrepare) || + delegation.execution.operationFingerprint !== controlDigest(operation) || + delegation.execution.fingerprint !== executionFingerprint({ kind, operationFingerprint: controlDigest(operation), - deadlineAt: grant.execution.deadlineAt, - parent: retainedControlParent(grant.parent), + deadlineAt: delegation.execution.deadlineAt, + parent: retainedControlParent(delegation.parent), }) || - canonicalJson(grant.serviceIds) !== canonicalJson(body.serviceIds) + canonicalJson(delegation.serviceIds) !== canonicalJson(body.serviceIds) ) - throw Error("Configuration fence grant identity conflict"); - const peer = verifyControlPeer(installation, { - rawHeaders, - transportPrincipal, - execution: grant.execution, - kind, - }); - return { - grant, - peer, - operation, - parent: verifiedParentForArtifactDelegation( - installation, - headers.delegation, - grant.execution, - ), - assertion: headers.delegation, - }; + throw Error("Configuration fence delegation identity conflict"); + return { delegation, operation }; } catch { - // Classify verification failures before any fence SQL or adoption. - throw forbidden("Pinned agent configuration fence grant rejected"); + // Classify verification failures before any fence SQL. + throw forbidden("Agent configuration fence delegation rejected"); } } - /** Writes the leaf's terminal effect and settles it in the agent's lease, so - * the agent can release its root once every child is known. */ - async function settleLeaf( - lease: Awaited>, + /** Writes the leaf's own terminal effect record. */ + function recordLeaf( authority: VerifiedExecutionAuthority, outcome: "committed" | "failed_without_effect", receipt: unknown, @@ -225,93 +177,71 @@ export function createDataConfigurationFenceStore( false, ); }); - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(installation, state.execution), - ); } - /** Adopts the agent's delegation as this receiver's leaf execution and runs - * the fence transition once. A repeated request returns the recorded result. */ + /** Records the agent's fence leaf as this receiver's execution and runs the + * fence transition once. A repeated request returns the recorded result. */ async function withLeaf( - verified: ReturnType, + verified: ReturnType, run: () => Promise, ): Promise { - const { grant, peer } = verified; - const lease = await adoptArtifactTargetLease({ - dataRoot: local.dataRoot, - assertion: verified.assertion, - peer, - execution: grant.execution, - serviceIds: grant.serviceIds, - }); - // The fence holds the whole application closure the agent signed into - // the lease. Startup registers only the release's built-in services; - // installed community-extension services are known to the agent alone. - local.registeredTargets = new Set([...local.registeredTargets, ...lease.serviceIds]); + const { delegation } = verified; + // The fence holds the whole application closure the agent leased. + // Startup registers only the release's built-in services; installed + // community-extension services are known to the agent alone. + local.registeredTargets = new Set([...local.registeredTargets, ...delegation.serviceIds]); + let authority: VerifiedExecutionAuthority | undefined; try { - const authority = withConfigurationFenceReceiverAdmission(lease, () => { - let admitted: VerifiedExecutionAuthority | undefined; - try { - admitted = restoreAdmittedControlExecution(installation, grant.execution); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - if (!admitted) { - assertArtifactTargetLeaseForward(lease); - admitted = retainAdmittedControlExecution(installation, { - execution: grant.execution, - kind: grant.kind, - operation: verified.operation, - admittedAt: Date.now(), - requestedDeadlineAt: grant.execution.deadlineAt, - parent: verified.parent, - serviceIds: grant.serviceIds, - }); - } - bindBorrowedControlExecution(lease, admitted); - return admitted; - }); - let prior: unknown; - try { - prior = readPrivateRecord(local, "effects", `${grant.execution.operationKey}.json`); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - if (prior !== undefined && prior !== null) { - const settlement = verifyControlEffectSettlement(installation, grant.execution); - // A crash between the effect record and the lease child leaves the - // child unsettled; settling it again is idempotent. - await settleArtifactTargetLeaseChild(lease, settlement); - const settled = childSettlementState(settlement).evidence; - if (settled.outcome !== "committed") throw new DataFenceBusy(); - const saved = await row(sql, "maintenance_id", verified.operation.maintenanceId); - if (!saved) throw new DataFenceBusy(); - return receiptOf(saved); - } - reserveControlTerminal(authority); - assertArtifactTargetLeaseForward(lease); - beginControlEffect(authority); - let receipt: DataConfigurationFenceReceipt; - try { - receipt = await run(); - } catch (error) { - // A busy refusal made no write: settle this leaf as failed without effect. - if (error instanceof DataFenceBusy) - await settleLeaf(lease, authority, "failed_without_effect", { - kind: "data-configuration-fence-refused", - maintenanceId: verified.operation.maintenanceId, - reason: "busy", - }); - throw error; - } - await settleLeaf(lease, authority, "committed", { - kind: "data-configuration-fence", - receipt: { ...receipt, sequence: 0, state: "prepared" }, + authority = restoreAdmittedControlExecution(installation, delegation.execution); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (!authority) { + if (Date.now() >= delegation.execution.deadlineAt) throw new DataFenceBusy(); + authority = retainAdmittedControlExecution(installation, { + execution: delegation.execution, + kind: delegation.kind, + operation: verified.operation, + admittedAt: Date.now(), + requestedDeadlineAt: delegation.execution.deadlineAt, + parent: delegation.verifiedParent, + serviceIds: delegation.serviceIds, }); - return receipt; - } finally { - closeArtifactTargetLeaseHandle(lease); } + let prior: unknown; + try { + prior = readPrivateRecord(local, "effects", `${delegation.execution.operationKey}.json`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (prior !== undefined && prior !== null) { + const settled = childSettlementState( + verifyControlEffectSettlement(installation, delegation.execution), + ).evidence; + if (settled.outcome !== "committed") throw new DataFenceBusy(); + const saved = await row(sql, "maintenance_id", verified.operation.maintenanceId); + if (!saved) throw new DataFenceBusy(); + return receiptOf(saved); + } + reserveControlTerminal(authority); + beginControlEffect(authority); + let receipt: DataConfigurationFenceReceipt; + try { + receipt = await run(); + } catch (error) { + // A busy refusal made no write: the leaf failed without effect. + if (error instanceof DataFenceBusy) + recordLeaf(authority, "failed_without_effect", { + kind: "data-configuration-fence-refused", + maintenanceId: verified.operation.maintenanceId, + reason: "busy", + }); + throw error; + } + recordLeaf(authority, "committed", { + kind: "data-configuration-fence", + receipt: { ...receipt, sequence: 0, state: "prepared" }, + }); + return receipt; } async function assertAdmission(tx: Tx) { if (!open) throw new DataFenceBusy(); @@ -321,119 +251,86 @@ export function createDataConfigurationFenceStore( async function inspectGenerationAdmission( value: unknown, rawHeaders: readonly string[], - parent: VerifiedParentExecution | undefined, + principal: ControlTransportPrincipal, tx: Tx, ) { - const request = startGenerationHardlinksExecutionSchema.parse(value), - headers = controlHeaders(rawHeaders); - if (!headers.delegation || headers.parent) throw forbidden("Signed generation grant required"); - const grant = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - assertion: headers.delegation, - installation, - }); + const request = startGenerationHardlinksExecutionSchema.parse(value); + let delegation: ReturnType; + try { + delegation = inspectRemoteDelegation(installation, rawHeaders, principal); + } catch { + throw forbidden("Generation delegation required"); + } if ( - grant.recipient !== "data-manager" || - grant.kind !== "generation-hardlinks" || - grant.phase !== "hardlinks" || - grant.execution.operationKey !== request.operationKey || - grant.execution.operationFingerprint !== controlDigest(request.input) || + delegation.kind !== "generation-hardlinks" || + delegation.phase !== "hardlinks" || + delegation.execution.operationKey !== request.operationKey || + delegation.execution.operationFingerprint !== controlDigest(request.input) || request.input.planDigest !== controlDigest(request.plan) || - request.plan.plan.some((entry) => !grant.serviceIds.includes(entry.consumerService)) + request.plan.plan.some((entry) => !delegation.serviceIds.includes(entry.consumerService)) ) - throw forbidden("Generation plan/grant binding conflict"); + throw forbidden("Generation plan/delegation binding conflict"); if (!open) throw new DataFenceBusy(); const fence = await row(tx, "state", "prepared"); if (!fence) return; const receipt = receiptOf(fence); + // During maintenance only the maintenance owner's own hardlink leaf runs: + // the agent issued it under the lease the fence was prepared for. if ( - grant.ownerCertificateDigest !== fence.owner_certificate_digest || - canonicalJson(grant.owner) !== canonicalJson(receipt.ownerPrepare) || - grant.owner.executor !== "ops-agent" || + principal !== "ops-agent" || + delegation.leaseId !== fence.owner_lease_id || + canonicalJson(delegation.owner) !== canonicalJson(receipt.ownerPrepare) || request.input.revisionId !== receipt.revisionId || request.input.planDigest !== receipt.hardlinkPlanDigest || request.input.coordinatorMode !== fence.input.coordinatorMode || - grant.serviceIds.some((id) => !fence.input.serviceIds.includes(id)) + delegation.serviceIds.some((id) => !fence.input.serviceIds.includes(id)) ) throw forbidden("Generation maintenance owner/revision/hash conflict"); - if (!parent) throw new DataFenceBusy(); - try { - verifyConfigurationMaintenanceGenerationWitness(installation, request, parent, grant, { - maintenanceId: receipt.maintenanceId, - candidateId: receipt.candidateId, - ownerPrepare: receipt.ownerPrepare, - revisionId: receipt.revisionId, - hardlinkPlanDigest: receipt.hardlinkPlanDigest!, - coordinatorMode: fence.input.coordinatorMode, - serviceIds: fence.input.serviceIds, - rootCertificateDigest: fence.owner_certificate_digest, - }); - } catch { - throw new DataFenceBusy(); - } if ((await tx.unsafe(UNKNOWN_DATA_JOB)).length) throw new DataFenceBusy(); - return { request, grant, fenceDigest: controlDigest(fence) }; + return { request, delegation, fenceDigest: controlDigest(fence) }; } return { - /** Negative receiver observation under the same admission lock. It cannot - * admit or replay the original expired preparation. */ - async recoverUnadmittedPreparation(value: unknown, rawHeaders: readonly string[]) { - const input = dataConfigurationFencePrepareSchema.parse(value), - headers = controlHeaders(rawHeaders); - if (!headers.delegation || headers.parent) - throw forbidden("Original signed configuration delegation required"); - const grant = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - installation, - assertion: headers.delegation, - }); + /** Whether the original expired preparation was ever admitted, observed + * under the admission lock. An expired preparation can no longer be + * admitted, so a negative answer stays true. */ + async recoverUnadmittedPreparation( + value: unknown, + rawHeaders: readonly string[], + transportPrincipal: ControlTransportPrincipal, + ) { + const input = dataConfigurationFencePrepareSchema.parse(value); + const { delegation } = inspectDelegation(input, rawHeaders, transportPrincipal); + if (Date.now() <= Math.max(delegation.execution.deadlineAt, delegation.owner.deadlineAt)) + throw conflict("Original configuration preparation has not expired"); return sql.transaction(async (tx) => { await lock(tx); const admitted = await tx.unsafe( "SELECT maintenance_id FROM data_manager.configuration_fence WHERE operation_key=$1 OR fence_id=$2::uuid OR maintenance_id=$3::uuid UNION ALL SELECT id FROM data_manager.jobs WHERE operation_key=$1 OR id=$2::uuid LIMIT 1", - [grant.execution.operationKey, grant.execution.executionId, input.maintenanceId], + [ + delegation.execution.operationKey, + delegation.execution.executionId, + input.maintenanceId, + ], ); if (admitted.length) throw conflict("Original configuration preparation was admitted"); - return settleUnadmittedConfigurationPrepare(installation, headers.delegation!, input); - }); - }, - /** A fixed receiver observes its own locked SQL and private admission facts. - * The bearer, an HTTP 404, or an owner assertion alone is never this proof. */ - async attestUnadoptedGenerationHardlinks(value: unknown, rawHeaders: readonly string[]) { - const request = startGenerationHardlinksExecutionSchema.parse(value), - headers = controlHeaders(rawHeaders); - if (!headers.delegation || headers.parent) - throw forbidden("Original signed delegation required"); - const observe = () => observeUnadoptedServiceHardlinkGrant(installation, headers.delegation!); - const original = observe(); - if ( - request.operationKey !== original.child.operationKey || - request.requestedDeadlineAt !== original.child.deadlineAt || - controlDigest(request.input) !== original.child.operationFingerprint || - request.input.planDigest !== controlDigest(request.plan) - ) - throw Error("Original hardlink request changed"); - return sql.transaction(async (tx) => { - await lock(tx); - const admitted = await tx.unsafe( - "SELECT id FROM data_manager.jobs WHERE operation_key=$1 OR id=$2 LIMIT 1", - [original.child.operationKey, original.child.executionId], - ); - if (admitted.length) throw conflict("Original hardlink execution was already admitted"); - for (const [namespace, name] of [ - ["executions", `${original.child.operationKey}.json`], - ["adoptions", `${original.delegationId}.json`], - ] as const) { + // Any private trace of the execution means it may have been admitted. + for (const namespace of ["executions", "effects", "effect-started"]) { try { - readPrivateRecord(local, namespace, name); - throw Error("Original receiver admission or adoption requires recovery"); + readPrivateRecord(local, namespace, `${delegation.execution.operationKey}.json`); + throw conflict("Original configuration preparation was admitted"); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } } - if (canonicalJson(observe()) !== canonicalJson(original)) - throw Error("Original canceled grant changed during observation"); - return signControl(local.privateKey, GENERATION_HARDLINKS_NON_ADMISSION_DOMAIN, original); + return { + version: 2 as const, + kind: "configuration-prepare-not-admitted" as const, + leaseId: delegation.leaseId, + delegationId: delegation.delegationId, + child: delegation.execution, + input, + observedAt: Date.now(), + }; }); }, /** Admission opens once the fence table is readable. */ @@ -446,18 +343,21 @@ export function createDataConfigurationFenceStore( async assertGenerationAdmission( value: unknown, rawHeaders: readonly string[], + principal: ControlTransportPrincipal, parent?: VerifiedParentExecution, ): Promise { - const checked = await inspectGenerationAdmission(value, rawHeaders, parent, sql); + const checked = await inspectGenerationAdmission(value, rawHeaders, principal, sql); if (!checked) return undefined; + if (!parent) throw new DataFenceBusy(); const proof = Object.freeze({}) as VerifiedPreparedGenerationAdmission; preparedGenerations.set(proof, { installation, request: checked.request, headers: Object.freeze([...rawHeaders]), - parent: parent!, + principal, + parent, fenceDigest: checked.fenceDigest, - execution: checked.grant.execution, + execution: checked.delegation.execution, }); return proof; }, @@ -488,13 +388,13 @@ export function createDataConfigurationFenceStore( const checked = await inspectGenerationAdmission( source.request, source.headers, - source.parent, + source.principal, tx, ); if ( !checked || checked.fenceDigest !== source.fenceDigest || - canonicalJson(checked.grant.execution) !== canonicalJson(source.execution) + canonicalJson(checked.delegation.execution) !== canonicalJson(source.execution) ) throw new DataFenceBusy(); }, @@ -510,11 +410,11 @@ export function createDataConfigurationFenceStore( async prepare( value: DataConfigurationFencePrepare, rawHeaders: readonly string[], - transportPrincipal: Exclude, + transportPrincipal: ControlTransportPrincipal, ) { const input = dataConfigurationFencePrepareSchema.parse(value), - verified = inspectGrant(input, rawHeaders, transportPrincipal), - execution = verified.grant.execution, + verified = inspectDelegation(input, rawHeaders, transportPrincipal), + execution = verified.delegation.execution, assertSame = (existing: FenceRow) => { if ( canonicalJson(existing.input) !== canonicalJson(input) || @@ -542,14 +442,14 @@ export function createDataConfigurationFenceStore( ); if (active.length || transfers.length) throw new DataFenceBusy(); const inserted = await tx.unsafe( - "INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,owner_certificate_digest,input,state) VALUES($1::uuid,$2::uuid,$3,$4::uuid,$5,$6,$7::text::jsonb,'prepared') RETURNING *", + "INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,owner_lease_id,input,state) VALUES($1::uuid,$2::uuid,$3,$4::uuid,$5,$6::uuid,$7::text::jsonb,'prepared') RETURNING *", [ input.maintenanceId, execution.executionId, execution.operationKey, input.candidateId, input.planDigest, - verified.grant.ownerCertificateDigest, + verified.delegation.leaseId, canonicalJson(input), ], ); @@ -557,44 +457,18 @@ export function createDataConfigurationFenceStore( }), ); }, - async inspect(maintenanceId: string, rawHeaders: readonly string[]) { - const index = rawHeaders.findIndex( - (name, i) => - i % 2 === 0 && name.toLowerCase() === "x-openmapx-configuration-fence-witness", - ), - encoded = rawHeaders[index + 1]; - if ( - index < 0 || - rawHeaders.some( - (name, i) => - i !== index && - i % 2 === 0 && - name.toLowerCase() === "x-openmapx-configuration-fence-witness", - ) - ) - throw Error("Purpose-bound agent fence witness required"); - if (!encoded || encoded.length > 4096 || !/^[A-Za-z0-9_-]+$/.test(encoded)) - throw Error("Invalid fence witness"); - const witness = verifyControl( - local.publicKeys["ops-agent"], - "openmapx-configuration-fence-inspect-v1", - JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")), - inspectWitnessSchema, - ), - now = Date.now(); - if ( - witness.installationId !== installation.installationId || - witness.maintenanceId !== maintenanceId || - witness.issuedAt > now || - witness.expiresAt <= now || - witness.expiresAt - witness.issuedAt > 15000 - ) - throw Error("Expired or foreign fence witness"); + /** The fence row of `maintenanceId`, for the agent only. */ + async inspect( + maintenanceId: string, + expected: { candidateId: string; ownerLeaseId: string }, + transportPrincipal: ControlTransportPrincipal, + ) { + if (transportPrincipal !== "ops-agent") throw forbidden("Agent transport required"); const saved = await row(sql, "maintenance_id", maintenanceId); if (!saved) return null; if ( - saved.candidate_id !== witness.candidateId || - saved.owner_certificate_digest !== witness.ownerCertificateDigest + saved.candidate_id !== expected.candidateId || + saved.owner_lease_id !== expected.ownerLeaseId ) throw Error("Configuration fence owner changed"); return receiptOf(saved); @@ -605,31 +479,22 @@ export function createDataConfigurationFenceStore( async settle( value: DataConfigurationFenceSettle, rawHeaders: readonly string[], - transportPrincipal: Exclude, + transportPrincipal: ControlTransportPrincipal, ) { const input = dataConfigurationFenceSettleSchema.parse(value); - let execution: ExecutionRef; + let execution: z.infer["input"]["ownerPrepare"]; try { - const headers = controlHeaders(rawHeaders); if (transportPrincipal !== "ops-agent") throw Error("Agent transport required"); - if (!headers.delegation || headers.parent) throw Error("Prepare delegation required"); - const grant = inspectArtifactDelegation({ - dataRoot: local.dataRoot, - assertion: headers.delegation, - installation, - }); + const delegation = inspectRemoteDelegation(installation, rawHeaders, transportPrincipal); if ( - grant.recipient !== "data-manager" || - grant.parent.executor !== "ops-agent" || - grant.kind !== "configuration-maintenance.prepare" || - grant.phase !== "configuration-fence" + delegation.parent.executor !== "ops-agent" || + delegation.kind !== "configuration-maintenance.prepare" || + delegation.phase !== "configuration-fence" ) - throw Error("Configuration fence grant identity conflict"); - // Only the owner that still holds the maintenance root may settle. - assertArtifactDelegationRootHeld(installation, headers.delegation); - execution = grant.execution; + throw Error("Configuration fence delegation identity conflict"); + execution = delegation.execution; } catch { - throw forbidden("Pinned agent configuration fence grant rejected"); + throw forbidden("Agent configuration fence delegation rejected"); } // The conditional UPDATE and its read-back are the guard: a racing data // admission sees the row either prepared (busy) or settled (open). @@ -643,7 +508,7 @@ export function createDataConfigurationFenceStore( saved.fence_id !== execution.executionId || saved.operation_key !== execution.operationKey ) - throw forbidden("Configuration fence grant identity conflict"); + throw forbidden("Configuration fence delegation identity conflict"); if (saved.candidate_id !== input.candidateId || saved.plan_digest !== input.planDigest) throw conflict("Configuration fence settlement conflict"); await tx.unsafe( diff --git a/packages/core/src/build-artifacts/data-execution-candidate.ts b/packages/core/src/build-artifacts/data-execution-candidate.ts index 5ccc632db..50d8d1235 100644 --- a/packages/core/src/build-artifacts/data-execution-candidate.ts +++ b/packages/core/src/build-artifacts/data-execution-candidate.ts @@ -415,7 +415,6 @@ export function retainOwnedDataTransitCandidate( ); if (canonicalJson(prior) !== canonicalJson(value)) throw Error("Original candidate bytes/owner changed"); - publishCandidate(authority, prior); return controlDigest(prior); } if (publication && (publication.binding || publication.publication.state !== "unissued")) @@ -429,7 +428,6 @@ export function retainOwnedDataTransitCandidate( signControl(s.installation.privateKey, "openmapx-data-candidate-v1", parsed), true, ); - publishCandidate(authority, parsed); return controlDigest(parsed); } export function assertOwnedDataTransitCandidate( @@ -459,66 +457,51 @@ const publishedSchema = z.strictObject({ operationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), serviceIds: targetSchema, }); -function sharedDirectory(local: ReturnType) { - const data = new AnchoredDirectory(local.dataRoot), - control = data.directory(".admin-target-control"), - executions = control.directory("executions"), - role = executions.directory("data-manager"); - return { - role, - close() { - role.close(); - executions.close(); - control.close(); - data.close(); - }, - }; -} -function publishCandidate( +/** + * The candidate a transit phase acts on, as the agent receives it with the + * child's grant. Null when the phase has no retained candidate. + */ +export function publishedDataTransitCandidate( authority: VerifiedExecutionAuthority, - candidate: z.infer, -): void { - const s = executionAuthorityState(authority), - slot = - s.reservations.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal) ?? - s.plan.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal); - if (!slot) throw Error("Dated original candidate slot required"); - assertControlExecutionForward(authority); - const payload = publishedSchema.parse({ - version: 2, - installationId: s.installation.installation.installationId, - origin: s.origin, - candidate, - childOperationKey: slot.execution.operationKey, - operationFingerprint: - "operationFingerprint" in slot - ? slot.operationFingerprint - : slot.execution.operationFingerprint, - serviceIds: [...slot.serviceIds], - }), - bytes = canonicalJson( - signControl(s.installation.privateKey, "openmapx-data-candidate-publication-v2", payload), - ), - directory = sharedDirectory(s.installation), - name = `${slot.execution.operationKey}.data-candidate.json`; + input: { phase: ControlChildPhase; ordinal: number }, +): Record | undefined { + const s = executionAuthorityState(authority); + let saved: unknown; try { - let prior: Buffer | undefined; - try { - prior = directory.role.read(name, 8192); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - if (prior) { - if (prior.toString("utf8") !== bytes) throw Error("Original published candidate changed"); - return; - } - directory.role.write(name, bytes, true); - } finally { - directory.close(); + saved = readPrivateRecord(s.installation, "observations", name(authority, input.ordinal), 8192); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; } + const candidate = verifyControl( + s.installation.installation.publicKey, + "openmapx-data-candidate-v1", + saved, + schema, + ); + const slot = + s.reservations.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal) ?? + s.plan.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal); + if (!slot || candidate.phase !== input.phase) + throw Error("Dated original candidate slot required"); + return publishedSchema.parse({ + version: 2, + installationId: s.installation.installation.installationId, + origin: s.origin, + candidate, + childOperationKey: slot.execution.operationKey, + operationFingerprint: + "operationFingerprint" in slot + ? slot.operationFingerprint + : slot.execution.operationFingerprint, + serviceIds: [...slot.serviceIds], + }); } -/** Fixed registry resource input. This reads shared signed evidence, never a - * sibling role's private state, and never authorizes execution by itself. */ +/** + * Fixed registry resource input: the candidate data-manager recorded with the + * child's grant in the lease store, re-checked against the files. It never + * authorizes execution by itself. + */ export function inspectDataTransitCandidate( installation: ControlInstallation, operationKey: string, @@ -529,93 +512,89 @@ export function inspectDataTransitCandidate( (operation.kind !== "motis.primary.promote" && operation.kind !== "feedProxy.validateAndReload") ) throw Error("Fixed transit candidate required"); - const local = installationState(installation), - directory = sharedDirectory(local); + const local = installationState(installation); + if (!local.leases.listSync) throw Error("This role cannot read delegations"); + const grant = local.leases + .listSync() + .flatMap((lease) => lease.children) + .find( + (child) => child.execution.operationKey === operationKey && child.recipient === "ops-agent", + ); + if (!grant?.evidence) throw Error("Transit candidate missing from the child grant"); + const proof = publishedSchema.parse(grant.evidence), + candidate = proof.candidate; + if ( + proof.installationId !== installation.installationId || + proof.childOperationKey !== operationKey || + proof.operationFingerprint !== controlDigest(operation) || + operationKey !== + controlChildOperationKey( + candidate.execution.operationKey, + candidate.phase, + candidate.ordinal, + ) || + candidate.execution.executor !== "data-manager" + ) + throw Error("Original candidate parent/phase/body changed"); + const expectedEpoch = verifyCatalogActivationCandidate(installation, candidate); + const promotion = operation.kind === "motis.primary.promote"; + if ( + promotion + ? candidate.phase !== "promote-primary" || + candidate.candidateId !== operation.preparedRunId || + candidate.candidateId !== expectedEpoch || + canonicalJson(proof.serviceIds) !== canonicalJson(["motis"]) + : !["proxy-stage", "proxy-commit", "proxy-restore"].includes(candidate.phase) || + candidate.candidateId !== operation.candidateId || + canonicalJson(proof.serviceIds) !== canonicalJson(["motis-feed-proxy"]) + ) + throw Error("Candidate identity/target mismatch"); + const root = realpathSync(local.dataRoot), + path = realpathSync(join(root, promotion ? "motis/live" : "motis-feed-proxy")); + if (!path.startsWith(root + sep)) throw Error("Candidate escaped data root"); + const confined = new AnchoredDirectory(path); try { - const proof = verifyControl( - local.publicKeys["data-manager"], - "openmapx-data-candidate-publication-v2", - JSON.parse( - directory.role.read(`${operationKey}.data-candidate.json`, 8192).toString("utf8"), - ), - publishedSchema, - ), - candidate = proof.candidate; - if ( - proof.installationId !== installation.installationId || - proof.childOperationKey !== operationKey || - proof.operationFingerprint !== controlDigest(operation) || - operationKey !== - controlChildOperationKey( - candidate.execution.operationKey, - candidate.phase, - candidate.ordinal, - ) || - candidate.execution.executor !== "data-manager" - ) - throw Error("Original candidate parent/phase/body changed"); - const expectedEpoch = verifyCatalogActivationCandidate(installation, candidate); - const promotion = operation.kind === "motis.primary.promote"; - if ( - promotion - ? candidate.phase !== "promote-primary" || - candidate.candidateId !== operation.preparedRunId || - candidate.candidateId !== expectedEpoch || - canonicalJson(proof.serviceIds) !== canonicalJson(["motis"]) - : !["proxy-stage", "proxy-commit", "proxy-restore"].includes(candidate.phase) || - candidate.candidateId !== operation.candidateId || - canonicalJson(proof.serviceIds) !== canonicalJson(["motis-feed-proxy"]) - ) - throw Error("Candidate identity/target mismatch"); - const root = realpathSync(local.dataRoot), - path = realpathSync(join(root, promotion ? "motis/live" : "motis-feed-proxy")); - if (!path.startsWith(root + sep)) throw Error("Candidate escaped data root"); - const confined = new AnchoredDirectory(path); - try { - if (canonicalJson(identity(confined.stat)) !== canonicalJson(candidate.directory)) - throw Error("Candidate directory replaced"); - const expected = promotion - ? ["motis-candidate-manifest.json"] - : ["conf/default.conf", "feed-proxy-vars.json"]; - if (canonicalJson(candidate.files.map((f) => f.name)) !== canonicalJson(expected)) - throw Error("Candidate files changed"); - let manifestBytes: Buffer | undefined; - for (const file of candidate.files) { - const parts = file.name.split("/"), - nested = parts.length === 2 ? confined.directory(parts[0]!) : undefined, - parent = nested ?? confined; - try { - const current = readCandidateFile(parent, parts.at(-1)!); - if ( - canonicalJson(current.inode) !== canonicalJson(file.inode) || - current.sha256 !== file.sha256 - ) - throw Error("Candidate inode/bytes replaced"); - if (promotion) manifestBytes = current.bytes; - } finally { - nested?.close(); - } - } - if (promotion) { + if (canonicalJson(identity(confined.stat)) !== canonicalJson(candidate.directory)) + throw Error("Candidate directory replaced"); + const expected = promotion + ? ["motis-candidate-manifest.json"] + : ["conf/default.conf", "feed-proxy-vars.json"]; + if (canonicalJson(candidate.files.map((f) => f.name)) !== canonicalJson(expected)) + throw Error("Candidate files changed"); + let manifestBytes: Buffer | undefined; + for (const file of candidate.files) { + const parts = file.name.split("/"), + nested = parts.length === 2 ? confined.directory(parts[0]!) : undefined, + parent = nested ?? confined; + try { + const current = readCandidateFile(parent, parts.at(-1)!); if ( - !manifestBytes || - candidate.consumer !== null || - candidateArtifacts(confined, manifestBytes, expectedEpoch) !== candidate.artifactsDigest + canonicalJson(current.inode) !== canonicalJson(file.inode) || + current.sha256 !== file.sha256 ) - throw Error("Prepared candidate artifacts changed"); - } else if ( - candidate.artifactsDigest !== null || - canonicalJson(proxyConsumer(confined, candidate.files[0]!)) !== - canonicalJson(candidate.consumer) - ) - throw Error("Mounted candidate inode/bytes replaced"); - } finally { - confined.close(); + throw Error("Candidate inode/bytes replaced"); + if (promotion) manifestBytes = current.bytes; + } finally { + nested?.close(); + } } - return { digest: controlDigest(proof), proof }; + if (promotion) { + if ( + !manifestBytes || + candidate.consumer !== null || + candidateArtifacts(confined, manifestBytes, expectedEpoch) !== candidate.artifactsDigest + ) + throw Error("Prepared candidate artifacts changed"); + } else if ( + candidate.artifactsDigest !== null || + canonicalJson(proxyConsumer(confined, candidate.files[0]!)) !== + canonicalJson(candidate.consumer) + ) + throw Error("Mounted candidate inode/bytes replaced"); } finally { - directory.close(); + confined.close(); } + return { digest: controlDigest(proof), proof }; } /** Recheck the actual admitted parent, target lease and snapshot at each fixed * Docker effect boundary; a previously returned resource claim is insufficient. */ @@ -669,8 +648,6 @@ export const { commitFeedProxyValidationFailure, recordFeedProxyReloadAttempted, inspectFeedProxyValidationFailure, - publishFeedProxyCanonicalTerminal, - verifyFeedProxyCanonicalTerminal, } = (() => { const digest = z.string().regex(/^[a-f0-9]{64}$/); const schema = z @@ -781,12 +758,6 @@ export const { const prior = read(authority); if (prior.candidateId !== candidateId || prior.candidateDigest !== candidateDigest) throw Error("Validation candidate changed"); - const dir = witnessDirectory(s.installation); - try { - dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096); - } finally { - dir.close(); - } return; } write( @@ -808,7 +779,6 @@ export const { }), true, ); - reserveWitness(authority); }); } function commitFeedProxyValidationFailure( @@ -868,195 +838,11 @@ export const { return proofDigest; } - const terminalSchema = z.strictObject({ - version: z.literal(2), - installationId: z.uuid(), - origin: controlOriginSchema, - execution: executionRefSchema, - admissionDigest: digest, - phase: z.enum(["pending", "canonical"]), - status: z.unknown().nullable(), - terminalAt: z.number().int().positive().safe().nullable(), - terminalDigest: digest.nullable(), - padding: z.string().max(4096), - }); - function witnessDirectory(local: ReturnType["installation"]) { - const data = new AnchoredDirectory(local.dataRoot), - control = data.directory(".admin-target-control"), - executions = control.directory("executions"), - role = executions.directory("ops-agent"); - return { - role, - close() { - role.close(); - executions.close(); - control.close(); - data.close(); - }, - }; - } - function writeWitness( - authority: VerifiedExecutionAuthority, - payload: z.infer, - exclusive: boolean, - ) { - const s = state(authority), - dir = witnessDirectory(s.installation), - name = `${s.execution.operationKey}.proxy-terminal.json`; - try { - const value = { ...payload, padding: "" }, - domain = "openmapx-proxy-canonical-terminal-v2"; - const room = - 4096 - - Buffer.byteLength(canonicalJson(signControl(s.installation.privateKey, domain, value))); - if (room < 0) throw Error("Proxy terminal cell capacity exceeded"); - value.padding = " ".repeat(room); - const bytes = canonicalJson(signControl(s.installation.privateKey, domain, value)); - dir.role.write(name, bytes, exclusive); - } finally { - dir.close(); - } - } - function reserveWitness(authority: VerifiedExecutionAuthority) { - const s = state(authority); - writeWitness( - authority, - terminalSchema.parse({ - version: 2, - installationId: s.installation.installation.installationId, - origin: s.origin, - execution: s.execution, - admissionDigest: s.receiptDigest, - phase: "pending", - status: null, - terminalAt: null, - terminalDigest: null, - padding: "", - }), - true, - ); - } - /** Only an already sealed original complete canonical tuple can publish this - * finite shared proxy witness. It creates no new terminal time or status. */ - function publishFeedProxyCanonicalTerminal( - authority: VerifiedExecutionAuthority, - input: OpsJobStatus, - terminalAt: number, - ) { - const s = state(authority), - status = parseOpsJobStatus(input), - terminalDigest = controlTerminalReceiptDigest(status); - const original = verifyControl( - s.installation.installation.publicKey, - "openmapx-control-terminal-v1", - readPrivateRecord( - s.installation, - "observations", - `terminal-${s.execution.operationKey}.json`, - 8192, - ), - z - .object({ - phase: z.literal("canonical"), - execution: executionRefSchema, - admissionDigest: digest, - terminalAt: z.number(), - terminalDigest: digest, - }) - .passthrough(), - ); - if ( - controlDigest(original.execution) !== controlDigest(s.execution) || - original.admissionDigest !== s.receiptDigest || - original.terminalAt !== terminalAt || - original.terminalDigest !== terminalDigest - ) - throw Error("Original canonical proxy terminal required"); - const dir = witnessDirectory(s.installation); - try { - const prior = verifyControl( - s.installation.installation.publicKey, - "openmapx-proxy-canonical-terminal-v2", - JSON.parse( - dir.role.read(`${s.execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), - ), - terminalSchema, - ); - if ( - prior.installationId !== s.installation.installation.installationId || - prior.admissionDigest !== s.receiptDigest || - !sameControlOrigin(prior.origin, s.origin) || - controlDigest(prior.execution) !== controlDigest(s.execution) - ) - throw Error("Original reserved proxy terminal changed"); - if (prior.phase === "canonical") { - if (prior.terminalDigest !== terminalDigest || prior.terminalAt !== terminalAt) - throw Error("Proxy canonical terminal changed"); - return; - } - // Diagnostics are observational and deliberately excluded from the exact - // canonical digest. Keep the fixed shared tuple bounded. - writeWitness( - authority, - { - ...prior, - phase: "canonical", - status: { ...status, diagnostics: { tail: [], truncated: false } }, - terminalDigest, - terminalAt, - }, - false, - ); - } finally { - dir.close(); - } - } - function verifyFeedProxyCanonicalTerminal( - parent: VerifiedExecutionAuthority, - execution: ExecutionRef, - ): OpsJobStatus { - const s = executionAuthorityState(parent), - dir = witnessDirectory(s.installation); - try { - const proof = verifyControl( - s.installation.publicKeys["ops-agent"], - "openmapx-proxy-canonical-terminal-v2", - JSON.parse( - dir.role.read(`${execution.operationKey}.proxy-terminal.json`, 4096).toString("utf8"), - ), - terminalSchema, - ); - if ( - proof.phase !== "canonical" || - proof.installationId !== s.installation.installation.installationId || - !sameControlOrigin(proof.origin, s.origin) || - controlDigest(proof.execution) !== controlDigest(execution) || - !proof.terminalAt - ) - throw Error("Original canonical proxy child missing"); - const status = parseOpsJobStatus(proof.status); - if ( - status.kind !== "feedProxy.validateAndReload" || - status.operationId !== execution.executionId || - status.operationKey !== execution.operationKey || - status.fingerprint !== execution.fingerprint || - status.deadlineAt !== execution.deadlineAt || - controlTerminalReceiptDigest(status) !== proof.terminalDigest - ) - throw Error("Canonical proxy status changed"); - return status; - } finally { - dir.close(); - } - } - return { feedProxyValidationCompletionDigest, reserveFeedProxyValidation, commitFeedProxyValidationFailure, recordFeedProxyReloadAttempted, inspectFeedProxyValidationFailure, - publishFeedProxyCanonicalTerminal, - verifyFeedProxyCanonicalTerminal, }; })(); diff --git a/packages/core/src/build-artifacts/data-execution-journal.ts b/packages/core/src/build-artifacts/data-execution-journal.ts index 146b63ea5..efe1e7610 100644 --- a/packages/core/src/build-artifacts/data-execution-journal.ts +++ b/packages/core/src/build-artifacts/data-execution-journal.ts @@ -28,8 +28,9 @@ import { controlDigest, executionAuthorityState, installationState, - publishDataControlTerminalWitness, + readOpsControlTerminalObservation, readPrivateRecord, + recordDataControlTerminalObservation, reserveControlTerminal, restoreAdmittedControlExecution, settleExecutionLifecycle, @@ -46,16 +47,12 @@ import { } from "./control-records"; import { retainedDataDatabaseTimestamp } from "./data-database-time"; import { DataExecutionPartialResult } from "./data-execution-outcome"; -import { verifyFeedProxyCanonicalTerminal } from "./feed-proxy-validation"; import { assertRetainedCatalogActivationChildrenSettled, inspectArtifactTargetLeaseChildSettlement, inspectRetainedOwnedArtifactTargetChildSettlement, restoreArtifactTargetLeaseForSettlement, - restoreGenerationHardlinksLeaseForSettlement, type TargetLeaseAuthority, - verifyRetainedArtifactTargetRootRelease, - verifyRetainedGenerationHardlinksSettlement, } from "./target-lease"; export interface DataExecutionJournalDatabase { @@ -280,11 +277,14 @@ export function createDataExecutionControlJournal( ) throw Error("Original proxy phase changed"); if (child.state !== "terminal" || !child.execution) continue; - const shared = inspectRetainedOwnedArtifactTargetChildSettlement( + const shared = await inspectRetainedOwnedArtifactTargetChildSettlement( authority, child.execution, ); - const canonical = verifyFeedProxyCanonicalTerminal(authority, child.execution); + const canonical = readOpsControlTerminalObservation( + s.installation.installation, + child.execution, + ); if ( restore && child.phase !== "proxy-restore" && @@ -309,8 +309,11 @@ export function createDataExecutionControlJournal( child.execution.operationKey !== restore.execution.operationKey ) throw Error("Original proxy restore obligation remains pending"); - const shared = inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution), - canonical = verifyFeedProxyCanonicalTerminal(authority, child.execution); + const shared = await inspectRetainedOwnedArtifactTargetChildSettlement( + authority, + child.execution, + ), + canonical = readOpsControlTerminalObservation(s.installation.installation, child.execution); const result = z .strictObject({ candidateId: z.string(), reloaded: z.literal(true) }) .parse(canonical.result); @@ -342,7 +345,7 @@ export function createDataExecutionControlJournal( controlDigest(input) !== state.execution.operationFingerprint ) throw Error("Original reviewed catalog activation required"); - assertRetainedCatalogActivationChildrenSettled(authority); + await assertRetainedCatalogActivationChildrenSettled(authority); const children = value.row.execution_children.map((raw) => z .object({ @@ -369,7 +372,7 @@ export function createDataExecutionControlJournal( slot.operationFingerprint !== child.operationFingerprint || controlDigest(child.operation) !== child.operationFingerprint || controlDigest(child.parentExecution) !== controlDigest(state.execution) || - !inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution) + !(await inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution)) ) throw Error("Original settled activation child required"); } @@ -380,7 +383,10 @@ export function createDataExecutionControlJournal( const child = matches[0]!, operation = pairedPinApprovalInputSchema.parse(child.operation), result = pairedPinApprovalResultSchema.parse(child.approvalResult); - const physical = inspectRetainedOwnedArtifactTargetChildSettlement(authority, child.execution); + const physical = await inspectRetainedOwnedArtifactTargetChildSettlement( + authority, + child.execution, + ); if ( !physical || physical.outcome !== "committed" || @@ -903,9 +909,10 @@ export function createDataExecutionControlJournal( ) throw Error("Original worker child snapshot changed"); await assertOwnedProxyRecoverySettled(execution); + // A hardlink leaf runs under its caller's lease; it holds none of its own. const lease = value.status.kind === "generation-hardlinks" - ? restoreGenerationHardlinksLeaseForSettlement(authority) + ? undefined : restoreArtifactTargetLeaseForSettlement(authority), error = proof.outcome === "failed" ? Error("Original contained worker failed") : undefined; const completion = Object.freeze({ [completionBrand]: true as const }); @@ -985,7 +992,7 @@ export function createDataExecutionControlJournal( for (const child of children) { if (child.state !== "terminal" || !child.execution || !child.settlement || !owned.lease) throw Error("Unsettled data descendant"); - const proof = inspectArtifactTargetLeaseChildSettlement(owned.lease, child.execution); + const proof = await inspectArtifactTargetLeaseChildSettlement(owned.lease, child.execution); if ( !proof || controlDigest(proof) !== @@ -1089,12 +1096,11 @@ export function createDataExecutionControlJournal( }); contained.set(execution.operationKey, proof.terminalDigest); }, - async publishContainedTerminal(execution: ExecutionRef) { - const { value, authority, proof } = await verifyStored(execution); - if (value.status.kind === "generation-hardlinks") - verifyRetainedGenerationHardlinksSettlement(authority); - else verifyRetainedArtifactTargetRootRelease(authority); - publishDataControlTerminalWitness(authority, value.status, proof.terminalAt); + /** Records the committed terminal once the execution's own lease is released, + * for later steps of this role that build on it. */ + async recordContainedTerminal(execution: ExecutionRef) { + const { value } = await verifyStored(execution); + recordDataControlTerminalObservation(installation, execution, value.status); }, repairTerminal: async (execution: ExecutionRef) => { await assertOwnedProxyRecoverySettled(execution); diff --git a/packages/core/src/build-artifacts/feed-proxy-validation.ts b/packages/core/src/build-artifacts/feed-proxy-validation.ts index f6101a411..b674e7911 100644 --- a/packages/core/src/build-artifacts/feed-proxy-validation.ts +++ b/packages/core/src/build-artifacts/feed-proxy-validation.ts @@ -3,8 +3,6 @@ export { commitFeedProxyValidationFailure, feedProxyValidationCompletionDigest, inspectFeedProxyValidationFailure, - publishFeedProxyCanonicalTerminal, recordFeedProxyReloadAttempted, reserveFeedProxyValidation, - verifyFeedProxyCanonicalTerminal, } from "./data-execution-candidate"; diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index ef26de5da..1b0c172be 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -140,10 +140,7 @@ it.each(["body", "source", "parent", "foreign-target", "protected-target", "capa }, ); -it("retains the frozen coordinator image in the cross-role graph used to reconstruct nonempty hardlinks", async () => { - const { configurationMaintenanceForwardGraphSchema } = await import( - "./configuration-maintenance-forward-records" - ); +it("rebuilds the same plan from the frozen coordinator image", () => { const source = input(); const coordinatorImage = { action: "apply" as const, @@ -154,24 +151,9 @@ it("retains the frozen coordinator image in the cross-role graph used to reconst pullSubmissionDigest: null, }; const plan = buildFixedConfigurationMaintenancePlan({ ...source, coordinatorImage }); - const { kind: _kind, ...binding } = plan.operation; - const graph = configurationMaintenanceForwardGraphSchema.parse({ - version: 1, - binding, - prepare: plan.prepare, - apiJobId: plan.apiParent.apiJobId, - parentOperationKey: plan.apiParent.operationKey, - parentFingerprint: plan.apiParent.fingerprint, - parentDeadlineAt: plan.apiParent.deadlineAt, - planDigest: plan.planDigest, - serviceIds: plan.serviceIds, - steps: plan.steps, - coordinatorImage, - }); - expect(graph.coordinatorImage).toEqual(coordinatorImage); const rebuilt = buildFixedConfigurationMaintenancePlan({ ...source, - coordinatorImage: graph.coordinatorImage, + coordinatorImage: structuredClone(coordinatorImage), }); expect(rebuilt.planDigest).toBe(plan.planDigest); expect(rebuilt.steps).toEqual(plan.steps); @@ -295,7 +277,6 @@ it("freezes the checked release helper graph and plans a complete 13-service sta sourceDigest: "b".repeat(64), environmentNames: [], preparerAdmissionDigest: "c".repeat(64), - ownerCertificateDigest: "d".repeat(64), leaseId: "88888888-8888-4888-8888-888888888888", delegationId: "99999999-9999-4999-8999-999999999999", assertion: "original-fixed-assertion", @@ -321,26 +302,7 @@ it("freezes the checked release helper graph and plans a complete 13-service sta operation: { ...intent.operation, mode: { kind: "configuration" } }, }).success, ).toBe(false); - const { configurationMaintenanceForwardGraphSchema } = await import( - "./configuration-maintenance-forward-records" - ); - const { kind: _kind, ...binding } = operation; - if (source.apiParent.issuer !== "api") throw Error("API graph required"); - const graph = configurationMaintenanceForwardGraphSchema.parse({ - version: 1, - binding, - prepare, - apiJobId: source.apiParent.apiJobId, - parentOperationKey: source.apiParent.operationKey, - parentFingerprint: source.apiParent.fingerprint, - parentDeadlineAt: source.apiParent.deadlineAt, - planDigest: plan.planDigest, - serviceIds: plan.serviceIds, - steps: plan.steps, - handoff: plan.handoff, - release: plan.release, - }); - expect(graph.release).toEqual(release); + expect(plan.release).toEqual(release); // A realistic stack must fit without relaxing the private-record limits. const names = [ @@ -460,25 +422,7 @@ it("freezes protected handoff after the original generation and ordinary childre recipient: "configuration-helper", serviceIds: handoff.serviceIds, }); - const { configurationMaintenanceForwardGraphSchema } = await import( - "./configuration-maintenance-forward-records" - ); - const { kind: _kind, ...binding } = operation; - if (source.apiParent.issuer !== "api") throw Error("API plan fixture required"); - const graph = configurationMaintenanceForwardGraphSchema.parse({ - version: 1, - binding, - prepare, - apiJobId: source.apiParent.apiJobId, - parentOperationKey: source.apiParent.operationKey, - parentFingerprint: source.apiParent.fingerprint, - parentDeadlineAt: source.apiParent.deadlineAt, - planDigest: plan.planDigest, - serviceIds: plan.serviceIds, - steps: plan.steps, - handoff: plan.handoff, - }); - expect(graph).toHaveProperty("handoff", handoff); + expect(plan.handoff).toEqual(handoff); expect(plan.nestedHandoff?.execution.operationKey).toBe( controlChildOperationKey(step.operationKey, "configuration-handoff", 0), ); diff --git a/packages/core/src/build-artifacts/fixed-data-execution.ts b/packages/core/src/build-artifacts/fixed-data-execution.ts index 77d8df04e..353dba6dc 100644 --- a/packages/core/src/build-artifacts/fixed-data-execution.ts +++ b/packages/core/src/build-artifacts/fixed-data-execution.ts @@ -21,10 +21,10 @@ import { controlDigest, executionAuthorityState, readPrivateRecord, + recordDataControlTerminalObservation, signControl, type VerifiedExecutionAuthority, verifyControl, - verifyDataControlTerminalWitness, withPrivateEffectWriter, writePrivateRecord, } from "./control-authority"; @@ -271,7 +271,12 @@ export function verifyCliDataTerminal( binding.execution.fingerprint !== status.fingerprint ) throw Error("Original bound data command required"); - return verifyDataControlTerminalWitness(s.installation.installation, binding.execution, status); + // The status was read from data-manager over its authenticated route. + return recordDataControlTerminalObservation( + s.installation.installation, + binding.execution, + status, + ); } export function cliDataReservation( diff --git a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts index 8b009b978..3481601bf 100644 --- a/packages/core/src/build-artifacts/generation-hardlinks-owner.ts +++ b/packages/core/src/build-artifacts/generation-hardlinks-owner.ts @@ -39,17 +39,18 @@ import { assertControlExecutionForward, controlRolePublicKey, executionAuthorityState, + readDataControlTerminalObservation, readPrivateRecord, type VerifiedExecutionAuthority, - verifyDataControlTerminalWitness, writePrivateRecord, } from "./control-authority"; import { controlDigest, signControl, verifyControl } from "./control-records"; import { assertArtifactTargetLeaseExecution, - inspectArtifactTargetLeaseChildSettlement, + inspectLocalLeaseChildSettlement, + settleDataManagerLeaseChild, type TargetLeaseAuthority, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, } from "./target-lease"; const schema = z.strictObject({ @@ -249,24 +250,14 @@ export async function executeOwnedGenerationHardlinks(options: { signal?.aborted ? "interrupted" : "transport", ); } - let contained = false; - const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt + 30000); - do { - try { - // The fixed DM publisher verifies physical settlement before signing this - // witness. Observe it without taking the lease writer; polling that writer - // here can itself make the publisher fail its nonblocking guard. - verifyDataControlTerminalWitness(s.installation.installation, bound.execution, status); - const settlement = inspectArtifactTargetLeaseChildSettlement(lease, bound.execution); - contained = settlement?.outcome === "committed" && settlement.publication === "committed"; - } catch { - /* SQL terminal and borrowed-node settlement precede the final shared - * contained-terminal witness. Both actual sources must be visible. */ - } - if (contained || Date.now() >= until) break; - await new Promise((resolve) => setTimeout(resolve, 50)); - } while (!contained && Date.now() < until); - if (!contained || status.status !== "ok" || status.result?.kind !== "generation-hardlinks") + // Data-manager's terminal status, read over its authenticated route, settles + // the leaf in this owner's lease. An unknown publication leaves it held. + const settlement = await settleDataManagerLeaseChild(lease, bound.execution, status); + if ( + settlement?.outcome !== "committed" || + status.status !== "ok" || + status.result?.kind !== "generation-hardlinks" + ) throw new DataExecutionObservationPending(identity, "transport"); return hardlinkApplySummarySchema.parse(status.result.hardlinks); } @@ -277,19 +268,17 @@ export function inspectOwnedGenerationHardlinks( authority: VerifiedExecutionAuthority, lease?: TargetLeaseAuthority, ) { - if (!lease) verifyRetainedArtifactTargetRootRelease(authority); + if (!lease) verifyLocalArtifactTargetRootRelease(authority); const owner = executionAuthorityState(authority), plan = readGenerationHardlinksOwnerPlan(authority); if (!plan.reservation) return "complete" as const; const binding = lookupControlChildPhaseBinding(authority, plan.reservation); if (!binding) return "not_attempted" as const; - const proof = verifyDataControlTerminalWitness( + const proof = readDataControlTerminalObservation( owner.installation.installation, binding.child.execution, ), - settled = lease - ? inspectArtifactTargetLeaseChildSettlement(lease, binding.child.execution) - : null; + settled = lease ? inspectLocalLeaseChildSettlement(lease, binding.child.execution) : null; if ( lease && (!settled || diff --git a/packages/core/src/build-artifacts/index.ts b/packages/core/src/build-artifacts/index.ts index b46494f8e..acd11f904 100644 --- a/packages/core/src/build-artifacts/index.ts +++ b/packages/core/src/build-artifacts/index.ts @@ -23,6 +23,7 @@ export { } from "./configuration-source-witness"; export { type ArtifactDelegationAssertion, + agentStoreLeases, assertControlExecutionForward, assertControlStorageAvailable, assertFreshControlRoot, @@ -48,6 +49,7 @@ export { executionAuthorityState, fixedCliArguments, inspectConfigurationPreEffectRefusal, + inspectRemoteDelegation, loadControlStartup, localFeedCredentialTemplateMac, markFeedProxyReloadAttempted, @@ -60,9 +62,13 @@ export { prepareFixedControlChild, provisionControlInstallation, readConfigurationHandoffSpawnIntent, + readDataControlTerminalObservation, readFixedCliSpawnIntent, + readOpsControlTerminalObservation, readPrivateRecord, readVerifiedStagedCoordinatorOwnership, + recordDataControlTerminalObservation, + recordOpsControlTerminalObservation, repairControlJournalTerminal, replaceControlRegisteredServiceIds, reserveControlTerminal, @@ -85,6 +91,7 @@ export { type VerifiedStagedCoordinatorOwnerBaseline, type VerifiedStagedCoordinatorOwnership, verifiedCoordinatorPullBaselineState, + verifiedRemoteParentExecution, verifiedRetainedControlParent, verifiedStagedCoordinatorOwnerBaselineState, verifiedStagedCoordinatorOwnershipState, @@ -92,7 +99,6 @@ export { verifyControlPeer, verifyControlRecoveryAuthority, verifyCoordinatorPullAdmissionSource, - verifyDataControlTerminalWitness, verifyFixedCliControlPeer, verifyMaintenanceCoordinatorPrerequisite, verifyParentExecution, @@ -115,6 +121,7 @@ export { assertDataTransitCandidateForExecution, assertOwnedDataTransitCandidate, inspectDataTransitCandidate, + publishedDataTransitCandidate, retainOwnedDataTransitCandidate, } from "./data-execution-candidate"; export { @@ -136,7 +143,6 @@ export { feedProxyValidationCompletionDigest, inspectFeedProxyValidationFailure, reserveFeedProxyValidation, - verifyFeedProxyCanonicalTerminal, } from "./feed-proxy-validation"; export { AnchoredDirectory } from "./filesystem"; export { @@ -175,6 +181,13 @@ export { readOwnedGenerationHardlinkPlan, retainGenerationHardlinksOwnerPlan, } from "./generation-hardlinks-owner"; +export { + type LeaseOperation, + type LeaseSender, + opsClientLeaseSender, + remoteLeaseBackend, +} from "./lease-remote"; +export * from "./lease-store"; export * from "./manifest"; export { CANDIDATE_MANIFEST_FILENAME, @@ -186,15 +199,7 @@ export { readCandidateManifest, verifyCandidateManifest, } from "./motis-candidate"; -export { - inspectControlAdmission, - inspectDataExecutionAdmission, - verifiedParentForArtifactDelegation, -} from "./peer-admission"; +export { inspectControlAdmission, inspectDataExecutionAdmission } from "./peer-admission"; export * from "./prepared-evidence"; -export { - reconcileFailedServiceLifecycle, - recoverServiceLifecycleByKey, -} from "./service-lifecycle-recovery"; export * from "./target-lease"; export { verifyConfigurationMaintenanceRootClosure as assertConfigurationMaintenanceRootClosure } from "./target-lease"; diff --git a/packages/core/src/build-artifacts/lease-remote.ts b/packages/core/src/build-artifacts/lease-remote.ts new file mode 100644 index 000000000..49b2d8afb --- /dev/null +++ b/packages/core/src/build-artifacts/lease-remote.ts @@ -0,0 +1,74 @@ +import type { OpsClient } from "../ops/client"; +import { type OpsOperation, type OpsResultFor, parseOpsResult } from "../ops/contract"; +import { TargetBusyError } from "./lease-store"; +import type { LeaseBackend } from "./target-lease"; + +export type LeaseOperation = Extract< + OpsOperation, + { kind: "lease.acquire" | "lease.read" | "lease.update" } +>; +/** Sends one lease operation to the agent and returns its typed result. */ +export type LeaseSender = (operation: LeaseOperation) => Promise; +/** Lease operations over the agent's HTTP route, with the caller's own token. */ +export function opsClientLeaseSender(client: OpsClient | (() => OpsClient)): LeaseSender { + return async (operation) => { + const submitted = await (typeof client === "function" ? client() : client).execute(operation); + if (submitted.execution !== "sync") throw Error("Lease operations answer at once"); + return submitted.value; + }; +} + +/** + * Leases through the agent, for data-manager and the host CLI. Every call is a + * typed agent operation under the caller's own token; the agent applies the + * same rules as to its own leases and refuses another principal's. + */ +export function remoteLeaseBackend(send: LeaseSender): LeaseBackend { + async function run( + operation: Extract, + ): Promise> { + return parseOpsResult(operation.kind, await send(operation)) as OpsResultFor; + } + const update = async ( + leaseId: string, + action: Extract["action"], + ) => (await run({ kind: "lease.update", leaseId, action })).lease; + return { + async acquire(input) { + const result = await run({ + kind: "lease.acquire", + owner: input.owner, + ownerKind: input.ownerKind, + targets: [...input.targets].sort(), + }); + if (result.outcome === "busy") throw new TargetBusyError(result.target, result.holder); + return result.lease; + }, + read: async (leaseId) => (await run({ kind: "lease.read", leaseId })).lease ?? undefined, + list: async () => { + throw Error("Only the agent lists every lease"); + }, + async delegate(input) { + const lease = await update(input.leaseId, { action: "delegate", child: input.child }); + if (!lease) throw Error("Target lease is not held"); + return lease; + }, + adopt: async () => { + throw Error("Children of another role are adopted in the agent"); + }, + async settle(input) { + const lease = await update(input.leaseId, { + action: "settle", + key: input.key, + settlement: input.settlement, + }); + if (!lease) throw Error("Target lease is not held"); + return lease; + }, + cancel: async (input) => + (await update(input.leaseId, { action: "cancel", key: input.key })) ?? undefined, + release: async (input) => { + await update(input.leaseId, { action: "release" }); + }, + }; +} diff --git a/packages/core/src/build-artifacts/lease-store.test.ts b/packages/core/src/build-artifacts/lease-store.test.ts new file mode 100644 index 000000000..8a0a0a5fa --- /dev/null +++ b/packages/core/src/build-artifacts/lease-store.test.ts @@ -0,0 +1,169 @@ +import { randomUUID } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ExecutionRef } from "../ops/execution-ref"; +import { + heldTargets, + type LeaseSettlement, + leaseStoreDirectory, + openLeaseStore, + TargetBusyError, +} from "./lease-store"; +import { localLeaseBackend } from "./target-lease"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function backend() { + const infraDir = mkdtempSync(join(tmpdir(), "omx-lease-store-")); + roots.push(infraDir); + const store = openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: "test" }); + return { infraDir, store, leases: localLeaseBackend(store) }; +} +function execution(executor: ExecutionRef["executor"] = "host-cli"): ExecutionRef { + return { + executor, + principal: executor === "ops-agent" ? "api" : executor, + executionId: + executor === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), + operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: Date.now() + 60_000, + }; +} +const settled = (identity: ExecutionRef): LeaseSettlement => ({ + identity, + outcome: "committed", + publication: "committed", + terminalReceiptDigest: "c".repeat(64), +}); + +describe("the agent's lease store", () => { + it("holds targets until the owner settles, then drops the released row", async () => { + const { infraDir, leases } = backend(), + owner = execution(); + const lease = await leases.acquire({ + owner, + ownerKind: "service.restart", + targets: ["motis", "alpha"], + }); + expect(lease.targets).toEqual(["alpha", "motis"]); + expect(heldTargets(infraDir)).toEqual(["alpha", "motis"]); + await expect(leases.release({ leaseId: lease.leaseId })).rejects.toThrow(/settlement/); + await leases.settle({ leaseId: lease.leaseId, key: null, settlement: settled(owner) }); + await leases.release({ leaseId: lease.leaseId }); + expect(heldTargets(infraDir)).toEqual([]); + }); + + it("refuses a held target and names the holder's operation", async () => { + const { leases } = backend(), + first = execution("ops-agent"); + await leases.acquire({ owner: first, ownerKind: "service.stop", targets: ["redis"] }); + const refusal = leases.acquire({ + owner: execution(), + ownerKind: "service.restart", + targets: ["osrm", "redis"], + }); + await expect(refusal).rejects.toBeInstanceOf(TargetBusyError); + await expect(refusal).rejects.toThrow(first.operationKey); + await expect(refusal).rejects.toThrow("openmapx control status"); + }); + + it("returns the same lease to a repeated acquire of the same owner", async () => { + const { leases } = backend(), + owner = execution(); + const first = await leases.acquire({ owner, ownerKind: "backup.create", targets: ["postgis"] }); + const again = await leases.acquire({ owner, ownerKind: "backup.create", targets: ["postgis"] }); + expect(again.leaseId).toBe(first.leaseId); + await expect( + leases.acquire({ owner, ownerKind: "backup.create", targets: ["redis"] }), + ).rejects.toThrow(/different root lease/); + }); + + it("keeps the targets held while a delegated child is unsettled", async () => { + const { leases } = backend(), + owner = execution("ops-agent"), + child = execution("data-manager"); + const lease = await leases.acquire({ owner, ownerKind: "data.update", targets: ["osrm"] }); + const key = randomUUID(); + await leases.delegate({ + leaseId: lease.leaseId, + child: { + key, + parentKey: null, + parent: owner, + execution: child, + kind: "download-osm", + phase: "osm", + recipient: "data-manager", + targets: ["osrm"], + permittedFurtherEdges: [], + depth: 1, + }, + }); + // A widened target subset and a replayed child key are refused. + await expect( + leases.delegate({ + leaseId: lease.leaseId, + child: { + key: randomUUID(), + parentKey: null, + parent: owner, + execution: execution("data-manager"), + kind: "download-osm", + phase: "osm", + recipient: "data-manager", + targets: ["motis"], + permittedFurtherEdges: [], + depth: 1, + }, + }), + ).rejects.toThrow(/Widened/); + await expect( + leases.settle({ leaseId: lease.leaseId, key: null, settlement: settled(owner) }), + ).rejects.toThrow(/Unsettled attempted child/); + await leases.settle({ leaseId: lease.leaseId, key, settlement: settled(child) }); + await leases.settle({ leaseId: lease.leaseId, key: null, settlement: settled(owner) }); + await leases.release({ leaseId: lease.leaseId }); + expect(await leases.list()).toEqual([]); + }); + + it("marks an unknown outcome as needing recovery and keeps the lease", async () => { + const { infraDir, leases } = backend(), + owner = execution(); + const lease = await leases.acquire({ owner, ownerKind: "service.stop", targets: ["alpha"] }); + const record = await leases.settle({ + leaseId: lease.leaseId, + key: null, + settlement: { ...settled(owner), outcome: "unknown", publication: "unknown" }, + }); + expect(record.state).toBe("recovery_required"); + await expect(leases.release({ leaseId: lease.leaseId })).rejects.toThrow(); + expect(heldTargets(infraDir)).toEqual(["alpha"]); + }); + + it("writes the store as plain, schema-checked JSON", async () => { + const { store, leases } = backend(), + owner = execution(); + await leases.acquire({ owner, ownerKind: "service.stop", targets: ["alpha"] }); + const file = JSON.parse(readFileSync(join(store.directory, "leases.json"), "utf8")); + expect(file).toMatchObject({ version: 1, leases: [{ owner, targets: ["alpha"] }] }); + expect(JSON.stringify(file)).not.toContain("signature"); + }); + + it("serializes concurrent writers on the store lock", async () => { + const { leases } = backend(); + const owners = Array.from({ length: 8 }, () => execution()); + const results = await Promise.allSettled( + owners.map((owner) => + leases.acquire({ owner, ownerKind: "service.stop", targets: ["shared"] }), + ), + ); + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + expect((await leases.list()).map((lease) => lease.targets)).toEqual([["shared"]]); + }); +}); diff --git a/packages/core/src/build-artifacts/lease-store.ts b/packages/core/src/build-artifacts/lease-store.ts new file mode 100644 index 000000000..58cccf33b --- /dev/null +++ b/packages/core/src/build-artifacts/lease-store.ts @@ -0,0 +1,181 @@ +import { randomUUID } from "node:crypto"; +import { lstatSync, mkdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import { canonicalJson } from "../ops/canonical-json"; +import type { ExecutionRef } from "../ops/execution-ref"; +import { + type LeaseChild, + type LeaseRecord, + type LeaseSettlement, + leaseRecordSchema, + leaseTargetsSchema, +} from "../ops/lease-records"; +import { atomicWrite } from "../release-files"; +import { type ContainerRunning, withRecoverableLock } from "../services/recoverable-lock"; + +/** + * Target leases, in the operations agent's own state directory, which no other + * service mounts. Every writer that changes containers claims its targets here + * first: the agent for its own operations and on behalf of data-manager and the + * host CLI, the configuration helper while the agent is replaced, the agent's + * fixed CLI children, and the host CLI directly only while the agent is down. + * + * The file is replaced atomically under a recoverable lock, held only for the + * read and write, never across a Docker effect. A lease names its owner and + * the children recorded under it before they start. It is released only when + * the owner and every child have a known outcome; an unknown outcome keeps it + * until an operator resolves it with `openmapx control`. + */ +export const LEASE_STORE_FILE = "leases.json"; +export const LEASE_STORE_LOCK = "leases.lock"; +const MAX_LEASES = 256; + +/** The store's lock is held by another live writer past the wait. */ +export { LockBusyError as LeaseStoreBusyError } from "../services/recoverable-lock"; +export type { LeaseChild, LeaseRecord, LeaseSettlement }; + +const storeSchema = z.strictObject({ + version: z.literal(1), + leases: z.array(leaseRecordSchema).max(MAX_LEASES), +}); + +export class TargetBusyError extends Error { + constructor( + readonly target: string, + readonly holder: Pick, + ) { + super( + `Target lease busy: ${target} is held by ${holder.owner.executor} operation ${holder.owner.operationKey} (${holder.ownerKind}, ${holder.state}). See \`openmapx control status\`.`, + ); + } +} + +export function knownSettlement(settlement: LeaseSettlement | undefined): boolean { + return ( + !!settlement && + settlement.outcome !== "unknown" && + settlement.publication !== "unknown" && + !(settlement.outcome === "failed_without_effect" && settlement.publication !== "not_started") + ); +} + +export function sameExecution(a: ExecutionRef, b: ExecutionRef): boolean { + return canonicalJson(a) === canonicalJson(b); +} + +export interface LeaseStore { + readonly directory: string; + /** Runs `change` on the current leases under the store lock and saves the result. */ + update(change: (leases: LeaseRecord[]) => T): Promise; + list(): LeaseRecord[]; + get(leaseId: string): LeaseRecord | undefined; +} + +function readStore(path: string): LeaseRecord[] { + let text: string; + try { + const stats = lstatSync(path); + if (!stats.isFile() || stats.size > 8 * 1024 * 1024) throw new Error("Unsafe lease store file"); + text = readFileSync(path, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + return storeSchema.parse(JSON.parse(text)).leases; +} + +export function openLeaseStore(options: { + /** The agent's state directory, `infra/docker/state/ops-agent`. */ + directory: string; + purpose: string; + containerRunning?: ContainerRunning; +}): LeaseStore { + const file = join(options.directory, LEASE_STORE_FILE); + const lock = join(options.directory, LEASE_STORE_LOCK); + let queue: Promise = Promise.resolve(); + return { + directory: options.directory, + update(change: (leases: LeaseRecord[]) => T): Promise { + const run = queue.then(() => { + mkdirSync(options.directory, { recursive: true, mode: 0o700 }); + return withRecoverableLock( + lock, + { purpose: options.purpose, containerRunning: options.containerRunning }, + () => { + const leases = readStore(file); + const before = canonicalJson(leases); + const result = change(leases); + if (leases.length > MAX_LEASES) throw new Error("Lease store is full"); + if (canonicalJson(leases) !== before) + atomicWrite( + file, + `${JSON.stringify(storeSchema.parse({ version: 1, leases }), null, 2)}\n`, + ); + return result; + }, + ); + }); + queue = run.catch(() => undefined); + return run; + }, + list: () => readStore(file), + get: (leaseId) => readStore(file).find((lease) => lease.leaseId === leaseId), + }; +} + +export function leaseStoreDirectory(infraDir: string): string { + return join(infraDir, "state", "ops-agent"); +} + +/** Every target some lease in the installation's store holds, read without the lock. */ +export function heldTargets(infraDir: string): string[] { + return [ + ...new Set( + readStore(join(leaseStoreDirectory(infraDir), LEASE_STORE_FILE)).flatMap( + (lease) => lease.targets, + ), + ), + ].sort(); +} + +/** Creates a held lease for `owner`, or refuses with the first holder named. */ +export function admitLease( + leases: LeaseRecord[], + input: { owner: ExecutionRef; ownerKind: string; targets: readonly string[]; now: number }, +): LeaseRecord { + const targets = leaseTargetsSchema.parse([...new Set(input.targets)].sort()); + if (leases.some((lease) => lease.owner.operationKey === input.owner.operationKey)) + throw new Error("Execution already acquired a root lease; reconcile its original identity"); + for (const target of targets) { + const holder = leases.find((lease) => lease.targets.includes(target)); + if (holder) throw new TargetBusyError(target, holder); + } + const record: LeaseRecord = { + leaseId: randomUUID(), + owner: input.owner, + ownerKind: input.ownerKind, + targets, + state: "held", + children: [], + createdAt: input.now, + }; + leases.push(record); + return record; +} + +/** Whether the owner, or the child `key`, and every ancestor may still begin effects. */ +export function leaseNodeForward(lease: LeaseRecord, key: string | null, now: number): boolean { + if (lease.state !== "held" || now >= lease.owner.deadlineAt) return false; + while (key !== null) { + const child = lease.children.find((entry) => entry.key === key); + if (!child || child.state !== "held" || now >= child.execution.deadlineAt) return false; + key = child.parentKey; + } + return true; +} + +/** Children issued directly by the owner (`null`) or by the child `key`. */ +export function leaseChildrenOf(lease: LeaseRecord, key: string | null): LeaseChild[] { + return lease.children.filter((child) => child.parentKey === key); +} diff --git a/packages/core/src/build-artifacts/motis-portable.test.ts b/packages/core/src/build-artifacts/motis-portable.test.ts index 8dac33ecd..e199fbb95 100644 --- a/packages/core/src/build-artifacts/motis-portable.test.ts +++ b/packages/core/src/build-artifacts/motis-portable.test.ts @@ -250,7 +250,6 @@ it.each([false, true])( serviceIds: ids, }); lease = await control.acquireArtifactTargetLease({ - dataRoot, serviceIds: ids, owner: authority, }); diff --git a/packages/core/src/build-artifacts/peer-admission.ts b/packages/core/src/build-artifacts/peer-admission.ts index a51d9b379..ef0670f4b 100644 --- a/packages/core/src/build-artifacts/peer-admission.ts +++ b/packages/core/src/build-artifacts/peer-admission.ts @@ -7,14 +7,19 @@ import { type ControlTransportPrincipal, controlDigest, controlHeaders, + inspectRemoteDelegation, installationState, - readParentGrant, + parseControlAssertion, retainedControlParent, + verifiedRemoteParentExecution, verifyControlPeer, - verifyParentExecution, } from "./control-authority"; import { adoptArtifactTargetLease, inspectArtifactDelegation } from "./target-lease"; -/** Both receivers keep transport auth and resource policy. This verifies only the exact signed internal attempt. */ +/** + * The agent's receiver for data-manager's delegated operations. The child was + * recorded in the agent's own lease store when data-manager asked for it; this + * checks the request against that row. Transport auth and resource policy stay. + */ export function inspectControlAdmission( installation: ControlInstallation, input: { @@ -56,26 +61,23 @@ export function inspectControlAdmission( throw Error("Receiving typed operation mismatch"); const { delegation } = controlHeaders(input.rawHeaders); if (!delegation) throw Error("Missing control assertion"); - const inspected = inspectArtifactDelegation({ - dataRoot: state.dataRoot, - assertion: delegation, - installation, - }); + const inspected = inspectArtifactDelegation({ assertion: delegation, installation }); if ( inspected.kind !== input.kind || inspected.execution.operationKey !== input.operationKey || inspected.execution.operationFingerprint !== input.operationFingerprint || inspected.execution.deadlineAt !== input.requestedDeadlineAt || - inspected.execution.executor !== installation.role + inspected.execution.executor !== installation.role || + inspected.owner.executor !== "data-manager" ) - throw Error("Signed immutable admission mismatch"); + throw Error("Recorded admission mismatch"); if ( operation.kind === "transitousLock.approve" && (inspected.phase !== "paired-pin-commit" || inspected.owner.executor !== "data-manager") ) throw Error("Original paired activation leaf required"); if (targets.some((id) => !state.registeredTargets.has(id) || !inspected.serviceIds.includes(id))) - throw Error("Grant does not contain operation-derived target closure"); + throw Error("Delegation does not contain operation-derived target closure"); const peer = verifyControlPeer(installation, { rawHeaders: input.rawHeaders, transportPrincipal: input.transportPrincipal, @@ -84,15 +86,14 @@ export function inspectControlAdmission( }); const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; verifiedParents.add(parent); - if (inspected.origin) verifiedParentOrigins.set(parent, inspected.origin); + verifiedParentOrigins.set(parent, parseControlAssertion(delegation).origin); return Object.freeze({ ...inspected, parent, async adopt(execution: ExecutionRef) { if (controlDigest(execution) !== controlDigest(inspected.execution)) - throw Error("Normalized execution identity differs from grant"); + throw Error("Normalized execution identity differs from delegation"); return adoptArtifactTargetLease({ - dataRoot: state.dataRoot, assertion: delegation, peer, execution, @@ -101,27 +102,9 @@ export function inspectControlAdmission( }, }); } -export function verifiedParentForArtifactDelegation( - installation: ControlInstallation, - assertion: string, - execution: ExecutionRef, -): VerifiedParentExecution { - const state = installationState(installation); - const inspected = inspectArtifactDelegation({ - dataRoot: state.dataRoot, - assertion, - installation, - }); - if (controlDigest(inspected.execution) !== controlDigest(execution)) - throw Error("Wrong delegated execution"); - const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; - verifiedParents.add(parent); - if (inspected.origin) verifiedParentOrigins.set(parent, inspected.origin); - return parent; -} -/** Fixed DM receiver: the only proposed ID comes from the authenticated grant. - * Reading the grant is observation; normalized equality is checked at adoption. */ +/** Data-manager's receiver for executions the agent or the host CLI issued. + * The caller's token says who sent them; the delegation or parent names them. */ export function inspectDataExecutionAdmission( installation: ControlInstallation, input: { @@ -133,78 +116,51 @@ export function inspectDataExecutionAdmission( requestedDeadlineAt?: number; }, ) { - const local = installationState(installation), - headers = controlHeaders(input.rawHeaders); if (installation.role !== "data-manager") throw Error("Data-manager receiver required"); + const headers = controlHeaders(input.rawHeaders); if (!headers.parent && !headers.delegation) return null; if (headers.parent) { - const grant = readParentGrant(local, headers.parent), - assertion = headers.parent; - if ( - grant.recipient !== "data-manager" || - grant.child.executor !== "data-manager" || - grant.kind !== input.input.kind || - grant.child.operationKey !== input.operationKey || - grant.child.operationFingerprint !== input.operationFingerprint || - grant.child.deadlineAt !== input.requestedDeadlineAt - ) - throw Error("Signed data parent identity mismatch"); - const peer = verifyControlPeer(installation, { - rawHeaders: input.rawHeaders, + const { parent, execution } = verifiedRemoteParentExecution(installation, headers.parent, { transportPrincipal: input.transportPrincipal, - execution: grant.child, - kind: grant.kind, - }); - const parent = verifyParentExecution(assertion, peer, { - execution: grant.child, - kind: grant.kind, - requestedDeadlineAt: grant.child.deadlineAt, + operationKey: input.operationKey, + operationFingerprint: input.operationFingerprint, + kind: input.input.kind, + requestedDeadlineAt: input.requestedDeadlineAt, }); return Object.freeze({ - execution: grant.child, + execution, parent, serviceIds: [] as string[], - async adopt(execution: ExecutionRef) { - if (controlDigest(execution) !== controlDigest(grant.child)) - throw Error("Normalized data child differs from grant"); + async adopt(adopted: ExecutionRef) { + if (controlDigest(adopted) !== controlDigest(execution)) + throw Error("Normalized data child differs from parent"); return undefined; }, }); } - const assertion = headers.delegation!; - const grant = inspectArtifactDelegation({ dataRoot: local.dataRoot, assertion, installation }); + const delegation = inspectRemoteDelegation( + installation, + input.rawHeaders, + input.transportPrincipal, + ); if ( input.input.kind !== "generation-hardlinks" || - grant.kind !== input.input.kind || - grant.execution.executor !== "data-manager" || - grant.execution.operationKey !== input.operationKey || - grant.execution.operationFingerprint !== input.operationFingerprint || - grant.execution.deadlineAt !== input.requestedDeadlineAt + delegation.kind !== input.input.kind || + delegation.execution.operationKey !== input.operationKey || + delegation.execution.operationFingerprint !== input.operationFingerprint || + delegation.execution.deadlineAt !== input.requestedDeadlineAt ) - throw Error("Unsupported or mismatched borrowed data leaf"); - const peer = verifyControlPeer(installation, { - rawHeaders: input.rawHeaders, - transportPrincipal: input.transportPrincipal, - execution: grant.execution, - kind: grant.kind, - }); - const parent = verifiedParentForArtifactDelegation(installation, assertion, grant.execution); + throw Error("Unsupported or mismatched delegated data leaf"); + const local = installationState(installation); + local.registeredTargets = new Set([...local.registeredTargets, ...delegation.serviceIds]); return Object.freeze({ - execution: grant.execution, - parent, - serviceIds: [] as string[], + execution: delegation.execution, + parent: delegation.verifiedParent, + serviceIds: delegation.serviceIds, async adopt(execution: ExecutionRef) { - if (controlDigest(execution) !== controlDigest(grant.execution)) - throw Error("Normalized data child differs from grant"); - const lease = await adoptArtifactTargetLease({ - dataRoot: local.dataRoot, - assertion, - peer, - execution, - serviceIds: grant.serviceIds, - }); - local.registeredTargets = new Set([...local.registeredTargets, ...lease.serviceIds]); - return lease; + if (controlDigest(execution) !== controlDigest(delegation.execution)) + throw Error("Normalized data child differs from delegation"); + return undefined; }, }); } diff --git a/packages/core/src/build-artifacts/peer-process.test.ts b/packages/core/src/build-artifacts/peer-process.test.ts index 89617c098..2dd0dfdcb 100644 --- a/packages/core/src/build-artifacts/peer-process.test.ts +++ b/packages/core/src/build-artifacts/peer-process.test.ts @@ -24,6 +24,7 @@ import { retainedControlParent, verifyControlEffectSettlement, } from "./control-authority"; +import { leaseStoreDirectory, openLeaseStore } from "./lease-store"; import { acquireArtifactTargetLease, closeArtifactTargetLeaseHandle, @@ -160,7 +161,6 @@ describe("separate-process physical-mount delegation", () => { plan: [plan], }); const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "host-alias/data"), serviceIds: ["motis"], owner: authority, }); @@ -241,7 +241,6 @@ it("keeps owner-death recovery fenced until a separately admitted child settles" }; const authority = retainAdmittedControlExecution(installation, admitted); const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -273,7 +272,6 @@ it("keeps owner-death recovery fenced until a separately admitted child settles" const recovery = verifyControlRecoveryAuthority(recovered); expect( await reconcileArtifactTargetLease({ - dataRoot: join(root, "host-alias/data"), leaseId: lease.leaseId, authority: recovery, }), @@ -281,7 +279,6 @@ it("keeps owner-death recovery fenced until a separately admitted child settles" expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); expect( await reconcileArtifactTargetLease({ - dataRoot: join(root, "data"), leaseId: lease.leaseId, authority: recovery, }), @@ -321,7 +318,6 @@ it.each([false, true])( plan: [plan], }); const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -459,7 +455,6 @@ it("keeps an attempted child fenced after SIGKILL of its actual owner process", const recovery = c.verifyControlRecoveryAuthority(authority); expect( await l.reconcileArtifactTargetLease({ - dataRoot: join(root, "data"), leaseId: owned.leaseId, authority: recovery, }), @@ -476,7 +471,6 @@ it("keeps an attempted child fenced after SIGKILL of its actual owner process", expect(await child({ ...input, cancel: true })).toMatchObject({ ok: true }); expect( await l.reconcileArtifactTargetLease({ - dataRoot: join(root, "data"), leaseId: owned.leaseId, authority: recovery, }), @@ -484,7 +478,6 @@ it("keeps an attempted child fenced after SIGKILL of its actual owner process", expect(await child({ ...input, settle: true })).toMatchObject({ ok: true }); expect( await l.reconcileArtifactTargetLease({ - dataRoot: join(root, "data"), leaseId: owned.leaseId, authority: recovery, }), @@ -631,7 +624,6 @@ it("keeps paired-pin approval a one-use leaf under its original activation root" plan: [plan], }); const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -707,7 +699,6 @@ it("rejects a logical configuration-fence delegation without the actual maintena plan: [plan], }); const lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -722,94 +713,12 @@ it("rejects a logical configuration-fence delegation without the actual maintena ).rejects.toThrow(); await expect(releaseArtifactTargetLease(lease)).rejects.toThrow(); expect( - readFileSync( - join(root, `data/.admin-target-control/leases/${lease.leaseId}/owner.json`), - "utf8", - ), - ).toContain(execution.operationKey); + openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }) + .list() + .map((value) => value.owner.operationKey), + ).toEqual([execution.operationKey]); } finally { closeArtifactTargetLeaseHandle(lease); c.closeControlInstallation(installation); } }); - -it("serializes a paused observer and a newer writer across actual processes", async () => { - const c = await import("./control-authority"); - const l = await import("./target-lease"); - const fs = await import("node:fs"); - const operation = { kind: "transit-sync" }; - const execution = ref("data-manager", operation.kind, operation); - const body = { kind: "motis.primary.stop" }; - const receiver = ref("ops-agent", body.kind, body, execution, "stop-primary"); - const installation = c.openControlInstallation({ - infraDir: root, - role: "data-manager", - serviceIds: ["motis"], - }); - const plan: ControlPlannedChild = { - phase: "stop-primary", - ordinal: 0, - kind: body.kind, - execution: receiver, - serviceIds: ["motis"], - recipient: "ops-agent", - }; - const authority = c.retainAdmittedControlExecution(installation, { - requestedDeadlineAt: execution.deadlineAt, - execution, - operation, - kind: operation.kind, - admittedAt: execution.deadlineAt - 60000, - serviceIds: ["motis"], - plan: [plan], - }); - const lease = await l.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["motis"], - owner: authority, - }); - const publicKey = JSON.parse(readFileSync(join(root, "secrets/control-verifiers.json"), "utf8")) - .publicKeys["ops-agent"]; - const assertion = await l.delegateArtifactTargetLease(lease, { - ...plan, - childPublicKey: publicKey, - }); - const ready = join(root, "observer-ready"), - resume = join(root, "observer-resume"); - const observing = child({ - mode: "observe-interleaving", - infraDir: root, - assertion, - ready, - resume, - }); - let busy = false; - try { - await expect.poll(() => fs.existsSync(ready)).toBe(true); - try { - await l.cancelArtifactTargetLease(lease); - } catch (error) { - busy = /writer.*busy/.test(String(error)); - } - } finally { - fs.writeFileSync(resume, "resume"); - } - expect(await observing).toMatchObject({ ok: true }); - expect(busy, "observer must hold the same CAS guard as the writer").toBe(true); - const ownerPath = join(root, `data/.admin-target-control/leases/${lease.leaseId}/owner.json`); - const old = readFileSync(ownerPath); - await l.cancelArtifactTargetLease(lease); - const mark = JSON.parse( - readFileSync( - join(root, `control-private/data-manager/observations/${lease.leaseId}.json`), - "utf8", - ), - ); - expect(mark.owner.sequence).toBe(JSON.parse(readFileSync(ownerPath, "utf8")).payload.sequence); - fs.writeFileSync(ownerPath, old); - expect(() => - l.inspectArtifactDelegation({ dataRoot: join(root, "data"), assertion, installation }), - ).toThrow(/rollback/); - l.closeArtifactTargetLeaseHandle(lease); - c.closeControlInstallation(installation); -}); diff --git a/packages/core/src/build-artifacts/publication-native.test.ts b/packages/core/src/build-artifacts/publication-native.test.ts index 5f310c71c..c55e00c16 100644 --- a/packages/core/src/build-artifacts/publication-native.test.ts +++ b/packages/core/src/build-artifacts/publication-native.test.ts @@ -1,19 +1,13 @@ import { spawn } from "node:child_process"; import { once } from "node:events"; -import { - existsSync, - lstatSync, - mkdtempSync, - readFileSync, - rmdirSync, - rmSync, - statSync, -} from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; import { executionRefSchema } from "../ops/execution-ref"; +import { configurationWriterLockPath } from "../services/configuration-ownership"; +import { readLockHolder } from "../services/recoverable-lock"; import type { ValidatedBuildBundle } from "./archive"; import * as control from "./control-authority"; import { inspectArtifactPublication, publishValidatedBuild } from "./publication"; @@ -24,7 +18,7 @@ afterEach(async () => { for (const close of cleanup.reverse().splice(0)) await close(); }); it.each(["old-directory", "build-record"])( - "restores the original complete set after native death at %s and explicit host writer recovery", + "restores the original complete set after native death at %s, reclaiming the dead writer's lock", async (point) => { const root = mkdtempSync(join(tmpdir(), "omx-publication-native-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); @@ -60,22 +54,15 @@ it.each(["old-directory", "build-record"])( }, { timeout: 30000, interval: 50 }, ); - const writerPaths = [join(root, "state/ops-agent/configuration/.writer")]; - const writersBefore = writerPaths.map((path) => ({ path, identity: lstatSync(path) })); + const writerLock = configurationWriterLockPath(root); + expect(readLockHolder(writerLock)?.pid).toBe(child.pid); child.kill("SIGKILL"); const [code, signal] = await exited; expect(code).toBeNull(); expect(signal).toBe("SIGKILL"); - // This isolated fixture owns its ONLY writer and has just killed it. Model - // the documented explicit host operation; production never steals this lock - // using elapsed time, a PID, or a missing process as execution authority. - // Explicit recovery removes only the exact empty fixture-owned writer lock. - for (const { path, identity: before } of writersBefore) { - const after = lstatSync(path); - expect([after.dev, after.ino]).toEqual([before.dev, before.ino]); - expect(after.isDirectory() && !after.isSymbolicLink()).toBe(true); - rmdirSync(path); // Empty, exact owned lock only; no recursive deletion. - } + // The killed writer's lock stays on disk; the next writer reclaims it + // because its holder process is provably gone. + expect(readLockHolder(writerLock)?.pid).toBe(child.pid); const state = JSON.parse(readFileSync(join(root, "fault.json"), "utf8")) as { execution: unknown; @@ -127,7 +114,7 @@ it.each(["old-directory", "build-record"])( outcome: "not_committed", reason: "failed", }); - expect(() => leases.verifyRetainedArtifactTargetRootRelease(authority)).toThrow(); + await expect(leases.verifyRetainedArtifactTargetRootRelease(authority)).rejects.toThrow(); control.retainControlEffectReceipt(authority, { outcome: "contained", publication: "not_started", @@ -141,7 +128,9 @@ it.each(["old-directory", "build-record"])( control.verifyControlEffectSettlement(installation, authority.execution), ); await leases.releaseArtifactTargetLease(lease); - expect(leases.verifyRetainedArtifactTargetRootRelease(authority).releasedAt).toBeGreaterThan(0); + expect( + (await leases.verifyRetainedArtifactTargetRootRelease(authority)).releasedAt, + ).toBeGreaterThan(0); }, 60000, ); diff --git a/packages/core/src/build-artifacts/publication.test.ts b/packages/core/src/build-artifacts/publication.test.ts index 8fbde1416..a75b3db91 100644 --- a/packages/core/src/build-artifacts/publication.test.ts +++ b/packages/core/src/build-artifacts/publication.test.ts @@ -111,7 +111,7 @@ async function fixture() { requestedDeadlineAt: deadlineAt, serviceIds: ["osrm"], }), - lease = await acquireArtifactTargetLease({ dataRoot, serviceIds: ["osrm"], owner: authority }); + lease = await acquireArtifactTargetLease({ serviceIds: ["osrm"], owner: authority }); cleanup.push(() => closeArtifactTargetLeaseHandle(lease)); return { root, dataRoot, stagingRoot, stageId, bundle, authority, lease, oldRecord }; } diff --git a/packages/core/src/build-artifacts/publication.ts b/packages/core/src/build-artifacts/publication.ts index 4c264f2d1..714928def 100644 --- a/packages/core/src/build-artifacts/publication.ts +++ b/packages/core/src/build-artifacts/publication.ts @@ -52,7 +52,7 @@ import { assertArtifactTargetLeaseExecution, assertArtifactTargetLeaseReceiptOwner, type TargetLeaseAuthority, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, } from "./target-lease"; export type ArtifactFilePublicationResult = @@ -863,7 +863,7 @@ export function cleanupRestoredArtifactPublication(authority: VerifiedExecutionA journal.executionDigest !== controlDigest(state.execution) ) throw Error("Original restored publication required"); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); const root = new AnchoredDirectory(state.installation.dataRoot); let source: ReturnType | undefined, backups: AnchoredDirectory | undefined; try { @@ -905,7 +905,7 @@ export function inspectSettledArtifactImport( const journal = readJournal({ ...owner, origin: owner.origin }); if (!journal || !["committed", "restored"].includes(journal.phase)) throw Error("Original settled import required"); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); const effect = childSettlementState( verifyControlEffectSettlement(owner.installation.installation, owner.execution), ).evidence; @@ -956,7 +956,7 @@ export function inspectUnstartedArtifactImport(authority: VerifiedExecutionAutho effect.publication !== "not_started" ) throw Error("Unstarted import containment required"); - verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); const admitted = readPrivateRecord( owner.installation, "executions", @@ -1088,5 +1088,5 @@ export function assertKnownArtifactStageReceipt(authority: VerifiedExecutionAuth ).evidence; if (effect.outcome === "unknown" || effect.publication === "unknown") throw Error("Prepared purpose remains unknown"); - if (owner.kind === "artifactTransfer.import") verifyRetainedArtifactTargetRootRelease(authority); + if (owner.kind === "artifactTransfer.import") verifyLocalArtifactTargetRootRelease(authority); } diff --git a/packages/core/src/build-artifacts/service-lifecycle-recovery.test.ts b/packages/core/src/build-artifacts/service-lifecycle-recovery.test.ts deleted file mode 100644 index 011590235..000000000 --- a/packages/core/src/build-artifacts/service-lifecycle-recovery.test.ts +++ /dev/null @@ -1,266 +0,0 @@ -import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { currentConfigurationGeneration } from "../services/configuration-generation"; -import { - lifecycleFailureHasNoSurvivor, - type ServiceLifecycleReceipt, -} from "../services/service-lifecycle-receipt"; -import * as c from "./control-authority"; -import { AnchoredDirectory } from "./filesystem"; -import { inspectDataExecutionAdmission } from "./peer-admission"; -import { reconcileFailedServiceLifecycle } from "./service-lifecycle-recovery"; -import * as leases from "./target-lease"; -import { failedServiceLifecycle } from "./test-fixtures/service-lifecycle-recovery"; - -vi.setConfig({ testTimeout: 30000 }); -let root: string; -const installations: c.ControlInstallation[] = []; -beforeEach(async () => { - root = mkdtempSync(join(tmpdir(), "omx-lifecycle-recovery-")); - mkdirSync(join(root, "data"), { mode: 0o700 }); - await c.provisionControlInstallation(root); -}); -afterEach(() => { - vi.restoreAllMocks(); - for (const installation of installations.splice(0)) c.closeControlInstallation(installation); - rmSync(root, { recursive: true, force: true }); -}); -async function failedLifecycle(adopt: boolean | "after-failure" = false, issueChild = true) { - const fixture = await failedServiceLifecycle(root, adopt, issueChild); - installations.push(...fixture.installations); - return fixture; -} - -it("releases only the expired failed update's unadopted hardlink grant without replay or receipt rewriting", async () => { - const f = await failedLifecycle(); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - await reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(false); - expect(readFileSync(f.path)).toEqual(f.receiptBytes); - expect(currentConfigurationGeneration(root)).toBe(f.revisionId); - await reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }); - expect(f.recoveryRequests).toEqual([ - { url: "http://127.0.0.1:4301/executions/generation-hardlinks/recovery", method: "POST" }, - ]); - const dm = c.openControlInstallation({ - infraDir: root, - role: "data-manager", - serviceIds: ["alpha"], - }); - installations.push(dm); - await expect( - Promise.resolve().then(async () => { - const receiver = inspectDataExecutionAdmission(dm, { - rawHeaders: ["x-openmapx-control-delegation", f.assertion], - transportPrincipal: "host-cli", - input: { - kind: "generation-hardlinks", - revisionId: f.revisionId, - coordinatorMode: "ensure-running", - planDigest: c.controlDigest(f.plan), - }, - operationKey: f.child.execution.operationKey, - operationFingerprint: f.reservation.operationFingerprint, - requestedDeadlineAt: f.child.execution.deadlineAt, - }); - return receiver!.adopt(f.child.execution); - }), - ).rejects.toThrow(); -}); -it("preserves an already adopted child and keeps its original update fenced", async () => { - const f = await failedLifecycle(true); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - await expect( - reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }), - ).rejects.toThrow(/adopted|unresolved/); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(true); - expect(readFileSync(f.path)).toEqual(f.receiptBytes); -}); -it("does not reinterpret a lost adopted child file as proof that no receiver started", async () => { - const f = await failedLifecycle("after-failure"); - const directory = join(root, "data/.admin-target-control/leases", f.leaseId, "children"); - const files = readdirSync(directory); - expect(files).toHaveLength(1); - rmSync(join(directory, files[0]!)); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - await expect( - reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }), - ).rejects.toThrow(); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(true); -}); -it("refuses recovery while the original update can still run", async () => { - const f = await failedLifecycle(); - await expect( - reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }), - ).rejects.toThrow(/expired/); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(true); -}); -it("a revoked unadopted grant cannot become an adoption before root release", async () => { - const f = await failedLifecycle(); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - const lease = leases.restoreArtifactTargetLeaseForSettlement(f.authority); - try { - await leases.prepareFailedServiceLifecycleRecovery(f.authority, lease); - await leases.revokeUnadoptedServiceHardlinks(f.authority, lease, await f.nonAdmission()); - const dm = c.openControlInstallation({ - infraDir: root, - role: "data-manager", - serviceIds: ["alpha"], - }); - installations.push(dm); - const receiver = inspectDataExecutionAdmission(dm, { - rawHeaders: ["x-openmapx-control-delegation", f.assertion], - transportPrincipal: "host-cli", - input: { - kind: "generation-hardlinks", - revisionId: f.revisionId, - coordinatorMode: "ensure-running", - planDigest: c.controlDigest(f.plan), - }, - operationKey: f.child.execution.operationKey, - operationFingerprint: f.reservation.operationFingerprint, - requestedDeadlineAt: f.child.execution.deadlineAt, - }); - await expect(receiver!.adopt(f.child.execution)).rejects.toThrow( - "Original unadopted grant was revoked", - ); - } finally { - leases.closeArtifactTargetLeaseHandle(lease); - } -}); -it("finishes the original release cursor after interruption without reopening forward authority", async () => { - const f = await failedLifecycle(); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - const remove = AnchoredDirectory.prototype.removeDirectory; - let interrupted = false; - const fault = vi - .spyOn(AnchoredDirectory.prototype, "removeDirectory") - .mockImplementation(function (this: AnchoredDirectory, name, ...args) { - if (name === "alpha" && !interrupted) { - interrupted = true; - throw Error("Release interrupted"); - } - return remove.call(this, name, ...args); - }); - await expect( - reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }), - ).rejects.toThrow("Release interrupted"); - fault.mockRestore(); - await reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(false); - expect(readFileSync(f.path)).toEqual(f.receiptBytes); -}); - -it("requires the recipient's exact signed non-admission proof before revocation", async () => { - const f = await failedLifecycle(); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - const lease = leases.restoreArtifactTargetLeaseForSettlement(f.authority); - try { - await leases.prepareFailedServiceLifecycleRecovery(f.authority, lease); - await expect(leases.revokeUnadoptedServiceHardlinks(f.authority, lease)).rejects.toThrow( - "Data Manager non-admission proof required", - ); - const proof = await f.nonAdmission(); - await expect( - leases.revokeUnadoptedServiceHardlinks(f.authority, lease, { - ...proof, - signature: "a".repeat(86), - }), - ).rejects.toThrow("Invalid control signature"); - const dm = f.installations.find((i) => i.role === "data-manager")!; - const changed = c.signControl( - c.installationState(dm).privateKey, - "openmapx-generation-hardlinks-non-admission-v1", - { ...proof.payload, grantDigest: "a".repeat(64) }, - ); - await expect( - leases.revokeUnadoptedServiceHardlinks(f.authority, lease, changed), - ).rejects.toThrow("Data Manager non-admission proof does not match original grant"); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(true); - expect(readFileSync(f.path)).toEqual(f.receiptBytes); - } finally { - leases.closeArtifactTargetLeaseHandle(lease); - } -}); -it("accepts a failure with no issued child while its root is still held", async () => { - const f = await failedLifecycle(false, false); - const source = leases.verifyFailedServiceLifecycleContainment(f.authority); - expect(source.failedReceiptDigest).toBe( - c.controlDigest(JSON.parse(readFileSync(f.path, "utf8")).current.payload.receipt), - ); - c.retainControlEffectReceipt(f.authority, { - outcome: "contained", - publication: "committed", - receipt: { - kind: "service-lifecycle-contained-failure", - failedReceiptDigest: source.failedReceiptDigest, - }, - }); -}); -it("still requires the unadopted revocation once a child was issued", async () => { - const f = await failedLifecycle(); - expect(() => leases.verifyFailedServiceLifecycleContainment(f.authority)).toThrow( - "Original unadopted hardlink revocation required", - ); -}); -it("recovers a no-child failure whose process stopped before releasing", async () => { - const f = await failedLifecycle(false, false); - await expect( - reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }), - ).rejects.toThrow(/expired/); - vi.spyOn(Date, "now").mockReturnValue(f.execution.deadlineAt + 1); - await reconcileFailedServiceLifecycle({ - infraDir: root, - authority: f.authority, - transport: f.transport, - }); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(false); - expect(readFileSync(f.path)).toEqual(f.receiptBytes); - expect(f.recoveryRequests).toEqual([]); -}); -it.each([ - ["failed", null, true], - ["unknown", null, true], - ["failed", { assertion: "issued" }, false], - ["unknown", new Error("unreadable attempt"), false], - ["recreated", null, false], - ["complete", null, false], -] as const)("a %s receipt with attempt %j has no survivor: %s", (phase, attempt, expected) => { - expect( - lifecycleFailureHasNoSurvivor({ phase } as unknown as ServiceLifecycleReceipt, attempt), - ).toBe(expected); -}); diff --git a/packages/core/src/build-artifacts/service-lifecycle-recovery.ts b/packages/core/src/build-artifacts/service-lifecycle-recovery.ts deleted file mode 100644 index db79cfb17..000000000 --- a/packages/core/src/build-artifacts/service-lifecycle-recovery.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { join, resolve } from "node:path"; -import { z } from "zod"; -import { parseControlRootOperationKey } from "../ops/control-root-key"; -import { executionRefSchema } from "../ops/execution-ref"; -import { - currentConfigurationGeneration, - withConfigurationGenerationWriter, -} from "../services/configuration-generation"; -import { recoverUnadmittedGenerationHardlinks } from "../services/data-manager/generated/data-manager.generated"; -import { - type DataManagerTransport, - unwrapDataManagerResponse, -} from "../services/data-manager/transport"; -import { readFailedServiceHardlinksReceipt } from "../services/service-lifecycle-receipt"; -import { lookupControlChildPhaseAttempt } from "./child-phase"; -import { - type ControlInstallation, - controlDigest, - executionAuthorityState, - installationState, - readPrivateRecord, - restoreAdmittedControlExecution, - retainControlEffectReceipt, - type VerifiedExecutionAuthority, - verifyControlRecoveryAuthority, -} from "./control-authority"; -import { readGenerationHardlinksOwnerPlan } from "./generation-hardlinks-owner"; -import { - closeArtifactTargetLeaseHandle, - prepareFailedServiceLifecycleRecovery, - reconcileArtifactTargetLease, - restoreArtifactTargetLeaseForSettlement, - revokeUnadoptedServiceHardlinks, - verifyFailedServiceLifecycleContainment, - verifyRetainedArtifactTargetRootRelease, -} from "./target-lease"; - -/** Recovery only: never issue, replay, render, pull or recreate. */ -export async function reconcileFailedServiceLifecycle(options: { - infraDir: string; - authority: VerifiedExecutionAuthority; - transport: DataManagerTransport; -}): Promise { - const { authority, infraDir } = options, - owner = executionAuthorityState(authority); - if (resolve(owner.installation.dataRoot) !== resolve(join(infraDir, "data"))) - throw Error("Original lifecycle installation required"); - if (Date.now() < owner.execution.deadlineAt) throw Error("Original lifecycle must be expired"); - await withConfigurationGenerationWriter(infraDir, async () => { - const failure = readFailedServiceHardlinksReceipt(authority); - let prior: ReturnType | undefined; - try { - const source = verifyFailedServiceLifecycleContainment(authority); - prior = source; - } catch { - /* An initial attempt has no revocation/effect receipt yet. */ - } - if (prior) { - retainControlEffectReceipt(authority, { - outcome: "contained", - publication: "committed", - receipt: { - kind: "service-lifecycle-contained-failure", - failedReceiptDigest: prior.failedReceiptDigest, - }, - }); - if ( - (await reconcileArtifactTargetLease({ - dataRoot: owner.installation.dataRoot, - leaseId: prior.leaseId, - authority: verifyControlRecoveryAuthority(authority), - })) !== "released" - ) - throw Error("Original lifecycle remains fenced"); - verifyRetainedArtifactTargetRootRelease(authority); - return; - } - if (currentConfigurationGeneration(infraDir) !== failure.finalRevisionId) - throw Error("Committed lifecycle generation changed; supervised recovery required"); - const lease = restoreArtifactTargetLeaseForSettlement(authority); - try { - await prepareFailedServiceLifecycleRecovery(authority, lease); - const plan = readGenerationHardlinksOwnerPlan(authority), - attempt = lookupControlChildPhaseAttempt(authority, { phase: "hardlinks", ordinal: 0 }); - if (!attempt || attempt.channel !== "delegation") - throw Error("Original hardlink outbox required"); - const nonAdmission = unwrapDataManagerResponse( - await recoverUnadmittedGenerationHardlinks( - { - operationKey: attempt.execution.operationKey, - requestedDeadlineAt: attempt.execution.deadlineAt, - input: { - kind: "generation-hardlinks", - revisionId: plan.revisionId, - coordinatorMode: plan.coordinatorMode, - planDigest: controlDigest(plan.plan), - }, - plan: plan.plan, - }, - { - transport: options.transport, - headers: { "x-openmapx-control-delegation": attempt.assertion }, - }, - ), - ); - await revokeUnadoptedServiceHardlinks(authority, lease, nonAdmission); - retainControlEffectReceipt(authority, { - outcome: "contained", - publication: "committed", - receipt: { - kind: "service-lifecycle-contained-failure", - failedReceiptDigest: controlDigest(failure), - }, - }); - if ( - (await reconcileArtifactTargetLease({ - dataRoot: owner.installation.dataRoot, - leaseId: lease.leaseId, - authority: verifyControlRecoveryAuthority(authority), - })) !== "released" - ) - throw Error("Original lifecycle remains fenced"); - verifyRetainedArtifactTargetRootRelease(authority); - } finally { - closeArtifactTargetLeaseHandle(lease); - } - }); -} - -/** The operator supplies only a lookup key; authority always comes from the - * original anchored, signed local execution, not from request identity fields. */ -export async function recoverServiceLifecycleByKey(options: { - infraDir: string; - installation: ControlInstallation; - operationKey: string; - transport: DataManagerTransport; -}) { - parseControlRootOperationKey(options.operationKey); - const local = installationState(options.installation); - if (options.installation.role !== "host-cli" || local.fixedChildId) - throw Error("Recovery requires the installation host"); - const original = z - .object({ payload: z.object({ execution: executionRefSchema }) }) - .parse(readPrivateRecord(local, "executions", `${options.operationKey}.json`)); - if (original.payload.execution.operationKey !== options.operationKey) - throw Error("Original lifecycle lookup identity changed"); - const authority = restoreAdmittedControlExecution( - options.installation, - original.payload.execution, - ); - await reconcileFailedServiceLifecycle({ - infraDir: options.infraDir, - authority, - transport: options.transport, - }); - return { - operationKey: options.operationKey, - revisionId: readFailedServiceHardlinksReceipt(authority).finalRevisionId!, - }; -} diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index 213d6a9d2..468771d42 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -1,13 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - renameSync, - rmdirSync, - rmSync, -} from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createControlRootOperationKey } from "@openmapx/core/ops"; @@ -23,6 +15,7 @@ import { retainControlEffectReceipt, verifyControlEffectSettlement, } from "./control-authority"; +import { leaseStoreDirectory, openLeaseStore } from "./lease-store"; let root: string; let installation: ControlInstallation; @@ -42,141 +35,39 @@ afterEach(() => { rmSync(root, { recursive: true, force: true }); }); -it.each([18, 256])( - "recovers release after target17 for %i maximum-length certified targets within the admission byte budget", - async (count) => { - const c = await import("./control-authority"), - a = await api(); - const { AnchoredDirectory } = await import("./filesystem"); - const targets = Array.from( - { length: count }, - (_, n) => `svc-${String(n).padStart(3, "0")}${"x".repeat(57)}`, - ); - closeControlInstallation(installation); - installation = openControlInstallation({ - infraDir: root, - role: "host-cli", - serviceIds: targets, - }); - const serviceId = targets[0]; - if (!serviceId) throw Error("Missing test target"); - const operation = { kind: "service.build" as const, serviceId }, - operationFingerprint = opsOperationFingerprint(operation), - deadlineAt = Date.now() + 120000; - const execution = { - executor: "host-cli" as const, - principal: "host-cli" as const, - executionId: randomUUID(), - operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), - operationFingerprint, - fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), - deadlineAt, - }; - const authority = retainAdmittedControlExecution(installation, { - execution, - kind: operation.kind, - operation, - admittedAt: deadlineAt - 120000, - requestedDeadlineAt: deadlineAt, - serviceIds: targets, - }); - const admissionPath = join( - root, - `control-private/host-cli/executions/${execution.operationKey}.json`, - ), - admitted = readFileSync(admissionPath); - const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: targets, - owner: authority, - }); - retainControlEffectReceipt(authority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - await a.settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(installation, execution), - ); - const original = AnchoredDirectory.prototype.write; - let interrupted = false, - maximumBytes = 0; - const fault = vi.spyOn(AnchoredDirectory.prototype, "write").mockImplementation(function ( - this: import("./filesystem").AnchoredDirectory, - ...args - ) { - original.apply(this, args); - if (args[0] !== "admission.json") return; - maximumBytes = Math.max(maximumBytes, Buffer.byteLength(args[1])); - const release = JSON.parse(args[1].toString()).payload.release; - if ( - !interrupted && - (release?.completedTargets ?? release?.locks?.length) === 17 && - release.pending === undefined - ) { - interrupted = true; - throw Error("contained stop after target17 publication"); - } - }); - await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow( - "contained stop after target17", - ); - expect(interrupted).toBe(true); - expect( - targets - .slice(0, 17) - .every((id) => !existsSync(join(root, `data/.admin-target-control/locks/${id}`))), - ).toBe(true); - expect(existsSync(join(root, `data/.admin-target-control/locks/${targets[17]}`))).toBe(true); - a.closeArtifactTargetLeaseHandle(lease); - closeControlInstallation(installation); - installation = openControlInstallation({ - infraDir: root, - role: "host-cli", - serviceIds: targets, - }); - const restored = c.restoreAdmittedControlExecution(installation, execution), - recovery = c.verifyControlRecoveryAuthority(restored); - const input = { dataRoot: join(root, "data"), leaseId: lease.leaseId, authority: recovery }; - await expect(a.reconcileArtifactTargetLease(input)).resolves.toBe("released"); - await expect(a.reconcileArtifactTargetLease(input)).resolves.toBe("released"); - fault.mockRestore(); - expect(maximumBytes).toBeLessThanOrEqual(4096); - const journalBytes = readFileSync( - join(root, `data/.admin-target-control/leases/${lease.leaseId}/admission.json`), - ); - expect(journalBytes.byteLength).toBeLessThanOrEqual(4096); - const finalJournal = JSON.parse(journalBytes.toString()).payload; - expect(finalJournal.release).toMatchObject({ - version: 2, - phase: "released", - completedTargets: count, - active: true, - }); - expect( - existsSync( - join( - root, - `data/.admin-target-control/admission-index-v1/active/${finalJournal.active.name}`, - ), - ), - ).toBe(false); - expect( - targets.every((id) => !existsSync(join(root, `data/.admin-target-control/locks/${id}`))), - ).toBe(true); - expect(readFileSync(admissionPath)).toEqual(admitted); - expect( - JSON.parse( - readFileSync( - join(root, `control-private/host-cli/effect-started/${execution.operationKey}.json`), - "utf8", - ), - ), - ).toBeNull(); - }, - 120000, -); +/** The leases the agent's store holds now. */ +function heldLeases() { + return openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }).list(); +} +it("reconciles a settled root after a crash between settlement and release", async () => { + const c = await import("./control-authority"), + a = await api(); + const authority = owner(); + const lease = await a.acquireArtifactTargetLease({ serviceIds: ["osrm"], owner: authority }); + retainControlEffectReceipt(authority, { + outcome: "failed_without_effect", + publication: "not_started", + receipt: { kind: "not-started" }, + }); + await a.settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(installation, authority.execution), + ); + a.closeArtifactTargetLeaseHandle(lease); + expect(heldLeases().map((value) => value.leaseId)).toEqual([lease.leaseId]); + closeControlInstallation(installation); + installation = openControlInstallation({ + infraDir: root, + role: "host-cli", + serviceIds: ["osrm", "motis"], + }); + const restored = c.restoreAdmittedControlExecution(installation, authority.execution), + recovery = c.verifyControlRecoveryAuthority(restored); + const input = { leaseId: lease.leaseId, authority: recovery }; + await expect(a.reconcileArtifactTargetLease(input)).resolves.toBe("released"); + await expect(a.reconcileArtifactTargetLease(input)).resolves.toBe("released"); + expect(heldLeases()).toEqual([]); +}); function owner() { const operation = { kind: "service.build" as const, serviceId: "osrm" }; const operationFingerprint = opsOperationFingerprint(operation); @@ -209,54 +100,27 @@ function owner() { async function api() { return import("./target-lease"); } -it("distinguishes an occupied original journal guard before running its source check", async () => { - const a = await api(), - authority = owner(), - lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["osrm"], - owner: authority, - }); - const directory = join(root, "data/.admin-target-control/leases", lease.leaseId), - guard = join(directory, ".journal-writer"), - before = readFileSync(join(directory, "owner.json")); - mkdirSync(guard, { mode: 0o700 }); - try { - expect(() => a.assertArtifactTargetLeaseForward(lease)).toThrow( - a.ArtifactTargetJournalBusyError, - ); - expect(readFileSync(join(directory, "owner.json"))).toEqual(before); - expect(existsSync(guard)).toBe(true); - } finally { - rmdirSync(guard); - } - a.assertArtifactTargetLeaseForward(lease); - a.closeArtifactTargetLeaseHandle(lease); -}); describe("certified target leases", () => { it("requires nominal durable ownership, then fences unknown own effects", async () => { const a = await api(); const authority = owner(); await expect( a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: { execution: authority.execution } as typeof authority, }), ).rejects.toThrow(/authority/); const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: authority, }); - await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/settlement|effect/); - expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(true); + await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/settle|effect/); + expect(heldLeases().map((value) => value.targets)).toEqual([["osrm"]]); }); it("releases a certified target after durable no-effect settlement", async () => { const a = await api(); const authority = owner(); const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: authority, }); @@ -270,61 +134,46 @@ describe("certified target leases", () => { verifyControlEffectSettlement(installation, authority.execution), ); await a.releaseArtifactTargetLease(lease); - expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(false); + expect(heldLeases()).toEqual([]); }); - it("rejects replacement lock inodes without removing the replacement", async () => { + it("refuses a second owner of a held target and names the holder", async () => { const a = await api(); - const authority = owner(); - const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["osrm"], - owner: authority, - }); - const path = join(root, "data/.admin-target-control/locks/osrm"); - renameSync(path, `${path}.old`); - mkdirSync(path, { mode: 0o700 }); - await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/inode|identity|changed/); - expect(existsSync(path)).toBe(true); + const first = owner(); + await a.acquireArtifactTargetLease({ serviceIds: ["osrm"], owner: first }); + const second = owner(); + vi.useFakeTimers({ toFake: ["setTimeout", "Date"] }); + try { + const refused = expect( + a.acquireArtifactTargetLease({ serviceIds: ["osrm"], owner: second }), + ).rejects.toThrow(first.execution.operationKey); + await vi.advanceTimersByTimeAsync(6000); + await refused; + } finally { + vi.useRealTimers(); + } + expect(heldLeases()).toHaveLength(1); }); }); it("never reacquires or expands a retained execution's root", async () => { const a = await api(); const authority = owner(); await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: authority, }); await expect( a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: authority, }), ).rejects.toThrow(/already acquired/); await expect( a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }), ).rejects.toThrow(/closure/); }); -it("detects rollback of a signed shared journal using private high-water evidence", async () => { - const a = await api(); - const authority = owner(); - const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["osrm"], - owner: authority, - }); - const path = join(root, `data/.admin-target-control/leases/${lease.leaseId}/owner.json`); - const previous = readFileSync(path); - await a.cancelArtifactTargetLease(lease); - const { writeFileSync } = await import("node:fs"); - writeFileSync(path, previous); - await expect(a.releaseArtifactTargetLease(lease)).rejects.toThrow(/rollback|replacement/); -}); it("rejects arbitrary caller text as effect evidence", async () => { const authority = owner(); expect(() => @@ -409,7 +258,6 @@ it("keeps a bare terminal maintenance summary fenced without the fixed owner sou plan: [plan], }); const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["motis"], owner: authority, }); @@ -462,21 +310,22 @@ it("keeps a bare terminal maintenance summary fenced without the fixed owner sou a.closeArtifactTargetLeaseHandle(lease); c.closeControlInstallation(agent); }); -it("holds certified target, configuration writer and release-store locks together", async () => { - const { acquireConfigurationOwnership } = await import("../services/configuration-ownership"); +it("holds a target lease, the configuration writer and the release-store lock together", async () => { + const { acquireConfigurationOwnership, configurationWriterLockPath } = await import( + "../services/configuration-ownership" + ); const { acquireReleaseStoreLock } = await import("../services/release-store-lock"); const a = await api(); const c = await import("./control-authority"); const authority = owner(); const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: ["osrm"], owner: authority, }); - expect(existsSync(join(root, "data/.admin-target-control/locks/osrm"))).toBe(true); + expect(heldLeases()).toHaveLength(1); const releaseWriter = await acquireConfigurationOwnership(root); const store = await acquireReleaseStoreLock(root, {}, { failIfBusy: true }); - expect(existsSync(join(root, "state/ops-agent/configuration/.writer"))).toBe(true); + expect(existsSync(configurationWriterLockPath(root))).toBe(true); store.release(); releaseWriter(); c.retainControlEffectReceipt(authority, { @@ -513,55 +362,6 @@ it("does not restart effects after retaining unused-execution evidence", async ( expect(() => c.beginControlEffect(authority)).toThrow(/cannot restart/); }); -it("admits from a bounded active index without opening retained terminal ledgers", async () => { - const a = await api(); - const { AnchoredDirectory } = await import("./filesystem"); - const { vi } = await import("vitest"); - // Real retained terminal histories, with valid signatures and released locks. - for (let i = 0; i < 12; i++) { - const authority = owner(); - const lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["osrm"], - owner: authority, - }); - retainControlEffectReceipt(authority, { - outcome: "failed_without_effect", - publication: "not_started", - receipt: { kind: "not-started" }, - }); - await a.settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(installation, authority.execution), - ); - await a.releaseArtifactTargetLease(lease); - } - const authority = owner(); - const read = AnchoredDirectory.prototype.read; - const historicalReads: string[] = []; - const spy = vi.spyOn(AnchoredDirectory.prototype, "read").mockImplementation(function ( - this: InstanceType, - name, - max, - mode, - ) { - if (name === "owner.json") historicalReads.push(this.path); - return read.call(this, name, max, mode); - }); - let lease: import("./target-lease").ArtifactTargetLease | undefined; - try { - lease = await a.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["osrm"], - owner: authority, - }); - } finally { - spy.mockRestore(); - } - expect(new Set(historicalReads).size, "new acquisition must read only its new ledger").toBe(1); - if (lease) a.closeArtifactTargetLeaseHandle(lease); -}, 180000); - it("does not accumulate temporary authority records after failed atomic updates", async () => { const { AnchoredDirectory } = await import("./filesystem"); const { vi } = await import("vitest"); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index 7f6b66a2f..b4e6721f0 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -1,33 +1,14 @@ -import { createPrivateKey, type KeyObject, randomUUID } from "node:crypto"; -import { fsyncSync, mkdirSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; +import { randomUUID } from "node:crypto"; import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; import { configurationApplyResultSchema, opsOperationSchema } from "../ops/contract"; import { type ExecutionRef, - executionFingerprint, executionRefSchema, + type VerifiedParentExecution, } from "../ops/execution-identity"; import { opsOperationFingerprint } from "../ops/fingerprint"; -import { - CONFIGURATION_PREPARE_NON_ADMISSION_DOMAIN, - configurationPrepareNonAdmissionPayloadSchema, - configurationPrepareNonAdmissionSchema, - type DataConfigurationFencePrepare, -} from "../services/data-manager/contracts/configuration-maintenance"; -import { - GENERATION_HARDLINKS_NON_ADMISSION_DOMAIN, - generationHardlinksNonAdmissionPayloadSchema, - generationHardlinksNonAdmissionSchema, -} from "../services/data-manager/contracts/executions"; -// biome-ignore-start lint/suspicious/noImportCycles: Original receipt recovery is called after initialization, never during module evaluation. -import { - readFailedServiceHardlinksReceipt, - readFailedServiceLifecycleReceipt, -} from "../services/service-lifecycle-receipt"; -// biome-ignore-end lint/suspicious/noImportCycles: Original receipt recovery is called after initialization, never during module evaluation. +import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { inspectControlChildPublication, @@ -54,30 +35,25 @@ import { // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { type ArtifactDelegationAssertion, - type AuthorityState, assertControlExecutionForward, - assertionSchema, type ChildSettlement, type ControlChildPhase, - type ControlPlannedChild, type ControlRecipient, - cancelControlExecution, childPhaseKinds, childSettlementState, configurationMaintenanceChildState, - controlChildOperationKey, controlPeerState, - encodePrivate, + encodeControlAssertion, executionAuthorityState, fixedControlChildId, type InstallationState, inspectConfigurationPreEffectRefusal, installationState, - newControlKey, parseControlAssertion, readCompletedConfigurationHandoff, - readControlExecutionWitness, + readDataControlTerminalObservation, readPrivateRecord, + recordDataControlTerminalObservation, recordLeaseControlAttempt, recoveryAuthorityState, restoreAdmittedControlExecution, @@ -90,7 +66,6 @@ import { verifyControlEffectSettlement, verifyControlPreparedTerminal, verifyControlRecoveryAuthority, - verifyDataControlTerminalWitness, verifyDataUpdatePrerequisites, withControlDelegationPublication, withControlMaintenanceDecisionPublication, @@ -99,24 +74,21 @@ import { writePrivateRecord, } from "./control-authority"; // biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. -import { - controlDigest, - controlOriginSchema, - digest, - sameControlOrigin, - signControl, - targetSchema, - uuid, - verifyControl, -} from "./control-records"; -import { - AnchoredDirectory, - acquireGuardDirectory, - type FileIdentity, - identity, - sameInode, -} from "./filesystem"; +import { controlChildOperationKey } from "./control-child-records"; +import { controlDigest, targetSchema, uuid, verifyControl } from "./control-records"; import { assertFixedConfigurationMaintenancePlan } from "./fixed-configuration-maintenance"; +import { + admitLease, + knownSettlement, + type LeaseChild, + type LeaseRecord, + type LeaseSettlement, + type LeaseStore, + leaseChildrenOf, + leaseNodeForward, + sameExecution, + TargetBusyError, +} from "./lease-store"; // biome-ignore lint/suspicious/noImportCycles: Type re-export for the existing shared settlement protocol. export type { ChildSettlement } from "./control-authority"; @@ -134,3406 +106,1456 @@ export type BorrowedArtifactTargetLease = { readonly mode: "borrowed"; }; export type TargetLeaseAuthority = ArtifactTargetLease | BorrowedArtifactTargetLease; -const inode = z.strictObject({ dev: z.string().regex(/^\d+$/), ino: z.string().regex(/^\d+$/) }); -const certificateSchema = z.strictObject({ - version: z.literal(2), - origin: controlOriginSchema, - installationId: uuid, - leaseId: uuid, - dataRootIdentity: inode, - lockIdentities: z.record(z.string(), inode), - owner: executionRefSchema, - ownerPublicKey: z.string().max(256), - serviceIds: targetSchema, - createdAt: z.number().int().nonnegative().safe(), -}); -type Certificate = z.infer; -const slotSchema = z.strictObject({ - name: z.string().regex(/^slot-\d{1,4}$/), - identity: inode, - generation: uuid.optional(), -}); -const releaseProgressSchema = z.strictObject({ - version: z.literal(2), - phase: z.enum(["planned", "released"]), - completedTargets: z.number().int().min(0).max(256), - pending: z.number().int().min(0).max(255).optional(), - active: z.boolean(), - activePlanned: z.boolean().optional(), -}); -const admissionSlotSchema = z.strictObject({ - version: z.literal(1), - installationId: uuid, - leaseId: uuid, - certificateDigest: digest, - active: slotSchema, - history: slotSchema, - release: releaseProgressSchema.optional(), - releasedAt: z.number().int().positive().safe().optional(), -}); -type AdmissionSlots = z.infer; -const signedCertificateSchema = z.strictObject({ - payload: certificateSchema, - signature: z.string(), -}); -const outcomeSchema = z.strictObject({ - identity: executionRefSchema, - outcome: z.enum(["contained", "committed", "failed_without_effect", "unknown"]), - publication: z.enum(["not_started", "committed", "unknown"]), - terminalReceiptDigest: digest, -}); -const phaseSchema = z.enum(["held", "cancel_pending", "recovery_required", "settled"]); -const rootSchema = z.strictObject({ - version: z.literal(2), - origin: controlOriginSchema, - certificate: signedCertificateSchema, - sequence: z.number().int().nonnegative().safe(), - phase: phaseSchema, - grantDigests: z.record(uuid, digest), - settlement: outcomeSchema.nullable(), -}); -type RootRecord = z.infer; -const grantSchema = z.strictObject({ - version: z.literal(2), - origin: controlOriginSchema, - installationId: uuid, - leaseId: uuid, - ownerCertificateDigest: digest, - delegationId: uuid, - parentNode: uuid.nullable(), - parent: executionRefSchema, - child: executionRefSchema, - childKind: z.string().max(128), - phase: z.string().max(64), - ordinal: z.number().int().nonnegative().max(63), - serviceIds: targetSchema, - recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), - childPublicKey: z.string().max(256), - permittedFurtherEdges: z.array(z.string()).max(1), - depth: z.number().int().min(1).max(2), -}); -type Grant = z.infer; -const childSchema = z.strictObject({ - version: z.literal(2), - origin: controlOriginSchema, - delegationId: uuid, - grantDigest: digest, - execution: executionRefSchema, - sequence: z.number().int().nonnegative().safe(), - phase: phaseSchema, - grantDigests: z.record(uuid, digest), - settlement: outcomeSchema.nullable(), - nonAdmission: configurationPrepareNonAdmissionSchema.optional(), -}); -type ChildRecord = z.infer; -const unadoptedLifecycleDomain = "openmapx-unadopted-service-hardlinks-v1"; -const unadoptedLifecycleSchema = z.strictObject({ - version: z.literal(1), - origin: controlOriginSchema, - leaseId: uuid, - certificateDigest: digest, - delegationId: uuid, - grantDigest: digest, - parent: executionRefSchema, - child: executionRefSchema, - failedReceiptDigest: digest, - canceledSequence: z.number().int().positive().safe(), - nonAdmission: generationHardlinksNonAdmissionSchema, -}); -type ObservedChild = ChildRecord & { - unadoptedRevocation?: true; - failedLifecycleReceiptDigest?: string; -}; -const transitionSchema = z.strictObject({ - delegationId: uuid, - grantDigest: digest, - sequence: z.literal(1), - state: z.literal("attempted"), -}); -interface State { - data: AnchoredDirectory; - control: AnchoredDirectory; - locks: AnchoredDirectory; - leases: AnchoredDirectory; - ledger: AnchoredDirectory; - installation: InstallationState; - certificate: Certificate; - certificateEnvelope: ReturnType>; - key: KeyObject; - node: string | null; - execution: ExecutionRef; - authority?: AuthorityState; - verifiedAuthority?: VerifiedExecutionAuthority; - closed: boolean; - journalGuardHeld?: boolean; - settlementOnly?: boolean; - slots?: AdmissionSlots; -} -const handles = new WeakMap(); -declare const maintenanceClosureBrand: unique symbol; -export type ConfigurationMaintenanceClosureObservation = Readonly<{ - [maintenanceClosureBrand]: true; -}>; -const maintenanceClosureObservations = new WeakMap< - ConfigurationMaintenanceClosureObservation, - { - authority: VerifiedExecutionAuthority; - certificateDigest: string; - children: Array<{ execution: ExecutionRef; settlement: ChildSettlement; grantDigest: string }>; - } ->(); -export function configurationMaintenanceClosureObservationState( - value: ConfigurationMaintenanceClosureObservation, - authority: VerifiedExecutionAuthority, -) { - const source = maintenanceClosureObservations.get(value); - if (!source || source.authority !== authority) - throw Error("Actual original target-tree closure observation required"); - return source; +/** + * Where a role keeps its leases. The agent, its fixed CLI children and its + * configuration helper use the store directly; data-manager and the host CLI + * ask the agent, which applies the same rules on their behalf. + */ +export interface LeaseBackend { + acquire(input: { + owner: ExecutionRef; + ownerKind: string; + targets: readonly string[]; + }): Promise; + read(leaseId: string): Promise; + /** Only the store itself can answer without waiting. */ + readSync?(leaseId: string): LeaseRecord | undefined; + listSync?(): LeaseRecord[]; + list(): Promise; + delegate(input: { + leaseId: string; + child: Omit; + }): Promise; + adopt(input: { leaseId: string; key: string; execution: ExecutionRef }): Promise; + settle(input: { + leaseId: string; + key: string | null; + settlement: LeaseSettlement; + }): Promise; + cancel(input: { leaseId: string; key: string | null }): Promise; + release(input: { leaseId: string }): Promise; } -function maintenanceObservationFromTree( - s: State, - authority: VerifiedExecutionAuthority, -): ConfigurationMaintenanceClosureObservation { - const current = tree(s), - children: Array<{ execution: ExecutionRef; settlement: ChildSettlement; grantDigest: string }> = - []; - for (const [id, grant] of current.grants) { - const child = current.children.get(id); - if (!child || child.phase !== "settled" || !child.settlement || !knownOutcome(child.settlement)) - throw Error("Every original maintenance descendant must be contained"); - children.push({ - execution: grant.child, - settlement: child.settlement, - grantDigest: child.grantDigest, - }); + +const MAX_DESCENDANTS = 64; +function nodeOf(lease: LeaseRecord, key: string | null) { + if (key === null) return lease; + const child = lease.children.find((entry) => entry.key === key); + if (!child) throw Error("Missing retained child node"); + return child; +} +function descendantsOf(lease: LeaseRecord, key: string | null): LeaseChild[] { + if (key === null) return lease.children; + const result: LeaseChild[] = []; + for (const child of lease.children) { + let parent = child.parentKey; + while (parent !== null) { + if (parent === key) { + result.push(child); + break; + } + parent = lease.children.find((entry) => entry.key === parent)?.parentKey ?? null; + } } - const observation = Object.freeze({}) as ConfigurationMaintenanceClosureObservation; - maintenanceClosureObservations.set(observation, { - authority, - certificateDigest: controlDigest(s.certificateEnvelope), - children, - }); - return observation; + return result; } -function assertMaintenanceClosureFromTree(s: State, authority: VerifiedExecutionAuthority): void { - const observation = maintenanceObservationFromTree(s, authority); - try { - assertConfigurationMaintenanceRootClosure(authority, observation); - } finally { - maintenanceClosureObservations.delete(observation); - } +function settledKnown(node: { state: string; settlement?: LeaseSettlement }): boolean { + return node.state === "settled" && knownSettlement(node.settlement); } -/** Fixed read-only inspector source, using the same authenticated complete tree - * as release. It creates no lease, issuer, forward capability or final proof. */ -export function observeConfigurationMaintenanceRuntimeTree( - authority: VerifiedExecutionAuthority, -): ConfigurationMaintenanceClosureObservation { - const owner = executionAuthorityState(authority); - if ( - owner.kind !== "configurationMaintenance.prepare" || - owner.installation.installation.role !== "ops-agent" || - owner.installation.fixedChildId - ) - throw Error("Original maintenance runtime tree required"); - const reference = z - .strictObject({ leaseId: uuid, certificateDigest: digest }) - .parse( - readPrivateRecord( - owner.installation, - "execution-leases", - `${owner.execution.operationKey}.json`, - ), - ), - s = openState(owner.installation.dataRoot, reference.leaseId, owner.installation, true); - try { + +/** The lease rules, applied under the store lock by every writer. */ +export const leaseRules = { + acquire(leases: LeaseRecord[], input: Parameters[0], now: number) { + const existing = leases.find((lease) => sameExecution(lease.owner, input.owner)); + if (existing) { + if ( + existing.ownerKind !== input.ownerKind || + canonicalJson(existing.targets) !== canonicalJson([...input.targets].sort()) + ) + throw Error("Execution already acquired a different root lease"); + return existing; + } + if (now >= input.owner.deadlineAt) throw Error("Target closure or execution deadline mismatch"); + return admitLease(leases, { ...input, now }); + }, + delegate(leases: LeaseRecord[], input: Parameters[0], now: number) { + const lease = leases.find((value) => value.leaseId === input.leaseId); + if (!lease) throw Error("Target lease is not held"); + const existing = lease.children.find((child) => child.key === input.child.key); + if (existing) { + if (!sameExecution(existing.execution, input.child.execution)) + throw Error("Attempted child cannot be replayed"); + return lease; + } if ( - controlDigest(s.execution) !== controlDigest(owner.execution) || - controlDigest(s.certificateEnvelope) !== reference.certificateDigest + !leaseNodeForward(lease, input.child.parentKey, now) || + now >= input.child.execution.deadlineAt ) - throw Error("Original maintenance runtime certificate changed"); - return guarded(s, () => maintenanceObservationFromTree(s, authority), !s.slots?.release); - } finally { - closeState(s); - } -} -/** Public observation uses the same authenticated tree as the actual release - * paths. No caller source array/boolean can satisfy the internal predicate. */ -export function verifyConfigurationMaintenanceRootClosure( - authority: VerifiedExecutionAuthority, -): void { - const owner = executionAuthorityState(authority), - reference = z - .strictObject({ leaseId: uuid, certificateDigest: digest }) - .parse( - readPrivateRecord( - owner.installation, - "execution-leases", - `${owner.execution.operationKey}.json`, - ), - ), - s = openState(owner.installation.dataRoot, reference.leaseId, owner.installation, true); - try { + throw Error("Lease does not permit forward delegation"); + if (lease.children.length >= MAX_DESCENDANTS) throw Error("Child intent capacity exhausted"); if ( - controlDigest(s.execution) !== controlDigest(owner.execution) || - controlDigest(s.certificateEnvelope) !== reference.certificateDigest + lease.children.some( + (child) => child.execution.operationKey === input.child.execution.operationKey, + ) ) - throw Error("Original maintenance observation certificate changed"); - guarded(s, () => assertMaintenanceClosureFromTree(s, authority), !s.slots?.release); - } finally { - closeState(s); - } + throw Error("Attempted child cannot be replayed"); + const parent = + input.child.parentKey === null ? undefined : nodeOf(lease, input.child.parentKey); + const scope = parent && "targets" in parent ? parent.targets : lease.targets; + if (input.child.targets.some((id) => !scope.includes(id))) throw Error("Widened target subset"); + if ( + parent && + "permittedFurtherEdges" in parent && + (!parent.permittedFurtherEdges.includes(input.child.phase) || parent.depth >= 2) + ) + throw Error("Unauthorized descendant edge"); + lease.children.push({ ...input.child, adopted: false, state: "held" }); + return lease; + }, + adopt(leases: LeaseRecord[], input: Parameters[0], now: number) { + const lease = leases.find((value) => value.leaseId === input.leaseId); + const child = lease?.children.find((value) => value.key === input.key); + if (!lease || !child || !sameExecution(child.execution, input.execution)) + throw Error("Wrong peer principal, kind, key, fingerprint, or subset"); + if (!child.adopted) { + if (!leaseNodeForward(lease, child.parentKey, now) || now >= child.execution.deadlineAt) + throw Error("Stale or canceled delegation"); + child.adopted = true; + } + return lease; + }, + settle(leases: LeaseRecord[], input: Parameters[0]) { + const lease = leases.find((value) => value.leaseId === input.leaseId); + if (!lease) throw Error("Target lease is not held"); + const node = nodeOf(lease, input.key); + if ( + !sameExecution(input.settlement.identity, "execution" in node ? node.execution : lease.owner) + ) + throw Error("Foreign settlement identity"); + for (const child of leaseChildrenOf(lease, input.key)) + if (!settledKnown(child)) throw Error("Unsettled attempted child retains target fence"); + if (node.state === "settled") { + if (canonicalJson(node.settlement) !== canonicalJson(input.settlement)) + throw Error("Original settlement changed"); + return lease; + } + node.settlement = input.settlement; + node.state = knownSettlement(input.settlement) ? "settled" : "recovery_required"; + return lease; + }, + cancel(leases: LeaseRecord[], input: Parameters[0]) { + const lease = leases.find((value) => value.leaseId === input.leaseId); + if (!lease) return undefined; + const node = nodeOf(lease, input.key); + if (node.state === "held") node.state = "cancel_pending"; + return lease; + }, + release(leases: LeaseRecord[], input: Parameters[0]) { + const index = leases.findIndex((value) => value.leaseId === input.leaseId); + const lease = leases[index]; + if (!lease) return; + if (!settledKnown(lease)) throw Error("Durable own-effect settlement is required"); + if (lease.children.some((child) => !settledKnown(child))) + throw Error("Every attempted descendant must settle"); + leases.splice(index, 1); + }, +}; + +/** The agent's own lease backend over its store. */ +export function localLeaseBackend(store: LeaseStore, now: () => number = Date.now): LeaseBackend { + const get = (leaseId: string) => store.get(leaseId); + return { + acquire: (input) => store.update((leases) => leaseRules.acquire(leases, input, now())), + read: async (leaseId) => get(leaseId), + readSync: get, + listSync: () => store.list(), + list: async () => store.list(), + delegate: (input) => store.update((leases) => leaseRules.delegate(leases, input, now())), + adopt: (input) => store.update((leases) => leaseRules.adopt(leases, input, now())), + settle: (input) => store.update((leases) => leaseRules.settle(leases, input)), + cancel: (input) => store.update((leases) => leaseRules.cancel(leases, input)), + release: (input) => store.update((leases) => leaseRules.release(leases, input)), + }; } -function required(value: T | undefined, message: string): T { - if (value === undefined) throw Error(message); - return value; + +interface State { + backend: LeaseBackend; + installation: InstallationState; + leaseId: string; + node: string | null; + owner: ExecutionRef; + execution: ExecutionRef; + serviceIds: readonly string[]; + authority?: VerifiedExecutionAuthority; + settlementOnly?: boolean; + /** A remote lease is only read when asked; this remembers what was last seen. */ + seen?: LeaseRecord; + canceled?: boolean; } +const handles = new WeakMap(); function stateOf(lease: TargetLeaseAuthority): State { const state = handles.get(lease); - if (!state || state.closed || (lease.mode === "owner") !== (state.node === null)) + if (!state || (lease.mode === "owner") !== (state.node === null)) throw Error("Unverified or mixed target lease authority"); - verifyAnchors(state); return state; } -function verifyAnchors(s: State) { - s.data.check(); - s.ledger.check(); - if (!sameInode(identity(s.data.stat), s.certificate.dataRootIdentity)) - throw Error("Data root physical identity changed"); - for (const id of s.certificate.serviceIds) { - const lock = s.locks.directory(id); - try { - if ( - !sameInode( - identity(lock.stat), - required(s.certificate.lockIdentities[id], "Missing certified target inode"), - ) - ) - throw Error("Target lock inode identity changed"); - if (lock.listBounded(1).length) throw Error("Unexplained nonempty target lock"); - } finally { - lock.close(); - } +function handleFor(state: State): TargetLeaseAuthority { + const lease = + state.node === null + ? Object.freeze({ + leaseId: state.leaseId, + owner: state.owner, + serviceIds: Object.freeze([...state.serviceIds]), + mode: "owner" as const, + }) + : Object.freeze({ + leaseId: state.leaseId, + owner: state.owner, + execution: state.execution, + serviceIds: Object.freeze([...state.serviceIds]), + mode: "borrowed" as const, + }); + handles.set(lease, state); + return lease; +} +/** The current record of a lease the role can read without waiting, else the last one seen. */ +function current(s: State): LeaseRecord | undefined { + if (s.backend.readSync) { + const record = s.backend.readSync(s.leaseId); + s.seen = record; + return record; } + return s.seen; } -function json(dir: AnchoredDirectory, name: string, max = 131072): unknown { - return JSON.parse(dir.read(name, max).toString("utf8")); -} -function readRoot(s: State): RootRecord { - return verifyControl( - s.certificate.ownerPublicKey, - "openmapx-lease-owner-v2", - json(s.ledger, "owner.json"), - rootSchema, - ); -} -function childDirectory(s: State, kind: "grants" | "children") { - return s.ledger.directory(kind); +async function fresh(s: State): Promise { + s.seen = await s.backend.read(s.leaseId); + return s.seen; } -function readGrant(s: State, id: string, parentKey: string): Grant { - const dir = childDirectory(s, "grants"); - try { - const envelope = z - .strictObject({ grant: z.unknown(), transition: z.unknown() }) - .parse(json(dir, `${uuid.parse(id)}.json`)); - const grant = verifyControl(parentKey, "openmapx-lease-grant-v2", envelope.grant, grantSchema); - const transition = verifyControl( - parentKey, - "openmapx-lease-intent-v1", - envelope.transition, - transitionSchema, - ); - if ( - !sameControlOrigin(grant.origin, s.certificate.origin) || - grant.delegationId !== id || - transition.delegationId !== id || - transition.grantDigest !== controlDigest(envelope.grant) || - grant.ownerCertificateDigest !== controlDigest(s.certificateEnvelope) || - grant.installationId !== s.installation.installation.installationId || - grant.leaseId !== s.certificate.leaseId - ) - throw Error("Anchored grant identity mismatch"); - return grant; - } finally { - dir.close(); - } +function remember(s: State, record: LeaseRecord | undefined) { + s.seen = record; } -function readChild(s: State, grant: Grant): ObservedChild | undefined { - const dir = childDirectory(s, "children"); - try { - let raw: unknown; - try { - raw = json(dir, `${grant.delegationId}.json`); - } catch (e) { - if ((e as NodeJS.ErrnoException).code === "ENOENT") return; - throw e; - } - if (raw && typeof raw === "object" && "unadoptedRevocation" in raw) { - const envelope = z.strictObject({ unadoptedRevocation: z.unknown() }).parse(raw), - revoked = verifyControl( - s.certificate.ownerPublicKey, - unadoptedLifecycleDomain, - envelope.unadoptedRevocation, - unadoptedLifecycleSchema, - ), - root = readRoot(s), - nonAdmission = verifyControl( - s.installation.publicKeys["data-manager"], - GENERATION_HARDLINKS_NON_ADMISSION_DOMAIN, - revoked.nonAdmission, - generationHardlinksNonAdmissionPayloadSchema, - ); - if ( - s.certificate.owner.executor !== "host-cli" || - grant.parentNode !== null || - grant.phase !== "hardlinks" || - grant.childKind !== "generation-hardlinks" || - grant.recipient !== "data-manager" || - grant.depth !== 1 || - revoked.leaseId !== s.certificate.leaseId || - revoked.certificateDigest !== controlDigest(s.certificateEnvelope) || - revoked.delegationId !== grant.delegationId || - revoked.grantDigest !== root.grantDigests[grant.delegationId] || - controlDigest(revoked.parent) !== controlDigest(s.certificate.owner) || - controlDigest(revoked.child) !== controlDigest(grant.child) || - !sameControlOrigin(revoked.origin, grant.origin) || - root.phase === "held" || - root.sequence < revoked.canceledSequence - ) - throw Error("Original unadopted lifecycle revocation changed"); + +export function verifyFiniteControlEdge( + role: ControlRecipient | "host-cli", + kind: string, + phase: ControlChildPhase, + recipient: ControlRecipient, +): string[] { + if (role === "data-manager") { + if (kind === "catalog.validate") { if ( - nonAdmission.installationId !== s.certificate.installationId || - nonAdmission.leaseId !== revoked.leaseId || - nonAdmission.certificateDigest !== revoked.certificateDigest || - nonAdmission.delegationId !== revoked.delegationId || - nonAdmission.grantDigest !== revoked.grantDigest || - controlDigest(nonAdmission.parent) !== controlDigest(revoked.parent) || - controlDigest(nonAdmission.child) !== controlDigest(revoked.child) || - controlDigest(nonAdmission.origin) !== controlDigest(revoked.origin) || - nonAdmission.canceledSequence !== revoked.canceledSequence + recipient !== "ops-agent" || + !["stop-staging", "restart-staging", "proxy-stage", "proxy-restore"].includes(phase) ) - throw Error("Data Manager non-admission binding changed"); - return { - version: 2, - origin: grant.origin, - delegationId: grant.delegationId, - grantDigest: revoked.grantDigest, - execution: grant.child, - sequence: 0, - phase: "settled", - grantDigests: {}, - unadoptedRevocation: true, - failedLifecycleReceiptDigest: revoked.failedReceiptDigest, - settlement: { - identity: grant.child, - outcome: "failed_without_effect", - publication: "not_started", - terminalReceiptDigest: controlDigest(envelope.unadoptedRevocation), - }, - }; + throw Error("Unauthorized catalog validation child edge"); + return []; } - const child = verifyControl(grant.childPublicKey, "openmapx-lease-child-v2", raw, childSchema); if ( - !sameControlOrigin(child.origin, grant.origin) || - child.delegationId !== grant.delegationId || - controlDigest(child.execution) !== controlDigest(grant.child) + !["transit-sync", "catalog.activate", "transit-maintenance"].includes(kind) || + recipient !== "ops-agent" || + ![ + "stop-primary", + "stop-staging", + "restart-staging", + "promote-primary", + "recover-primary", + "proxy-stage", + "proxy-commit", + "proxy-restore", + "paired-pin-commit", + ].includes(phase) || + (phase === "paired-pin-commit" && kind !== "catalog.activate") ) - throw Error("Child durable identity mismatch"); - if (child.nonAdmission) { - const proof = verifyControl( - grant.childPublicKey, - CONFIGURATION_PREPARE_NON_ADMISSION_DOMAIN, - child.nonAdmission, - configurationPrepareNonAdmissionPayloadSchema, - ); - if ( - grant.recipient !== "data-manager" || - grant.childKind !== "configuration-maintenance.prepare" || - grant.phase !== "configuration-fence" || - grant.parentNode !== null || - proof.installationId !== s.certificate.installationId || - proof.leaseId !== s.certificate.leaseId || - proof.certificateDigest !== controlDigest(s.certificateEnvelope) || - proof.delegationId !== grant.delegationId || - proof.grantDigest !== child.grantDigest || - !sameControlOrigin(proof.origin, grant.origin) || - controlDigest(proof.child) !== controlDigest(grant.child) || - controlDigest(proof.input.ownerPrepare) !== controlDigest(grant.parent) || - controlDigest({ kind: grant.childKind, ...proof.input }) !== - grant.child.operationFingerprint || - proof.observedAt <= Math.max(grant.child.deadlineAt, grant.parent.deadlineAt) || - proof.canceledSequence > readRoot(s).sequence || - child.phase !== "settled" || - Object.keys(child.grantDigests).length || - child.settlement?.outcome !== "failed_without_effect" || - child.settlement.publication !== "not_started" || - child.settlement.terminalReceiptDigest !== controlDigest(proof) - ) - throw Error("Original unadmitted preparation settlement changed"); - } - return child; - } finally { - dir.close(); - } -} -function highWater(s: State, nodes: Record) { - const name = `${s.certificate.leaseId}.json`; - let prior: Record = {}; - try { - prior = z - .record( - z.string(), - z.strictObject({ sequence: z.number().int().nonnegative().safe(), digest }), - ) - .parse(readPrivateRecord(s.installation, "observations", name)); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; + throw Error("Unauthorized data-manager child edge"); + return []; } - for (const [key, old] of Object.entries(prior)) { - const current = nodes[key]; + if ( + role === "ops-agent" && + recipient === "fixed-cli" && + ["lifecycle", "build", "data-link", "data-clean", "data-finalization"].includes(phase) && + childPhaseKinds[phase].includes(kind) + ) + return phase === "data-clean" ? [] : ["hardlinks"]; + if ( + ["ops-agent", "fixed-cli", "host-cli"].includes(role) && + recipient === "data-manager" && + phase === "hardlinks" && + [ + "service.start", + "service.update", + "service.build", + "services.buildAll", + "data.link", + "configuration.hardlinks.apply", + "data.update", + "artifactTransfer.import", + ].includes(kind) + ) + return []; + if ( + role === "ops-agent" && + recipient === "configuration-helper" && + phase === "configuration-handoff" && + kind === "configurationHandoff.prepare" + ) + return []; + if (role === "ops-agent" && kind === "configurationMaintenance.prepare") { + if ( + recipient === "data-manager" && + ["configuration-fence", "configuration-fence-settle"].includes(phase) + ) + return []; if ( - !current || - current.sequence < old.sequence || - (current.sequence === old.sequence && current.digest !== old.digest) + recipient === "ops-agent" && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(phase) ) - throw Error("Shared ledger rollback or replacement"); + return phase === "maintenance-hardlinks" + ? ["hardlinks"] + : phase === "maintenance-handoff" + ? ["configuration-handoff"] + : []; } - writePrivateRecord(s.installation, "observations", name, nodes, false); + throw Error("Unauthorized finite control edge"); } -type LeaseTree = { - root: RootRecord; - grants: Map; - children: Map; -}; -function tree(s: State): LeaseTree { - if (!s.journalGuardHeld) return guarded(s, () => tree(s), false); - const root = readRoot(s); + +export async function acquireArtifactTargetLease(options: { + serviceIds: readonly string[]; + owner: VerifiedExecutionAuthority; + signal?: AbortSignal; +}): Promise { + const authority = executionAuthorityState(options.owner); + if (authority.installation.fixedChildId) + throw Error("Fixed children must borrow and cannot acquire a root"); + const serviceIds = targetSchema.parse(options.serviceIds); if ( - !sameControlOrigin(root.origin, s.certificate.origin) || - controlDigest(root.certificate) !== controlDigest(s.certificateEnvelope) + serviceIds.some((id) => !authority.serviceIds.includes(id)) || + Date.now() >= authority.execution.deadlineAt ) - throw Error("Owner certificate replacement"); - const nodes: Record = { - owner: { sequence: root.sequence, digest: controlDigest(root) }, - }; - const grants = new Map(); - const children = new Map(); - let count = 0; - function walk( - parent: ExecutionRef, - node: string | null, - subset: readonly string[], - publicKey: string, - digests: Record, - depth: number, - edges?: readonly string[], - ) { - for (const [id, expected] of Object.entries(digests)) { - if (++count > 64) throw Error("Child intent capacity exceeded"); - const grant = readGrant(s, id, publicKey); - const dir = childDirectory(s, "grants"); - try { - const raw = json(dir, `${id}.json`) as { grant: unknown }; - if (controlDigest(raw.grant) !== expected) throw Error("Grant digest changed"); - } finally { - dir.close(); - } - if ( - grant.child.fingerprint !== - executionFingerprint({ - kind: grant.childKind, - operationFingerprint: grant.child.operationFingerprint, - deadlineAt: grant.child.deadlineAt, - parent: retainedControlParent(grant.parent), - }) || - grant.parentNode !== node || - controlDigest(grant.parent) !== controlDigest(parent) || - grant.depth !== depth || - grant.child.deadlineAt > parent.deadlineAt || - grant.serviceIds.some((id) => !subset.includes(id)) || - (edges && !edges.includes(grant.phase)) - ) - throw Error("Grant ancestry/deadline/subset mismatch"); - const witness = readControlExecutionWitness( - s.installation, - parent, - node === null ? s.installation.publicKeys[parent.executor] : publicKey, - ); - const allowedKinds = childPhaseKinds[grant.phase as ControlChildPhase]; - if ( - !sameControlOrigin(witness.origin, grant.origin) || - !allowedKinds?.includes(grant.childKind) || - grant.child.operationKey !== - controlChildOperationKey( - parent.operationKey, - grant.phase as ControlChildPhase, - grant.ordinal, - ) - ) - throw Error("Grant kind or stable phase key mismatch"); - const parentRecipient = - node === null ? s.certificate.owner.executor : grants.get(node)?.recipient; - if (!parentRecipient) throw Error("Missing parent recipient"); - const expectedEdges = verifyFiniteControlEdge( - parentRecipient, - witness.kind, - grant.phase as ControlChildPhase, - grant.recipient, - ); - if ( - canonicalJson(expectedEdges) !== canonicalJson(grant.permittedFurtherEdges) || - grant.serviceIds.some((id) => !witness.serviceIds.includes(id)) - ) - throw Error("Grant widens certified parent policy"); - grants.set(id, grant); - const child = readChild(s, grant); - if (child) { - if (child.grantDigest !== expected) throw Error("Adoption grant digest mismatch"); - children.set(id, child); - nodes[id] = { sequence: child.sequence, digest: controlDigest(child) }; - walk( - grant.child, - id, - grant.serviceIds, - grant.childPublicKey, - child.grantDigests, - depth + 1, - grant.permittedFurtherEdges, - ); - } - } - } - walk( - s.certificate.owner, - null, - s.certificate.serviceIds, - s.certificate.ownerPublicKey, - root.grantDigests, - 1, - ); - const grantDir = childDirectory(s, "grants"), - childDir = childDirectory(s, "children"); - try { - if ( - grantDir.listBounded(64).some((n) => !grants.has(n.replace(/\.json$/, ""))) || - childDir.listBounded(64).some((n) => !children.has(n.replace(/\.json$/, ""))) - ) - throw Error("Unexplained protocol record; recovery required"); - } finally { - grantDir.close(); - childDir.close(); - } - highWater(s, nodes); - return { root, grants, children }; -} -function ledgerBytes(s: State) { - let bytes = - s.ledger.read("owner.json", 131072).length + s.ledger.read("admission.json", 4096).length; - for (const kind of ["grants", "children"] as const) { - const dir = childDirectory(s, kind); - try { - if (dir.listBounded(64).length > 64) throw Error("Protocol record capacity"); - for (const name of dir.listBounded(64)) bytes += dir.read(name, 131072).length; - } finally { - dir.close(); - } - } - return bytes; -} -function writeBounded( - s: State, - kind: "owner" | "grants" | "children", - name: string, - value: unknown, - exclusive = false, -) { - const dir = kind === "owner" ? s.ledger : childDirectory(s, kind); - try { - const bytes = canonicalJson(value); - if (name === "admission.json" && Buffer.byteLength(bytes) > 4096) - throw Error("Admission release journal exceeds byte capacity"); - let old = 0; - try { - old = dir.read(name, 131072).length; - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - if (ledgerBytes(s) - old + Buffer.byteLength(bytes) > 131072) - throw Error("Lease ledger capacity exceeded"); - if (s.certificate.origin) dir.write(name, bytes, exclusive); - else dir.write(name, bytes, exclusive); - } finally { - if (dir !== s.ledger) dir.close(); - } -} -/** Only failure to acquire the existing journal guard, before its body runs. - * This is coordination evidence, never permission to reclaim a held lock. */ -export class ArtifactTargetJournalBusyError extends Error { - constructor(cause: unknown) { - super("Lease journal writer is busy or requires supervised recovery", { cause }); - } -} -function guarded(s: State, run: () => T, checkTargets = true): T { - if (s.journalGuardHeld) throw Error("Nested lease journal writer"); - if (checkTargets) verifyAnchors(s); - else { - s.data.check(); - s.ledger.check(); - } - const path = s.ledger.name(".journal-writer"); - try { - acquireGuardDirectory(path); - fsyncSync(s.ledger.fd); - } catch (cause) { - throw new ArtifactTargetJournalBusyError(cause); - } - const guard = s.ledger.directory(".journal-writer"); - const expected = identity(guard.stat); - try { - s.journalGuardHeld = true; - return run(); - } finally { - s.journalGuardHeld = false; - guard.close(); - s.ledger.removeDirectory(".journal-writer", expected); - } -} -function writeNode(s: State, node: RootRecord | ChildRecord) { - const domain = s.node === null ? "openmapx-lease-owner-v2" : "openmapx-lease-child-v2"; - writeBounded( - s, - s.node === null ? "owner" : "children", - s.node === null ? "owner.json" : `${s.node}.json`, - signControl(s.key, domain, node), - ); -} -function closeState(s: State) { - if (s.closed) return; - s.closed = true; - s.ledger.close(); - s.leases.close(); - s.locks.close(); - s.control.close(); - s.data.close(); -} -function knownOutcome(e: ChildSettlement | null | undefined) { - return ( - !!e && - e.outcome !== "unknown" && - e.publication !== "unknown" && - !(e.outcome === "failed_without_effect" && e.publication !== "not_started") - ); -} -function admissionIndex( - control: AnchoredDirectory, - leases: AnchoredDirectory, - installation: InstallationState, -) { - let created = false; - try { - mkdirSync(control.name("admission-index-v1"), { mode: 0o700 }); - fsyncSync(control.fd); - created = true; - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; - } - const index = control.directory("admission-index-v1"); - try { - if (created) { - // Existing installations require supervised indexing; never silently omit an old fence. - if (leases.listBounded(1).length) - throw Error("Existing leases require supervised admission-index recovery"); - index.directory("active", true).close(); - index.directory("history", true).close(); - index.write( - "ready.json", - canonicalJson({ installationId: installation.installation.installationId }), - true, - ); - } - const ready = z.strictObject({ installationId: uuid }).parse(json(index, "ready.json", 1024)); - if (ready.installationId !== installation.installation.installationId) - throw Error("Foreign admission index"); - return index; - } catch (e) { - index.close(); - throw e; - } -} -/** At most this many leases are held at once, and this many are retained - * until the control prune removes released ones. */ -const ACTIVE_LEASES = 256, - RETAINED_LEASES = 1024; -function reserveAuthoritySlot(dir: AnchoredDirectory, maximum: number, until: number) { - for (let slot = 0; slot < maximum; slot++) { - if (Date.now() >= until) throw Error("Target lease acquisition deadline elapsed"); - const name = `slot-${slot}`; - try { - mkdirSync(dir.name(name), { mode: 0o700 }); - } catch (e) { - if ((e as NodeJS.ErrnoException).code === "EEXIST") continue; - throw e; - } - fsyncSync(dir.fd); - const child = dir.directory(name); - try { - const generation = randomUUID(); - child.write("generation.json", JSON.stringify({ generation }), true); - return { name, identity: identity(child.stat), generation }; - } finally { - child.close(); - } - } - throw Error("Retained lease capacity exhausted; await the control prune or containment"); -} -function removeSlot(dir: AnchoredDirectory, slot: z.infer) { - const child = dir.directory(slot.name); - try { - if (!sameInode(identity(child.stat), slot.identity)) - throw Error("Replaced admission slot inode"); - if (slot.generation) { - const value = z - .strictObject({ generation: uuid }) - .parse(json(child, "generation.json", 1024)); - if (value.generation !== slot.generation) throw Error("Reused admission slot generation"); - child.removeFile("generation.json"); - } - } finally { - child.close(); - } - dir.removeDirectory(slot.name, slot.identity); -} -/** Normal coordinator staging owns no target lease. Observe the bounded shared - * index without a writer, allocation, high-water write or sibling private read. - * Partial admission/release and concurrent replacement are fail-busy. */ -export function assertCoordinatorPullLeaseCompatibility( - authority: VerifiedExecutionAuthority, -): void { - assertImagePullTargetLeaseCompatibility(authority, "service.pull"); -} -/** Conservative reciprocal target check for the one fixed immutable image stage. */ -export function assertImageStageLeaseCompatibility(authority: VerifiedExecutionAuthority): void { - assertImagePullTargetLeaseCompatibility(authority, "service.image.stage"); -} -function assertImagePullTargetLeaseCompatibility( - authority: VerifiedExecutionAuthority, - kind: "service.pull" | "service.image.stage", -): void { - const state = executionAuthorityState(authority), - raw = readPrivateRecord( - state.installation, - "executions", - `${state.execution.operationKey}.json`, - 65536, - ), - admitted = verifyControl( - state.installation.installation.publicKey, - "openmapx-control-execution-v2", - raw, - z - .object({ - kind: z.literal(kind), - execution: executionRefSchema, - operation: opsOperationSchema, - }) - .passthrough(), - ); - if ( - state.installation.fixedChildId || - state.kind !== kind || - controlDigest(raw) !== state.receiptDigest || - controlDigest(admitted.execution) !== controlDigest(state.execution) || - admitted.operation.kind !== kind || - !("serviceId" in admitted.operation) || - (admitted.operation.kind === "service.pull" && - (admitted.operation.serviceId !== "data-manager" || !admitted.operation.candidateId)) || - opsOperationFingerprint(admitted.operation) !== state.execution.operationFingerprint - ) - throw Error("Actual normal coordinator pull required"); - if (!("serviceId" in admitted.operation)) throw Error("Image target required"); - const target = admitted.operation.serviceId; - if (!state.installation.registeredTargets.has(target)) - throw Error("Registered image target required"); - const opened: AnchoredDirectory[] = [], - open = (parent: AnchoredDirectory, name: string) => { - const dir = parent.directory(name); - opened.push(dir); - return dir; - }, - data = new AnchoredDirectory(state.installation.dataRoot); - try { - let control: AnchoredDirectory; - try { - control = open(data, ".admin-target-control"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - data.check(); - return; - } - throw error; - } - // Before the first root, execution witnesses may exist without a target - // index. Only actual absence of both lock and retained-owner entries is safe. - if (!control.listBounded(16).includes("admission-index-v1")) { - for (const name of ["locks", "leases"]) { - let dir: AnchoredDirectory; - try { - dir = open(control, name); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; - throw error; - } - if (dir.hasEntries()) throw Error("Unindexed physical target ownership"); - } - control.check(); - return; - } - const index = open(control, "admission-index-v1"), - active = open(index, "active"), - locks = open(control, "locks"), - leases = open(control, "leases"), - ready = z.strictObject({ installationId: uuid }).parse(json(index, "ready.json", 1024)); - if (ready.installationId !== state.installation.installation.installationId) - throw Error("Foreign target index"); - const snapshot = () => - Object.fromEntries( - active - .listBounded(ACTIVE_LEASES) - .sort() - .map((name) => { - if (!/^slot-\d{1,4}$/.test(name) || Number(name.slice(5)) >= ACTIVE_LEASES) - throw Error("Unknown active target slot"); - const dir = active.directory(name); - try { - return [ - name, - { - identity: identity(dir.stat), - generation: z - .strictObject({ generation: uuid }) - .parse(json(dir, "generation.json", 1024)).generation, - }, - ]; - } finally { - dir.close(); - } - }), - ), - before = snapshot(), - matched = new Set(); - if (locks.listBounded(256).includes(target)) throw Error("Coordinator target is held"); - const names = leases.listBounded(RETAINED_LEASES + 3).sort(); - for (const name of names) { - uuid.parse(name); - const owner = openState(state.installation.dataRoot, name, state.installation, true); - try { - const root = readRoot(owner), - slots = owner.slots; - if ( - !slots || - !sameControlOrigin(root.origin, owner.certificate.origin) || - controlDigest(root.certificate) !== controlDigest(owner.certificateEnvelope) - ) - throw Error("Changed physical owner"); - if (slots.release?.phase === "released" && slots.releasedAt) { - if (root.phase !== "settled" || !knownOutcome(root.settlement)) - throw Error("Unproved physical release"); - continue; - } - if (owner.certificate.serviceIds.includes(target)) - throw Error("Coordinator target is unreleased"); - const slot = before[slots.active.name]; - if ( - !slot || - matched.has(slots.active.name) || - !sameInode(slot.identity, slots.active.identity) || - slot.generation !== slots.active.generation - ) - throw Error("Unindexed active physical owner"); - matched.add(slots.active.name); - } finally { - closeState(owner); - } - } - if ( - matched.size !== Object.keys(before).length || - canonicalJson(snapshot()) !== canonicalJson(before) || - canonicalJson(leases.listBounded(RETAINED_LEASES + 3).sort()) !== canonicalJson(names) || - locks.listBounded(256).includes(target) - ) - throw Error("Physical target index is changing"); - for (const dir of opened) dir.check(); - } finally { - for (const dir of opened.reverse()) dir.close(); - data.close(); - } -} -/** Names who holds a target. The refusal stands even when the holder cannot be read. */ -function heldTargetError( - installation: InstallationState, - leases: AnchoredDirectory, - target: string, -): Error { - let holder: string | undefined; - try { - for (const name of leases.listBounded(RETAINED_LEASES + 3)) { - if (holder || !uuid.safeParse(name).success) continue; - try { - const owner = openState(installation.dataRoot, name, installation, true); - try { - if (owner.certificate.serviceIds.includes(target) && !owner.slots?.releasedAt) - holder = owner.execution.operationKey; - } finally { - closeState(owner); - } - } catch { - /* A lease being written or retired is not the one to name. */ - } - } - } catch { - /* The lease directory could not be listed. */ - } - return Error( - holder - ? `Target lease busy: ${target} is held by operation ${holder}. Wait for it to finish; a service start or update records its state in infra/docker/data/ops-agent/lifecycle/${holder}.json.` - : `Target lease busy: ${target} is held by an operation that could not be identified. Inspect infra/docker/data/.admin-target-control/leases.`, - ); -} -export async function acquireArtifactTargetLease(options: { - dataRoot: string; - serviceIds: readonly string[]; - owner: VerifiedExecutionAuthority; - signal?: AbortSignal; -}): Promise { - const beganAt = Date.now(); - const authority = executionAuthorityState(options.owner); - if (authority.installation.fixedChildId) - throw Error("Fixed children must borrow and cannot acquire a root"); - const serviceIds = targetSchema.parse(options.serviceIds); - if ( - serviceIds.some((id) => !authority.serviceIds.includes(id)) || - Date.now() >= authority.execution.deadlineAt - ) - throw Error("Target closure or execution deadline mismatch"); - if (authority.kind === "data.update") await verifyDataUpdatePrerequisites(options.owner); - assertControlExecutionForward(options.owner); - options.signal?.throwIfAborted(); - try { - readPrivateRecord( - authority.installation, - "execution-leases", - `${authority.execution.operationKey}.json`, - ); - throw Error("Execution already acquired a root lease; reconcile its original identity"); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - const data = new AnchoredDirectory(options.dataRoot); - const trusted = new AnchoredDirectory(authority.installation.dataRoot); - try { - if (!sameInode(identity(data.stat), identity(trusted.stat))) - throw Error("Foreign physical data root"); - } finally { - trusted.close(); - } - const control = data.directory(".admin-target-control", true, true), - locks = control.directory("locks", true, true), - leases = control.directory("leases", true, true); - const acquired: Record = {}; - let published = false; - let ledger: AnchoredDirectory | undefined; - let activeSlot: z.infer | undefined, - historySlot: z.infer | undefined; - let index: AnchoredDirectory | undefined; - try { - const until = Math.min(beganAt + 5000, authority.execution.deadlineAt); - index = admissionIndex(control, leases, authority.installation); - const history = index.directory("history"), - active = index.directory("active"); - try { - historySlot = reserveAuthoritySlot(history, RETAINED_LEASES, until); - activeSlot = reserveAuthoritySlot(active, ACTIVE_LEASES, until); - } finally { - active.close(); - history.close(); - } - for (const id of serviceIds) { - for (;;) { - options.signal?.throwIfAborted(); - if (Date.now() >= until) throw heldTargetError(authority.installation, leases, id); - try { - mkdirSync(locks.name(id), { mode: 0o700 }); - fsyncSync(locks.fd); - const lock = locks.directory(id); - acquired[id] = identity(lock.stat); - lock.close(); - break; - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e; - if (Date.now() >= until) throw heldTargetError(authority.installation, leases, id); - await delay(Math.min(50, until - Date.now()), undefined, { signal: options.signal }); - } - } - } - const leaseId = randomUUID(); - ledger = leases.directory(leaseId, true, true); - ledger.directory("grants", true).close(); - ledger.directory("children", true).close(); - const key = newControlKey(); - const certificate: Certificate = { - version: 2, - origin: authority.origin, - installationId: authority.installation.installation.installationId, - leaseId, - dataRootIdentity: identity(data.stat), - lockIdentities: acquired, - owner: authority.execution, - ownerPublicKey: key.publicKey, - serviceIds, - createdAt: Date.now(), - }; - const certificateEnvelope = signControl( - authority.installation.privateKey, - "openmapx-lease-certificate-v2", - certificate, - ); - const slots: AdmissionSlots = { - version: 1, - installationId: certificate.installationId, - leaseId, - certificateDigest: controlDigest(certificateEnvelope), - active: activeSlot, - history: historySlot, - }; - const admissionBytes = canonicalJson( - signControl( - authority.installation.privateKey, - "openmapx-admission-slots-v1", - admissionSlotSchema.parse(slots), - ), - ); - if (Buffer.byteLength(admissionBytes) > 4096) - throw Error("Admission journal exceeds byte capacity"); - ledger?.write("admission.json", admissionBytes, true); - writePrivateRecord( - authority.installation, - "leases", - `${leaseId}.json`, - { - certificate: certificateEnvelope, - privateKey: encodePrivate(key.privateKey), - executionReceiptDigest: authority.receiptDigest, - }, - true, - ); - const record: RootRecord = { - version: 2, - origin: authority.origin, - certificate: certificateEnvelope, - sequence: 0, - phase: "held", - grantDigests: {}, - settlement: null, - }; - writePrivateRecord( - authority.installation, - "execution-leases", - `${authority.execution.operationKey}.json`, - { leaseId, certificateDigest: controlDigest(certificateEnvelope) }, - true, - ); - published = true; - const ownerBytes = canonicalJson( - signControl(key.privateKey, "openmapx-lease-owner-v2", record), - ); - ledger?.write("owner.json", ownerBytes, true); - const state: State = { - data, - control, - locks, - leases, - ledger, - installation: authority.installation, - certificate, - certificateEnvelope, - key: key.privateKey, - node: null, - execution: authority.execution, - authority, - verifiedAuthority: options.owner, - slots, - closed: false, - }; - const lease = Object.freeze({ - leaseId, - owner: certificate.owner, - serviceIds: Object.freeze([...serviceIds]), - mode: "owner" as const, - }); - handles.set(lease, state); - tree(state); - if (Date.now() >= until) - throw Error( - "Target lease acquisition deadline elapsed after publication; reconcile original identity", - ); - index.close(); - return lease; - } catch (e) { - if (!published && index) { - for (const [kind, slot] of [ - ["active", activeSlot], - ["history", historySlot], - ] as const) { - if (!slot) continue; - if (kind === "history" && ledger) continue; // Partial records consume retained capacity too. - const dir = index.directory(kind); - try { - removeSlot(dir, slot); - } finally { - dir.close(); - } - } - } - index?.close(); - if (!published) - for (const [id, expected] of Object.entries(acquired)) locks.removeDirectory(id, expected); - ledger?.close(); - leases.close(); - locks.close(); - control.close(); - data.close(); - throw e; - } -} -export function verifyFiniteControlEdge( - role: ControlRecipient | "host-cli", - kind: string, - phase: ControlChildPhase, - recipient: ControlRecipient, -): string[] { - if (role === "data-manager") { - if (kind === "catalog.validate") { - if ( - recipient !== "ops-agent" || - !["stop-staging", "restart-staging", "proxy-stage", "proxy-restore"].includes(phase) - ) - throw Error("Unauthorized catalog validation child edge"); - return []; - } - if ( - !["transit-sync", "catalog.activate", "transit-maintenance"].includes(kind) || - recipient !== "ops-agent" || - ![ - "stop-primary", - "stop-staging", - "restart-staging", - "promote-primary", - "recover-primary", - "proxy-stage", - "proxy-commit", - "proxy-restore", - "paired-pin-commit", - ].includes(phase) || - (phase === "paired-pin-commit" && kind !== "catalog.activate") - ) - throw Error("Unauthorized data-manager child edge"); - return []; - } - if ( - role === "ops-agent" && - recipient === "fixed-cli" && - ["lifecycle", "build", "data-link", "data-clean", "data-finalization"].includes(phase) && - childPhaseKinds[phase].includes(kind) - ) - return phase === "data-clean" ? [] : ["hardlinks"]; - if ( - ["ops-agent", "fixed-cli", "host-cli"].includes(role) && - recipient === "data-manager" && - phase === "hardlinks" && - [ - "service.start", - "service.update", - "service.build", - "services.buildAll", - "data.link", - "configuration.hardlinks.apply", - "data.update", - "artifactTransfer.import", - ].includes(kind) - ) - return []; - if ( - role === "ops-agent" && - recipient === "configuration-helper" && - phase === "configuration-handoff" && - kind === "configurationHandoff.prepare" - ) - return []; - if (role === "ops-agent" && kind === "configurationMaintenance.prepare") { - if ( - recipient === "data-manager" && - ["configuration-fence", "configuration-fence-settle"].includes(phase) - ) - return []; - if ( - recipient === "ops-agent" && - [ - "configuration-commit", - "maintenance-hardlinks", - "maintenance-recreate", - "maintenance-handoff", - ].includes(phase) - ) - return phase === "maintenance-hardlinks" - ? ["hardlinks"] - : phase === "maintenance-handoff" - ? ["configuration-handoff"] - : []; - } - throw Error("Unauthorized finite control edge"); -} -function permitted(s: State, child: ControlPlannedChild, parentGrant?: Grant): string[] { - if ( - parentGrant && - (!parentGrant.permittedFurtherEdges.includes(child.phase) || parentGrant.depth >= 2) - ) - throw Error("Unauthorized descendant edge"); - const key = parentGrant?.childPublicKey ?? s.certificate.ownerPublicKey; - // Root execution witnesses use the pinned role key; borrowed witnesses use the certified child key. - const witness = readControlExecutionWitness( - s.installation, - s.execution, - s.node === null ? s.installation.publicKeys[s.execution.executor] : key, - ); - return verifyFiniteControlEdge( - parentGrant?.recipient ?? s.installation.installation.role, - witness.kind, - child.phase, - child.recipient, - ); -} -function forwardAncestry(t: ReturnType, node: string | null): boolean { - if (t.root.phase !== "held") return false; - while (node !== null) { - const grant = t.grants.get(node); - if (!grant || t.children.get(node)?.phase !== "held" || Date.now() >= grant.child.deadlineAt) - return false; - node = grant.parentNode; - } - return Date.now() < t.root.certificate.payload.owner.deadlineAt; -} -export async function delegateArtifactTargetLease( - lease: TargetLeaseAuthority, - child: { - execution: ExecutionRef; - phase: ControlChildPhase; - serviceIds: readonly string[]; - recipient: ControlRecipient; - childPublicKey: string; - }, -): Promise { - const s = stateOf(lease); - if (s.settlementOnly) throw Error("Recovered lease permits settlement only"); - const authority = s.verifiedAuthority; - if (!authority) throw Error("A durable local execution must own child grant publication"); - return guarded(s, () => - withControlDelegationPublication(authority, child.phase, () => - publishReservedLeaseGrant(s, lease, child, authority), - ), - ); -} -/** Lease → ancestor execution scopes → local execution scope stay held through - * private reference/outbox, cancellation index and shared/node publication. */ -function publishReservedLeaseGrant( - s: State, - lease: TargetLeaseAuthority, - child: { - execution: ExecutionRef; - phase: ControlChildPhase; - serviceIds: readonly string[]; - recipient: ControlRecipient; - childPublicKey: string; - }, - authority: VerifiedExecutionAuthority, -): ArtifactDelegationAssertion { - const current = tree(s); - const node = - s.node === null - ? current.root - : required(current.children.get(s.node), "Missing retained child node"); - if (!forwardAncestry(current, s.node) || Date.now() >= s.execution.deadlineAt) - throw Error("Lease does not permit forward delegation"); - let plan = resolveControlChildPlan(authority, child); - if (plan && canonicalJson(plan.serviceIds) !== canonicalJson(child.serviceIds)) - throw Error("Reserved child target subset changed"); - if (!plan && s.node !== null) { - const inherited = required(current.grants.get(s.node), "Missing retained child grant"); - const kinds = childPhaseKinds[child.phase]; - if ( - inherited.permittedFurtherEdges.includes(child.phase) && - kinds.length === 1 && - child.execution.operationKey === - controlChildOperationKey(s.execution.operationKey, child.phase, 0) - ) - plan = { ...child, kind: required(kinds[0], "Missing finite child kind"), ordinal: 0 }; - } - if (!plan) throw Error("Child is absent from immutable admitted plan"); - if (current.grants.size >= 64) throw Error("Child intent capacity exhausted"); - if ( - [...current.grants.values()].some((g) => g.child.operationKey === child.execution.operationKey) - ) - throw Error("Attempted child cannot be replayed"); - if (child.serviceIds.some((id) => !lease.serviceIds.includes(id))) - throw Error("Widened target subset"); - const parentGrant = s.node === null ? undefined : current.grants.get(s.node); - const edges = permitted(s, plan, parentGrant); - if ( - s.authority?.kind === "configurationMaintenance.prepare" && - [ - "configuration-commit", - "maintenance-hardlinks", - "maintenance-recreate", - "maintenance-handoff", - ].includes(child.phase) - ) - verifyControlPreparedTerminal(s.installation, s.execution); - let delegationId: string = randomUUID(); - if (child.recipient === "fixed-cli" || child.recipient === "configuration-helper") { - delegationId = fixedControlChildId(s.installation, child.childPublicKey, s.execution); - } else if (child.childPublicKey !== s.installation.publicKeys[child.recipient]) - throw Error("Child role key mismatch"); - const grant: Grant = { - version: 2, - origin: s.certificate.origin, - installationId: s.certificate.installationId, - leaseId: lease.leaseId, - ownerCertificateDigest: controlDigest(s.certificateEnvelope), - delegationId, - parentNode: s.node, - parent: s.execution, - child: child.execution, - childKind: plan.kind, - phase: child.phase, - ordinal: plan.ordinal, - serviceIds: [...child.serviceIds], - recipient: child.recipient, - childPublicKey: child.childPublicKey, - permittedFurtherEdges: edges, - depth: (parentGrant?.depth ?? 0) + 1, - }; - const envelope = signControl(s.key, "openmapx-lease-grant-v2", grant); - const grantDigest = controlDigest(envelope); - const transition = signControl(s.key, "openmapx-lease-intent-v1", { - delegationId, - grantDigest, - sequence: 1 as const, - state: "attempted" as const, - }); - const ancestors: string[] = []; - let ancestor = s.node; - while (ancestor) { - ancestors.unshift(ancestor); - ancestor = current.grants.get(ancestor)?.parentNode ?? null; - } - const fields = { - version: 1 as const, - installationId: s.certificate.installationId, - leaseId: lease.leaseId, - ownerCertificateDigest: controlDigest(s.certificateEnvelope), - delegationId, - grantDigest, - ancestorDelegationIds: ancestors, - }; - const signature = signControl(s.key, "openmapx-lease-assertion-v1", fields).signature; - const assertion = Buffer.from(canonicalJson({ ...fields, signature })).toString( - "base64url", - ) as ArtifactDelegationAssertion; - parseControlAssertion(assertion); - prepareControlChildPublication(authority, plan, { - delegationId, - grantDigest, - leaseId: lease.leaseId, - }); - recordLeaseControlAttempt(authority, { - child: child.execution, - leaseId: lease.leaseId, - delegationId, - grantDigest, - }); - writePrivateRecord( - s.installation, - "attempts", - `${delegationId}.json`, - { grant: envelope, transition, assertion }, - true, - ); - writeBounded(s, "grants", `${delegationId}.json`, { grant: envelope, transition }, true); - node.grantDigests[delegationId] = grantDigest; - node.sequence++; - writeNode(s, node); - tree(s); - return assertion; -} -function openState( - dataRoot: string, - leaseId: string, - installation: InstallationState, - terminalObservation = false, -): State { - uuid.parse(leaseId); - const data = new AnchoredDirectory(dataRoot); - const control = data.directory(".admin-target-control"), - locks = control.directory("locks"), - leases = control.directory("leases"), - ledger = leases.directory(leaseId); - try { - const raw = z.object({ payload: rootSchema }).parse(json(ledger, "owner.json")); - const envelope = raw.payload.certificate; - const cert = certificateSchema.parse(envelope.payload); - const certificate = verifyControl( - installation.publicKeys[cert.owner.executor], - "openmapx-lease-certificate-v2", - envelope, - certificateSchema, - ); - if ( - certificate.installationId !== installation.installation.installationId || - certificate.leaseId !== leaseId - ) - throw Error("Foreign lease installation"); - const slots = verifyControl( - installation.publicKeys[certificate.owner.executor], - "openmapx-admission-slots-v1", - json(ledger, "admission.json", 4096), - admissionSlotSchema, - ); - if ( - slots.leaseId !== leaseId || - slots.installationId !== certificate.installationId || - slots.certificateDigest !== controlDigest(envelope) - ) - throw Error("Admission slot identity mismatch"); - const s = { - slots, - data, - control, - locks, - leases, - ledger, - installation, - certificate, - certificateEnvelope: envelope, - key: installation.privateKey, - node: null, - execution: certificate.owner, - closed: false, - } satisfies State; - if (terminalObservation && readRoot(s).phase === "settled") { - s.data.check(); - s.ledger.check(); - if (!sameInode(identity(s.data.stat), s.certificate.dataRootIdentity)) - throw Error("Changed physical root"); - } else verifyAnchors(s); - return s; - } catch (e) { - ledger.close(); - leases.close(); - locks.close(); - control.close(); - data.close(); - throw e; - } -} -export async function adoptArtifactTargetLease(options: { - dataRoot: string; - assertion: string; - peer: VerifiedControlPeer; - execution: ExecutionRef; - serviceIds: readonly string[]; -}): Promise { - const peer = controlPeerState(options.peer); - if ( - options.assertion !== peer.assertion || - controlDigest(options.execution) !== controlDigest(peer.execution) - ) - throw Error("Peer execution binding mismatch"); - const assertion = parseControlAssertion(options.assertion); - const s = openState(options.dataRoot, assertion.leaseId, peer.installation); - try { - return await guarded(s, () => { - const t = tree(s); - const grant = t.grants.get(assertion.delegationId); - if ( - !grant || - controlDigest(s.certificateEnvelope) !== assertion.ownerCertificateDigest || - controlDigest(grant.child) !== controlDigest(options.execution) || - grant.childKind !== peer.kind || - grant.recipient !== peer.recipient || - grant.childPublicKey !== peer.childPublicKey || - canonicalJson(grant.serviceIds) !== canonicalJson(targetSchema.parse(options.serviceIds)) - ) - throw Error("Wrong peer principal, kind, key, fingerprint, or subset"); - const parentGrant = grant.parentNode ? t.grants.get(grant.parentNode) : undefined; - const parentKey = parentGrant?.childPublicKey ?? s.certificate.ownerPublicKey; - const { signature, ...fields } = assertion; - verifyControl( - parentKey, - "openmapx-lease-assertion-v1", - { payload: fields, signature }, - assertionSchema.omit({ signature: true }), - ); - const parent = t.children.get(grant.parentNode ?? "") ?? t.root; - const expectedAncestors: string[] = []; - let ancestor = grant.parentNode; - while (ancestor) { - expectedAncestors.unshift(ancestor); - ancestor = t.grants.get(ancestor)?.parentNode ?? null; - } - if ( - canonicalJson(expectedAncestors) !== canonicalJson(assertion.ancestorDelegationIds) || - parent.grantDigests[grant.delegationId] !== assertion.grantDigest - ) - throw Error("Wrong grant ancestry/digest"); - const issuer = parentGrant?.recipient ?? s.certificate.owner.executor; - if ( - peer.recipient === "ops-agent" && - issuer !== "data-manager" && - !( - issuer === "ops-agent" && - peer.local && - [ - "configuration-commit", - "maintenance-hardlinks", - "maintenance-recreate", - "maintenance-handoff", - ].includes(grant.phase) - ) - ) - throw Error("Wrong peer principal"); - if ( - peer.recipient === "data-manager" && - !["ops-agent", "fixed-cli", "host-cli"].includes(issuer) - ) - throw Error("Wrong peer principal"); - const retained = t.children.get(grant.delegationId); - if (retained?.unadoptedRevocation) throw Error("Original unadopted grant was revoked"); - if ( - !retained && - (!forwardAncestry(t, grant.parentNode) || Date.now() >= grant.child.deadlineAt) - ) - throw Error("Stale or canceled delegation"); - const admissionName = `${grant.delegationId}.json`; - const pending = { - version: 1, - grantDigest: assertion.grantDigest, - execution: grant.child, - state: "pending-adoption", - assertion: options.assertion, - }; - let local: unknown; - try { - local = readPrivateRecord(peer.installation, "adoptions", admissionName); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - if (local) { - const old = z - .object({ - grantDigest: digest, - execution: executionRefSchema, - state: z.enum(["pending-adoption", "ready"]), - }) - .parse(local); - if ( - old.grantDigest !== assertion.grantDigest || - controlDigest(old.execution) !== controlDigest(grant.child) - ) - throw Error("Durable duplicate admission conflict"); - } else writePrivateRecord(peer.installation, "adoptions", admissionName, pending, true); - s.key = peer.childPrivateKey; - s.node = grant.delegationId; - s.execution = grant.child; - if (!retained) - writeBounded( - s, - "children", - `${s.node}.json`, - signControl(s.key, "openmapx-lease-child-v2", { - version: 2, - origin: grant.origin, - delegationId: s.node, - grantDigest: assertion.grantDigest, - execution: grant.child, - sequence: 0, - phase: "held", - grantDigests: {}, - settlement: null, - } satisfies ChildRecord), - true, - ); - writePrivateRecord( - peer.installation, - "adoptions", - admissionName, - { - ...pending, - state: "ready", - }, - false, - ); - tree(s); - const handle = Object.freeze({ - leaseId: assertion.leaseId, - owner: s.certificate.owner, - execution: grant.child, - serviceIds: Object.freeze([...grant.serviceIds]), - mode: "borrowed" as const, - }); - handles.set(handle, s); - return handle; - }); - } catch (e) { - closeState(s); - throw e; - } -} -/** The fixed PREPARE receiver serializes private admission with cancellation - * and negative recovery under the existing physical journal guard. */ -export function withConfigurationFenceReceiverAdmission( - lease: BorrowedArtifactTargetLease, - run: () => VerifiedExecutionAuthority, -): VerifiedExecutionAuthority { - const s = stateOf(lease); - if (!s.node || s.installation.installation.role !== "data-manager" || s.installation.fixedChildId) - throw Error("Original normal configuration receiver required"); - return guarded(s, () => { - const grant = tree(s).grants.get(s.node!); - if ( - grant?.childKind !== "configuration-maintenance.prepare" || - grant.phase !== "configuration-fence" - ) - throw Error("Original configuration preparation grant required"); - return run(); - }); -} - -/** A fixed receiver supplies its own durable execution authority only after adoption. */ -export function bindBorrowedControlExecution( - lease: BorrowedArtifactTargetLease, - authority: VerifiedExecutionAuthority, -): void { - const s = stateOf(lease), - a = executionAuthorityState(authority); - if ( - controlDigest(a.execution) !== controlDigest(lease.execution) || - !sameControlOrigin(a.origin, s.certificate.origin) || - a.installation.installation.installationId !== s.certificate.installationId - ) - throw Error("Borrower execution authority mismatch"); - s.authority = a; - s.verifiedAuthority = authority; - if ( - ["generation-hardlinks", "configuration-maintenance.prepare"].includes(a.kind) && - a.installation.installation.role === "data-manager" - ) { - if (!s.node) throw Error("Original generation adoption required"); - const payload = { - version: 2, - origin: a.origin, - execution: a.execution, - admissionDigest: a.receiptDigest, - leaseId: s.certificate.leaseId, - delegationId: s.node, - certificateDigest: controlDigest(s.certificateEnvelope), - }; - const name = `${a.execution.operationKey}.json`; - const domain = - a.kind === "generation-hardlinks" - ? "openmapx-generation-adoption-v2" - : "openmapx-configuration-fence-adoption-v2"; - let prior: unknown; - try { - prior = readPrivateRecord(a.installation, "execution-adoptions", name, 8192); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - if (prior !== undefined) { - const saved = verifyControl( - a.installation.installation.publicKey, - domain, - prior, - z.unknown(), - ); - if (controlDigest(saved) !== controlDigest(payload)) - throw Error("Generation adoption changed"); - } else - writePrivateRecord( - a.installation, - "execution-adoptions", - name, - signControl(a.installation.privateKey, domain, payload), - true, - ); - } -} -/** Source-only recovery address for the original normal DM PREPARE adopter. - * It allocates no lease/grant or deadline, and reads only this role's records. */ -export function retainedConfigurationFenceAdoption(authority: VerifiedExecutionAuthority) { - const owner = executionAuthorityState(authority); - if ( - owner.kind !== "configuration-maintenance.prepare" || - owner.installation.installation.role !== "data-manager" - ) - throw Error("Original normal DM fence adopter required"); - const value = verifyControl( - owner.installation.installation.publicKey, - "openmapx-configuration-fence-adoption-v2", - readPrivateRecord( - owner.installation, - "execution-adoptions", - `${owner.execution.operationKey}.json`, - 8192, - ), - z.strictObject({ - version: z.literal(2), - origin: controlOriginSchema, - execution: executionRefSchema, - admissionDigest: digest, - leaseId: uuid, - delegationId: uuid, - certificateDigest: digest, - }), - ); - if ( - canonicalJson(value.execution) !== canonicalJson(owner.execution) || - value.admissionDigest !== owner.receiptDigest || - canonicalJson(value.origin) !== canonicalJson(owner.origin) - ) - throw Error("Original normal DM fence adoption changed"); - return Object.freeze(value); -} -/** Reopen only the original authenticated generation leaf for settlement. - * There is no adopt, target acquire, deadline renewal or owner-release path. */ -export function restoreGenerationHardlinksLeaseForSettlement( - authority: VerifiedExecutionAuthority, -): BorrowedArtifactTargetLease { - const a = executionAuthorityState(authority); - if (a.kind !== "generation-hardlinks" || a.installation.installation.role !== "data-manager") - throw Error("Original generation receiver required"); - const retained = verifyControl( - a.installation.installation.publicKey, - "openmapx-generation-adoption-v2", - readPrivateRecord( - a.installation, - "execution-adoptions", - `${a.execution.operationKey}.json`, - 8192, - ), - z.strictObject({ - version: z.literal(2), - origin: controlOriginSchema, - execution: executionRefSchema, - admissionDigest: digest, - leaseId: uuid, - delegationId: uuid, - certificateDigest: digest, - }), - ); - if ( - controlDigest(retained.execution) !== controlDigest(a.execution) || - retained.admissionDigest !== a.receiptDigest || - !sameControlOrigin(retained.origin, a.origin) - ) - throw Error("Generation retained adoption identity changed"); - const s = openState(a.installation.dataRoot, retained.leaseId, a.installation); - try { - return guarded(s, () => { - const adoption = z - .object({ - state: z.literal("ready"), - execution: executionRefSchema, - assertion: z.string(), - grantDigest: digest, - }) - .parse( - readPrivateRecord(a.installation, "adoptions", `${retained.delegationId}.json`, 8192), - ); - const assertion = parseControlAssertion(adoption.assertion); - if ( - assertion.leaseId !== retained.leaseId || - assertion.delegationId !== retained.delegationId || - assertion.grantDigest !== adoption.grantDigest || - retained.certificateDigest !== controlDigest(s.certificateEnvelope) || - controlDigest(adoption.execution) !== controlDigest(a.execution) || - !sameControlOrigin(a.origin, s.certificate.origin) - ) - throw Error("Original ready generation grant required"); - s.node = retained.delegationId; - s.execution = a.execution; - s.key = a.installation.privateKey; - s.authority = a; - s.verifiedAuthority = authority; - s.settlementOnly = true; - const t = tree(s), - grant = t.grants.get(s.node), - child = t.children.get(s.node); - if ( - !grant || - !child || - grant.recipient !== "data-manager" || - grant.childKind !== "generation-hardlinks" || - grant.childPublicKey !== a.installation.installation.publicKey || - controlDigest(grant.child) !== controlDigest(a.execution) || - controlDigest(child.execution) !== controlDigest(a.execution) - ) - throw Error("Original generation child is missing or foreign"); - const handle = Object.freeze({ - leaseId: retained.leaseId, - owner: s.certificate.owner, - execution: a.execution, - serviceIds: Object.freeze([...grant.serviceIds]), - mode: "borrowed" as const, - }); - handles.set(handle, s); - return handle; - }); - } catch (error) { - closeState(s); - throw error; - } -} -/** Exact borrowed generation leaf containment; its parent root remains held. - * This authenticates the existing own-effect and physical terminal sources, - * never a root release, new adoption or forward capability. */ -export function verifyRetainedGenerationHardlinksSettlement( - authority: VerifiedExecutionAuthority, -): void { - const lease = restoreGenerationHardlinksLeaseForSettlement(authority), - s = stateOf(lease); - try { - guarded(s, () => { - const current = tree(s), - child = s.node && current.children.get(s.node), - effect = childSettlementState( - verifyControlEffectSettlement(s.installation.installation, s.execution), - ).evidence; - if ( - !child || - child.phase !== "settled" || - !knownOutcome(child.settlement) || - controlDigest(child.settlement) !== controlDigest(effect) || - [...current.grants.values()].some((value) => value.parentNode === s.node) - ) - throw Error("Original generation leaf containment required"); - }); - } finally { - closeArtifactTargetLeaseHandle(lease); - } -} -export async function settleArtifactTargetLeaseChild( - lease: TargetLeaseAuthority, - evidence: VerifiedChildSettlement, -): Promise { - const s = stateOf(lease); - const proof = childSettlementState(evidence); - if ( - proof.state !== s.installation || - controlDigest(proof.evidence.identity) !== controlDigest(s.execution) - ) - throw Error("Foreign settlement identity"); - await guarded(s, () => { - const t = tree(s); - if (s.node === null && s.authority?.kind === "configurationMaintenance.prepare") { - if (!s.verifiedAuthority) throw Error("Original maintenance completion authority required"); - assertMaintenanceClosureFromTree(s, s.verifiedAuthority); - } - const node = - s.node === null ? t.root : required(t.children.get(s.node), "Missing retained child node"); - for (const grant of t.grants.values()) { - if (grant.parentNode === s.node) { - const child = t.children.get(grant.delegationId); - if (child?.phase !== "settled" || !knownOutcome(child.settlement)) - throw Error("Unsettled attempted child retains target fence"); - } - } - node.settlement = proof.evidence; - node.phase = knownOutcome(proof.evidence) ? "settled" : "recovery_required"; - node.sequence++; - writeNode(s, node); - tree(s); - }); -} -/** Complete only the original terminal handoff preparer after its exact fixed - * helper has committed and physically settled. No target/root is acquired. */ -export async function settlePreparedConfigurationHandoff( - authority: VerifiedExecutionAuthority, -): Promise { - const owner = executionAuthorityState(authority); - if ( - owner.kind !== "configurationHandoff.prepare" || - owner.installation.fixedChildId || - owner.installation.installation.role !== "ops-agent" - ) - throw Error("Original handoff preparer required"); - const admitted = readPrivateRecord( - owner.installation, - "executions", - `${owner.execution.operationKey}.json`, - 65536, - ) as { payload: { operation: unknown } }, - operation = opsOperationSchema.parse(admitted.payload.operation); - if (operation.kind !== "configurationHandoff.prepare") - throw Error("Original handoff operation required"); - const source = readCompletedConfigurationHandoff( - owner.installation.installation, - operation.handoffId, - ), - s = openState(owner.installation.dataRoot, source.intent.leaseId, owner.installation); - try { - const current = tree(s), - parent = [...current.grants.values()].filter( - (grant) => canonicalJson(grant.child) === canonicalJson(owner.execution), - ), - helper = current.grants.get(source.intent.delegationId), - child = current.children.get(source.intent.delegationId); - if ( - parent.length !== 1 || - canonicalJson(source.intent.preparer) !== canonicalJson(owner.execution) || - canonicalJson(source.intent.operation) !== canonicalJson(operation) || - source.intent.preparerAdmissionDigest !== owner.receiptDigest || - source.intent.ownerCertificateDigest !== controlDigest(s.certificateEnvelope) || - canonicalJson(source.intent.ownerPrepare) !== canonicalJson(s.execution) || - helper?.parentNode !== parent[0]!.delegationId || - helper.recipient !== "configuration-helper" || - helper.phase !== "configuration-handoff" || - helper.childKind !== "configuration-helper" || - helper.childPublicKey !== source.intent.helperPublicKey || - canonicalJson(helper.child) !== canonicalJson(source.intent.helper) || - child?.phase !== "settled" || - canonicalJson(child.settlement) !== canonicalJson(source.settlement) || - parent[0]!.childPublicKey !== owner.installation.installation.publicKey - ) - throw Error("Original prepared helper tree changed"); - verifyControlPreparedTerminal(owner.installation, owner.execution); - const evidence = verifyControlEffectSettlement( - owner.installation.installation, - owner.execution, - ), - proof = childSettlementState(evidence).evidence; - if (proof.outcome !== "committed" || proof.publication !== "committed") - throw Error("Original prepared handoff effect required"); - const prior = current.children.get(parent[0]!.delegationId); - if (prior?.phase === "settled") { - if (canonicalJson(prior.settlement) !== canonicalJson(proof)) - throw Error("Original handoff settlement changed"); - return; - } - s.node = parent[0]!.delegationId; - s.execution = owner.execution; - s.key = owner.installation.privateKey; - s.authority = owner; - s.verifiedAuthority = authority; - s.settlementOnly = true; - const lease = Object.freeze({ - leaseId: source.intent.leaseId, - owner: s.certificate.owner, - execution: owner.execution, - serviceIds: parent[0]!.serviceIds, - mode: "borrowed" as const, - }); - handles.set(lease, s); - try { - await settleArtifactTargetLeaseChild(lease, evidence); - } finally { - handles.delete(lease); - } - } finally { - closeState(s); - } -} -/** Receipt-only reopening of the original completed fixed helper. The signed - * completion/effect and original ready adoption must already exist. No target - * acquire/adopt, forward capability or clock renewal is available here. */ -export async function settleRetainedConfigurationHandoffHelper( - authority: VerifiedExecutionAuthority, -): Promise { - const owner = executionAuthorityState(authority); - if (owner.kind !== "configuration-helper" || !owner.installation.fixedChildId) - throw Error("Original completed fixed helper required"); - const raw = readPrivateRecord( - owner.installation, - "executions", - `${owner.execution.operationKey}.json`, - 65536, - ) as { payload: { operation: { handoffId?: unknown } } }, - source = readCompletedConfigurationHandoff( - owner.installation.installation, - uuid.parse(raw.payload.operation.handoffId), - ), - adopted = readPrivateRecord( - owner.installation, - "adoptions", - `${source.intent.delegationId}.json`, - 8192, - ), - adoption = z - .object({ - state: z.literal("ready"), - execution: executionRefSchema, - assertion: z.string(), - grantDigest: digest, - }) - .parse(adopted), - assertion = parseControlAssertion(adoption.assertion); - if ( - canonicalJson(source.intent.helper) !== canonicalJson(owner.execution) || - source.intent.helperPublicKey !== owner.installation.installation.publicKey || - source.intent.delegationId !== owner.installation.fixedChildId || - adoption.assertion !== source.intent.assertion || - canonicalJson(adoption.execution) !== canonicalJson(owner.execution) || - assertion.leaseId !== source.intent.leaseId || - assertion.delegationId !== source.intent.delegationId || - assertion.grantDigest !== adoption.grantDigest - ) - throw Error("Original helper ready adoption changed"); - const s = openState(owner.installation.dataRoot, source.intent.leaseId, owner.installation); - try { - const current = tree(s), - grant = current.grants.get(source.intent.delegationId), - child = current.children.get(source.intent.delegationId); - if ( - controlDigest(s.certificateEnvelope) !== source.intent.ownerCertificateDigest || - !sameControlOrigin(s.certificate.origin, owner.origin) || - grant?.recipient !== "configuration-helper" || - grant.childKind !== "configuration-helper" || - grant.childPublicKey !== owner.installation.installation.publicKey || - child?.grantDigest !== adoption.grantDigest || - canonicalJson(grant.child) !== canonicalJson(owner.execution) || - !child - ) - throw Error("Original retained helper tree changed"); - if (child.phase === "settled") { - if (canonicalJson(child.settlement) !== canonicalJson(source.settlement)) - throw Error("Original helper settlement changed"); - return; - } - verifyControlPreparedTerminal(owner.installation, source.intent.preparer); - s.node = source.intent.delegationId; - s.execution = owner.execution; - s.key = owner.installation.privateKey; - s.authority = owner; - s.verifiedAuthority = authority; - s.settlementOnly = true; - const lease = Object.freeze({ - leaseId: source.intent.leaseId, - owner: s.certificate.owner, - execution: owner.execution, - serviceIds: grant.serviceIds, - mode: "borrowed" as const, - }); - handles.set(lease, s); - try { - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(owner.installation.installation, owner.execution), - ); - } finally { - handles.delete(lease); - } - } finally { - closeState(s); - } -} -/** Verify the exact agent-signed shared child node; no sibling private mount or - * caller status can stand in for actual target containment. */ -export function inspectArtifactTargetLeaseChildSettlement( - lease: TargetLeaseAuthority, - execution: ExecutionRef, -): ChildSettlement | null { - const s = stateOf(lease); - return guarded(s, () => { - const t = tree(s), - entries = [...t.grants.values()].filter( - (grant) => - grant.parentNode === s.node && controlDigest(grant.child) === controlDigest(execution), - ); - if (entries.length !== 1) throw Error("Exact retained child grant required"); - const child = t.children.get(entries[0]!.delegationId); - if (!child || child.phase !== "settled" || !knownOutcome(child.settlement)) return null; - if (controlDigest(child.execution) !== controlDigest(execution)) - throw Error("Shared child execution changed"); - return child.settlement; - }); -} - -/** Observe an original owned child's signed settlement after its root released - * physical locks. This returns data only: no lease/issuer/forward capability. */ -function retainedOwnedTargetState(authority: VerifiedExecutionAuthority) { - const owner = executionAuthorityState(authority), - reference = z - .strictObject({ leaseId: uuid, certificateDigest: digest }) - .parse( - readPrivateRecord( - owner.installation, - "execution-leases", - `${owner.execution.operationKey}.json`, - ), - ), - saved = z - .strictObject({ - certificate: signedCertificateSchema, - privateKey: z.string(), - executionReceiptDigest: digest, - }) - .parse(readPrivateRecord(owner.installation, "leases", `${reference.leaseId}.json`)), - s = openState(owner.installation.dataRoot, reference.leaseId, owner.installation, true); - try { - if ( - controlDigest(s.execution) !== controlDigest(owner.execution) || - controlDigest(s.certificateEnvelope) !== reference.certificateDigest || - controlDigest(saved.certificate) !== reference.certificateDigest || - saved.executionReceiptDigest !== owner.receiptDigest || - !sameControlOrigin(owner.origin, s.certificate.origin) - ) - throw Error("Original retained target owner changed"); - return s; - } catch (error) { - closeState(s); - throw error; - } -} -/** Fixed terminal-only source check for a recovered catalog activation. */ -export function assertRetainedCatalogActivationChildrenSettled( - authority: VerifiedExecutionAuthority, -): void { - const owner = executionAuthorityState(authority); - if (owner.kind !== "catalog.activate" || owner.installation.installation.role !== "data-manager") - throw Error("Original catalog activation owner required"); - const s = retainedOwnedTargetState(authority); - try { - guarded( - s, - () => { - const current = tree(s); - for (const grant of current.grants.values()) { - const child = current.children.get(grant.delegationId); - if (child?.phase !== "settled" || !knownOutcome(child.settlement)) - throw Error("Catalog child tree remains unsettled"); - } - }, - !s.slots?.release, - ); - } finally { - closeState(s); - } -} -export function inspectRetainedOwnedArtifactTargetChildSettlement( - authority: VerifiedExecutionAuthority, - execution: ExecutionRef, -): ChildSettlement | null { - const s = retainedOwnedTargetState(authority); - try { - return guarded( - s, - () => { - const current = tree(s), - grants = [...current.grants.values()].filter( - (grant) => - grant.parentNode === null && controlDigest(grant.child) === controlDigest(execution), - ); - if (grants.length !== 1) throw Error("Exact original child grant required"); - const child = current.children.get(grants[0]!.delegationId); - if (!child || child.phase !== "settled" || !knownOutcome(child.settlement)) return null; - if (controlDigest(child.execution) !== controlDigest(execution)) - throw Error("Original child identity changed"); - return child.settlement; - }, - !s.slots?.release, - ); - } finally { - closeState(s); - } -} -/** Source-only observation of the original held maintenance root. It creates - * no forward authority, lease, grant or renewed deadline. */ -export function configurationMaintenanceTargetOwner(lease: ArtifactTargetLease) { - const s = stateOf(lease); - return guarded(s, () => { - const current = tree(s); - if ( - s.node !== null || - s.installation.installation.role !== "ops-agent" || - s.authority?.kind !== "configurationMaintenance.prepare" || - current.root.phase === "settled" || - s.slots?.release - ) - throw Error("Held original maintenance target owner required"); - return Object.freeze({ - installation: s.installation.installation, - execution: s.execution, - origin: s.certificate.origin, - admissionDigest: s.authority.receiptDigest, - certificateDigest: controlDigest(s.certificateEnvelope), - serviceIds: s.certificate.serviceIds, - }); - }); -} -export function withConfigurationMaintenanceOwnerInitialization( - lease: ArtifactTargetLease, - authority: VerifiedExecutionAuthority, - run: () => T, -): T { - const s = stateOf(lease); - if ( - s.node !== null || - s.verifiedAuthority !== authority || - s.authority?.kind !== "configurationMaintenance.prepare" || - s.settlementOnly - ) - throw Error("Original live maintenance initialization required"); - return guarded(s, () => - withControlMaintenanceInitialization(authority, () => - withPrivateEffectWriter(executionAuthorityState(authority), run), - ), - ); -} -/** Publication of already proved guard observations remains possible after - * cancellation/expiry. This returns no forward lease or dispatch capability. */ -export function withConfigurationMaintenanceOwnerObservation( - lease: ArtifactTargetLease, - authority: VerifiedExecutionAuthority, - run: () => T, -): T { - const s = stateOf(lease), - owner = executionAuthorityState(authority); - if ( - s.node !== null || - s.authority?.kind !== "configurationMaintenance.prepare" || - owner.receiptDigest !== s.authority.receiptDigest || - controlDigest(owner.execution) !== controlDigest(s.execution) - ) - throw Error("Original configuration maintenance recovery owner required"); - return guarded(s, () => { - if (tree(s).root.phase === "settled" || s.slots?.release) - throw Error("Maintenance observation owner already released"); - return withPrivateEffectWriter(owner, run); - }); -} -/** Lease -> original forward ancestry/terminal-prepare -> private writer. - * This is a metadata CAS scope, not restoration of a forward lease. */ -export function withConfigurationMaintenanceDecisionPublication( - lease: ArtifactTargetLease, - authority: VerifiedExecutionAuthority, - run: () => T, -): T { - const s = stateOf(lease), - owner = executionAuthorityState(authority); - if ( - s.node !== null || - s.authority?.kind !== "configurationMaintenance.prepare" || - owner.receiptDigest !== s.authority.receiptDigest || - controlDigest(owner.execution) !== controlDigest(s.execution) - ) - throw Error("Original maintenance decision owner required"); - return guarded(s, () => { - if (tree(s).root.phase !== "held" || s.slots?.release) - throw Error("Original prepared maintenance root is fenced"); - return withControlMaintenanceDecisionPublication(authority, () => - withPrivateEffectWriter(owner, run), - ); - }); -} -/** Fixed frozen-phase metadata publication follows the same live original - * lease/ancestry/private-writer order as that phase's grant. */ -export function withConfigurationMaintenancePhasePublication( - lease: ArtifactTargetLease, - authority: VerifiedExecutionAuthority, - phase: - | "configuration-commit" - | "maintenance-hardlinks" - | "maintenance-recreate" - | "maintenance-handoff", - run: () => T, -): T { - const s = stateOf(lease), - owner = executionAuthorityState(authority); - if ( - s.node !== null || - s.authority?.kind !== "configurationMaintenance.prepare" || - s.settlementOnly || - s.verifiedAuthority !== authority - ) - throw Error("Original frozen maintenance issuer required"); - return guarded(s, () => - withControlDelegationPublication(authority, phase, () => { - if (tree(s).root.phase !== "held" || s.slots?.release) - throw Error("Original maintenance phase is fenced"); - return withPrivateEffectWriter(owner, run); - }), - ); -} -/** Original PREPARE reservation publication can restore its own retained handle, - * but the handle never escapes and its forward clock/ancestry are rechecked. */ -export function withConfigurationMaintenanceReservationPublication( - authority: VerifiedExecutionAuthority, - run: () => T, -): T { - const lease = restoreArtifactTargetLeaseForSettlement(authority), - s = stateOf(lease), - owner = executionAuthorityState(authority); - try { - if (owner.kind !== "configurationMaintenance.prepare" || s.node !== null) - throw Error("Original maintenance reservation owner required"); - return guarded(s, () => { - if (tree(s).root.phase !== "held" || s.slots?.release) - throw Error("Original pre-effect maintenance root required"); - return withControlDelegationPublication(authority, "configuration-fence", () => - withPrivateEffectWriter(owner, run), - ); - }); - } finally { - closeArtifactTargetLeaseHandle(lease); - } -} -function finishOwnedRelease(s: State): void { - guarded( - s, - () => { - const t = tree(s); - if (t.root.phase !== "settled" || !knownOutcome(t.root.settlement)) - throw Error("Durable own-effect settlement is required"); - if (s.authority?.kind === "configurationMaintenance.prepare") { - if (!s.verifiedAuthority) throw Error("Original maintenance release authority required"); - assertMaintenanceClosureFromTree(s, s.verifiedAuthority); - } - for (const id of t.grants.keys()) { - const child = t.children.get(id); - if (child?.phase !== "settled" || !knownOutcome(child.settlement)) - throw Error("Every attempted descendant must settle"); - } - if (!s.slots) throw Error("Missing admission slots"); - const persist = () => - writeBounded( - s, - "owner", - "admission.json", - signControl( - s.installation.privateKey, - "openmapx-admission-slots-v1", - admissionSlotSchema.parse(s.slots), - ), - ); - const saved = s.slots.release; - const release: z.infer = saved ?? { - version: 2, - phase: "planned", - completedTargets: 0, - active: false, - }; - releaseProgressSchema.parse(release); - if ( - release.completedTargets > s.certificate.serviceIds.length || - (release.pending !== undefined && - (release.pending !== release.completedTargets || - release.pending >= s.certificate.serviceIds.length)) || - ((release.active || release.activePlanned) && - (release.completedTargets !== s.certificate.serviceIds.length || - release.pending !== undefined)) || - (release.phase === "released" && !release.active) - ) - throw Error("Invalid release journal certified prefix"); - s.slots.release = release; - for (let index = release.completedTargets; index < s.certificate.serviceIds.length; index++) { - const id = required(s.certificate.serviceIds[index], "Missing certified target"); - if (release.pending !== undefined && release.pending !== index) - throw Error("Conflicting physical release step"); - if (release.pending === undefined) { - const lock = s.locks.directory(id); - try { - if ( - !sameInode( - identity(lock.stat), - required(s.certificate.lockIdentities[id], "Missing certified inode"), - ) || - lock.hasEntries() - ) - throw Error("Changed unplanned target release"); - } finally { - lock.close(); - } - release.pending = index; - persist(); - } - try { - s.locks.removeDirectory( - id, - required(s.certificate.lockIdentities[id], "Missing certified target inode"), - ); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - release.completedTargets = index + 1; - delete release.pending; - persist(); - } - if (!release.active) { - const previouslyPlanned = release.activePlanned === true; - const index = s.control.directory("admission-index-v1"), - active = index.directory("active"); - try { - if (!previouslyPlanned) { - const current = active.directory(s.slots.active.name); - try { - if (!sameInode(identity(current.stat), s.slots.active.identity)) - throw Error("Replaced unplanned active slot"); - if ( - s.slots.active.generation && - z.strictObject({ generation: uuid }).parse(json(current, "generation.json", 1024)) - .generation !== s.slots.active.generation - ) - throw Error("Reused unplanned active slot"); - } finally { - current.close(); - } - release.activePlanned = true; - persist(); - } - let slot: AnchoredDirectory | undefined; - try { - slot = active.directory(s.slots.active.name); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - if (slot) { - try { - if (!sameInode(identity(slot.stat), s.slots.active.identity)) - throw Error("Replaced active slot inode"); - if (s.slots.active.generation) { - try { - const g = z - .strictObject({ generation: uuid }) - .parse(json(slot, "generation.json", 1024)); - if (g.generation !== s.slots.active.generation) - throw Error("Reused active slot generation"); - slot.removeFile("generation.json"); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e; - } - } - } finally { - slot.close(); - } - active.removeDirectory(s.slots.active.name, s.slots.active.identity); - } - } finally { - active.close(); - index.close(); - } - release.active = true; - persist(); - } - release.phase = "released"; - s.slots.releasedAt ??= Date.now(); - persist(); - }, - !s.slots?.release, - ); -} -export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Promise { - const s = stateOf(lease); - if (s.node !== null) throw Error("Borrower cannot release root"); - finishOwnedRelease(s); - closeState(s); -} -/** Whether a lease in the shared ledger is gone or has released its root. - * Read-only; used by the prune to keep borrowers of a still-held lease. */ -export function artifactTargetLeaseReleased( - installation: InstallationState, - leaseId: string, -): boolean { - let s: State; + throw Error("Target closure or execution deadline mismatch"); + if (authority.kind === "data.update") await verifyDataUpdatePrerequisites(options.owner); + assertControlExecutionForward(options.owner); + options.signal?.throwIfAborted(); + const name = `${authority.execution.operationKey}.json`; try { - s = openState(installation.dataRoot, leaseId, installation, true); + readPrivateRecord(authority.installation, "execution-leases", name); + throw Error("Execution already acquired a root lease; reconcile its original identity"); } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return true; - throw error; - } - try { - return s.slots?.release?.phase === "released"; - } finally { - closeState(s); + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } -} -/** Removes the released root lease of an execution whose records are being - * pruned: first its retained history slot, then its ledger. */ -export function removeReleasedArtifactTargetLease(authority: VerifiedExecutionAuthority): void { - verifyRetainedArtifactTargetRootRelease(authority); - const a = executionAuthorityState(authority), - { leaseId } = z - .object({ leaseId: uuid }) - .parse( - readPrivateRecord(a.installation, "execution-leases", `${a.execution.operationKey}.json`), - ), - s = openState(a.installation.dataRoot, leaseId, a.installation, true); - try { - if (s.slots) { - const index = s.control.directory("admission-index-v1"), - history = index.directory("history"); - try { - removeSlot(history, s.slots.history); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } finally { - history.close(); - index.close(); - } + // A holder that is finishing gets five seconds to release, within the deadline. + const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt); + let record: LeaseRecord; + for (;;) { + try { + record = await authority.installation.leases.acquire({ + owner: authority.execution, + ownerKind: authority.kind, + targets: serviceIds, + }); + break; + } catch (error) { + if (!(error instanceof TargetBusyError) || Date.now() >= until) throw error; + await new Promise((resolve) => setTimeout(resolve, 50)); + options.signal?.throwIfAborted(); } - } finally { - closeState(s); - } - rmSync(join(a.installation.dataRoot, ".admin-target-control", "leases", leaseId), { - recursive: true, - force: true, - }); -} -/** Read-only original root release proof for a data command's parent. */ -export function verifyRetainedArtifactTargetRootRelease( - authority: VerifiedExecutionAuthority, -): Readonly<{ releasedAt: number }> { - const a = executionAuthorityState(authority), - reference = z - .strictObject({ leaseId: uuid, certificateDigest: digest }) - .parse( - readPrivateRecord(a.installation, "execution-leases", `${a.execution.operationKey}.json`), - ); - const s = openState(a.installation.dataRoot, reference.leaseId, a.installation, true); - try { - const t = tree(s), - effect = childSettlementState( - verifyControlEffectSettlement(a.installation.installation, a.execution), - ).evidence; - if ( - controlDigest(s.execution) !== controlDigest(a.execution) || - controlDigest(s.certificateEnvelope) !== reference.certificateDigest || - !Number.isSafeInteger(s.slots?.releasedAt) || - !s.slots?.releasedAt || - s.slots?.release?.phase !== "released" || - t.root.phase !== "settled" || - controlDigest(t.root.settlement) !== controlDigest(effect) || - [...t.grants.keys()].some( - (id) => - t.children.get(id)?.phase !== "settled" || !knownOutcome(t.children.get(id)?.settlement), - ) - ) - throw Error("Original root release is not proved"); - return Object.freeze({ releasedAt: s.slots.releasedAt! }); - } finally { - closeState(s); } + // The lease is recorded before this owner's private reference to it, so an + // owner that dies in between leaves a held lease that is still visible. + writePrivateRecord( + authority.installation, + "execution-leases", + name, + { leaseId: record.leaseId }, + true, + ); + return handleFor({ + backend: authority.installation.leases, + installation: authority.installation, + leaseId: record.leaseId, + node: null, + owner: authority.execution, + execution: authority.execution, + serviceIds, + authority: options.owner, + seen: record, + }) as ArtifactTargetLease; } -/** Reopen the original owned lease for cancellation/settlement only. This never - * acquires a target, creates a certificate, renews a deadline or permits grants. */ -export function restoreArtifactTargetLeaseForSettlement( - authority: VerifiedExecutionAuthority, -): ArtifactTargetLease { - const a = executionAuthorityState(authority); - const retained = z - .strictObject({ leaseId: uuid, certificateDigest: digest }) - .parse( - readPrivateRecord(a.installation, "execution-leases", `${a.execution.operationKey}.json`), - ); - const s = openState(a.installation.dataRoot, retained.leaseId, a.installation); - try { - const saved = z - .strictObject({ - certificate: signedCertificateSchema, - privateKey: z.string(), - executionReceiptDigest: digest, - }) - .parse(readPrivateRecord(a.installation, "leases", `${retained.leaseId}.json`)); + +export async function delegateArtifactTargetLease( + lease: TargetLeaseAuthority, + child: { + execution: ExecutionRef; + phase: ControlChildPhase; + serviceIds: readonly string[]; + recipient: ControlRecipient; + childPublicKey: string; + evidence?: Record; + }, +): Promise { + const s = stateOf(lease); + if (s.settlementOnly) throw Error("Recovered lease permits settlement only"); + const authority = s.authority; + if (!authority) throw Error("A durable local execution must own child grant publication"); + const owner = executionAuthorityState(authority); + const record = await fresh(s); + if (!record) throw Error("Target lease is not held"); + if (!leaseNodeForward(record, s.node, Date.now()) || Date.now() >= s.execution.deadlineAt) + throw Error("Lease does not permit forward delegation"); + let plan = resolveControlChildPlan(authority, child); + if (plan && canonicalJson(plan.serviceIds) !== canonicalJson(child.serviceIds)) + throw Error("Reserved child target subset changed"); + const parent = s.node === null ? undefined : record.children.find((c) => c.key === s.node); + if (!plan && parent) { + const kinds = childPhaseKinds[child.phase]; if ( - controlDigest(s.execution) !== controlDigest(a.execution) || - retained.certificateDigest !== controlDigest(s.certificateEnvelope) || - controlDigest(saved.certificate) !== retained.certificateDigest || - saved.executionReceiptDigest !== a.receiptDigest || - !sameControlOrigin(a.origin, s.certificate.origin) + parent.permittedFurtherEdges.includes(child.phase) && + kinds.length === 1 && + child.execution.operationKey === + controlChildOperationKey(s.execution.operationKey, child.phase, 0) ) - throw Error("Original retained lease identity changed"); - s.key = createPrivateKey({ - key: Buffer.from(saved.privateKey, "base64url"), - type: "pkcs8", - format: "der", - }); - s.authority = a; - s.verifiedAuthority = authority; - s.settlementOnly = true; - const lease = Object.freeze({ - leaseId: retained.leaseId, - owner: s.execution, - serviceIds: s.certificate.serviceIds, - mode: "owner" as const, - }); - handles.set(lease, s); - return lease; - } catch (error) { - closeState(s); - throw error; + plan = { ...child, kind: kinds[0] as string, ordinal: 0 }; } -} -/** Reopen only the actual prepared owner's already-authorized frozen phase - * issuer. Its ordinary terminal execution still cannot perform a target - * effect; only the existing finite delegation publisher consumes this handle. - * No lease/root/certificate, clock, target or reservation is allocated. */ -export function restoreConfigurationMaintenancePhaseIssuer( - authority: VerifiedExecutionAuthority, - phase: - | "configuration-commit" - | "maintenance-hardlinks" - | "maintenance-recreate" - | "maintenance-handoff", -): ArtifactTargetLease { - const owner = executionAuthorityState(authority); + if (!plan) throw Error("Child is absent from immutable admitted plan"); + if (child.serviceIds.some((id) => !lease.serviceIds.includes(id))) + throw Error("Widened target subset"); + if (parent && (!parent.permittedFurtherEdges.includes(child.phase) || parent.depth >= 2)) + throw Error("Unauthorized descendant edge"); + const edges = verifyFiniteControlEdge( + parent?.recipient ?? owner.installation.installation.role, + owner.kind, + child.phase, + child.recipient, + ); if ( - owner.kind !== "configurationMaintenance.prepare" || - ![ + owner.kind === "configurationMaintenance.prepare" && + [ "configuration-commit", "maintenance-hardlinks", "maintenance-recreate", "maintenance-handoff", - ].includes(phase) + ].includes(child.phase) ) - throw Error("Original authorized maintenance phase issuer required"); - const lease = restoreArtifactTargetLeaseForSettlement(authority), - s = stateOf(lease); - try { - guarded(s, () => - withControlDelegationPublication(authority, phase, () => { - if ( - s.node !== null || - tree(s).root.phase !== "held" || - s.slots?.release || - !owner.reservations.some((value) => value.phase === phase) - ) - throw Error("Original frozen maintenance root is fenced"); - s.settlementOnly = false; - }), + verifyControlPreparedTerminal(owner.installation, s.execution); + const delegationId = + child.recipient === "fixed-cli" || child.recipient === "configuration-helper" + ? fixedControlChildId(owner.installation, child.childPublicKey, s.execution) + : randomUUID(); + const row = { + key: delegationId, + parentKey: s.node, + parent: s.execution, + execution: child.execution, + kind: plan.kind, + phase: child.phase, + recipient: child.recipient, + targets: [...child.serviceIds], + permittedFurtherEdges: edges, + depth: (parent?.depth ?? 0) + 1, + }; + // What the child acts on stays in the store row, out of the header; a + // recipient that needs it reads the store. + const grantDigest = controlDigest(row); + const assertion = encodeControlAssertion({ + installationId: owner.installation.installation.installationId, + leaseId: s.leaseId, + delegationId, + owner: s.owner, + origin: owner.origin, + grant: row, + }); + // The private outbox is written before the store row: a child is attempted + // only once both say so, and recovery reads the outbox first. + withControlDelegationPublication(authority, child.phase, () => { + prepareControlChildPublication(authority, plan, { + delegationId, + grantDigest, + leaseId: s.leaseId, + }); + recordLeaseControlAttempt(authority, { + child: child.execution, + leaseId: s.leaseId, + delegationId, + grantDigest, + }); + writePrivateRecord( + owner.installation, + "attempts", + `${delegationId}.json`, + { grant: row, assertion }, + true, ); - return lease; - } catch (error) { - closeArtifactTargetLeaseHandle(lease); - throw error; - } + }); + remember( + s, + await s.backend.delegate({ + leaseId: s.leaseId, + child: { ...row, ...(child.evidence ? { evidence: child.evidence } : {}) }, + }), + ); + return assertion; +} + +/** Verification-only preview for receivers that can read the store. No adoption or effect authority. */ +export function inspectArtifactDelegation(options: { + assertion: string; + installation: import("./control-authority").ControlInstallation; +}) { + const parsed = parseControlAssertion(options.assertion); + const local = installationState(options.installation); + if (!local.leases.readSync) throw Error("This role cannot read delegations"); + const lease = local.leases.readSync(parsed.leaseId), + grant = lease?.children.find((child) => child.key === parsed.delegationId); + if (!lease || !grant) throw Error("Unanchored delegation"); + const { + state: _state, + adopted: _adopted, + settlement: _settlement, + evidence: _evidence, + ...row + } = grant; + if (controlDigest(row) !== parsed.grantDigest) throw Error("Grant digest mismatch"); + return Object.freeze({ + leaseId: parsed.leaseId, + owner: lease.owner, + parent: grant.parent, + execution: grant.execution, + kind: grant.kind, + serviceIds: Object.freeze([...grant.targets]), + recipient: grant.recipient, + phase: grant.phase, + delegationId: grant.key, + retained: grant.adopted, + }); } -/** Receiver-only settlement after the fixed receiver has locked SQL and - * excluded both this execution and maintenance identity. No admission, effect - * authority or forward handle is created, including for the expired child. */ -export function settleUnadmittedConfigurationPrepare( + +/** The parent that issued a delegation, for a recipient that reads the store. */ +export function verifiedParentForArtifactDelegation( installation: import("./control-authority").ControlInstallation, assertion: string, - input: DataConfigurationFencePrepare, -) { - const local = installationState(installation), - parsed = parseControlAssertion(assertion), - inspected = inspectArtifactDelegation({ dataRoot: local.dataRoot, installation, assertion }); + execution: ExecutionRef, +): VerifiedParentExecution { + const inspected = inspectArtifactDelegation({ assertion, installation }); + if (!sameExecution(inspected.execution, execution)) throw Error("Wrong delegated execution"); + const parent = Object.freeze(retainedControlParent(inspected.parent)) as VerifiedParentExecution; + verifiedParents.add(parent); + verifiedParentOrigins.set(parent, parseControlAssertion(assertion).origin); + return parent; +} +export async function adoptArtifactTargetLease(options: { + assertion: string; + peer: VerifiedControlPeer; + execution: ExecutionRef; + serviceIds: readonly string[]; +}): Promise { + const peer = controlPeerState(options.peer); + if ( + options.assertion !== peer.assertion || + controlDigest(options.execution) !== controlDigest(peer.execution) + ) + throw Error("Peer execution binding mismatch"); + const assertion = parseControlAssertion(options.assertion); + const inspected = inspectArtifactDelegation({ + assertion: options.assertion, + installation: peer.installation.installation, + }); + const lease = peer.installation.leases.readSync?.(assertion.leaseId); + if (!lease) throw Error("Unanchored delegation"); + if ( + !sameExecution(inspected.execution, options.execution) || + inspected.kind !== peer.kind || + inspected.recipient !== peer.recipient || + canonicalJson(inspected.serviceIds) !== canonicalJson(targetSchema.parse(options.serviceIds)) + ) + throw Error("Wrong peer principal, kind, key, fingerprint, or subset"); + const parent = lease.children.find((child) => child.key === inspected.delegationId)?.parentKey; + const issuer = + (parent && lease.children.find((child) => child.key === parent)?.recipient) ?? + lease.owner.executor; if ( - installation.role !== "data-manager" || - local.fixedChildId || - inspected.owner.executor !== "ops-agent" || - inspected.owner.principal !== "api" || - inspected.kind !== "configuration-maintenance.prepare" || - inspected.phase !== "configuration-fence" || - inspected.recipient !== "data-manager" || - controlDigest(inspected.parent) !== controlDigest(inspected.owner) || - controlDigest(input.ownerPrepare) !== controlDigest(inspected.owner) || - canonicalJson(input.serviceIds) !== canonicalJson(inspected.serviceIds) || - controlDigest({ kind: inspected.kind, ...input }) !== - inspected.execution.operationFingerprint || - Date.now() <= Math.max(inspected.execution.deadlineAt, inspected.owner.deadlineAt) + peer.recipient === "ops-agent" && + issuer !== "data-manager" && + !( + issuer === "ops-agent" && + peer.local && + [ + "configuration-commit", + "maintenance-hardlinks", + "maintenance-recreate", + "maintenance-handoff", + ].includes(inspected.phase) + ) ) - throw Error("Original expired configuration preparation required"); - const s = openState(local.dataRoot, inspected.leaseId, local); + throw Error("Wrong peer principal"); + const name = `${inspected.delegationId}.json`; + const pending = { + version: 2, + grantDigest: assertion.grantDigest, + execution: inspected.execution, + state: "pending-adoption", + assertion: options.assertion, + }; + let local: unknown; try { - return guarded(s, () => { - const t = tree(s), - grant = t.grants.get(parsed.delegationId), - child = t.children.get(parsed.delegationId); - if ( - t.root.phase !== "cancel_pending" || - t.grants.size !== 1 || - t.children.size !== 1 || - !grant || - grant.parentNode !== null || - !child || - Object.keys(child.grantDigests).length || - (!child.nonAdmission && (child.phase !== "held" || child.settlement !== null)) - ) - throw Error("Original frozen unadmitted preparation child required"); - const name = `${grant.delegationId}.json`, - raw = readPrivateRecord(local, "adoptions", name, 8192), - adoption = z - .object({ - state: z.literal("ready"), - execution: executionRefSchema, - grantDigest: digest, - assertion: z.string(), - }) - .parse(raw); - if ( - adoption.assertion !== assertion || - adoption.grantDigest !== parsed.grantDigest || - controlDigest(adoption.execution) !== controlDigest(grant.child) - ) - throw Error("Original ready receiver adoption required"); - // This is checked inside the physical journal guard. An admitted worker - // always retains its private execution BEFORE any SQL/guard/effect work. - for (const [namespace, file] of [ - ["executions", `${grant.child.operationKey}.json`], - ["effects", `${grant.child.operationKey}.json`], - ["effect-started", `${grant.child.operationKey}.json`], - ["execution-adoptions", `${grant.child.operationKey}.json`], - ["observations", `terminal-${grant.child.operationKey}.json`], - ]) { - try { - readPrivateRecord(local, namespace!, file!); - throw Error("Admitted or ambiguous receiver preparation remains fenced"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - } - if (child.nonAdmission) { - if ( - child.nonAdmission.payload.adoptionDigest !== controlDigest(raw) || - canonicalJson(child.nonAdmission.payload.input) !== canonicalJson(input) - ) - throw Error("Original negative receiver proof changed"); - return child.nonAdmission; - } - const proof = signControl( - local.privateKey, - CONFIGURATION_PREPARE_NON_ADMISSION_DOMAIN, - configurationPrepareNonAdmissionPayloadSchema.parse({ - version: 1, - kind: "configuration-prepare-not-admitted", - installationId: installation.installationId, - origin: grant.origin, - leaseId: inspected.leaseId, - certificateDigest: inspected.ownerCertificateDigest, - delegationId: grant.delegationId, - grantDigest: parsed.grantDigest, - child: grant.child, - input, - adoptionDigest: controlDigest(raw), - canceledSequence: t.root.sequence, - observedAt: Date.now(), - }), - ); - // Preserve the ready adoption and immutable failed agent canonical. Only - // this exact DM-signed child advances to a negative physical settlement. - writeBounded( - s, - "children", - `${grant.delegationId}.json`, - signControl(local.privateKey, "openmapx-lease-child-v2", { - ...child, - sequence: child.sequence + 1, - phase: "settled", - nonAdmission: proof, - settlement: { - identity: grant.child, - outcome: "failed_without_effect", - publication: "not_started", - terminalReceiptDigest: controlDigest(proof.payload), - }, - }), - ); - tree(s); - return proof; - }); - } finally { - closeState(s); + local = readPrivateRecord(peer.installation, "adoptions", name); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (local) { + const old = z.object({ grantDigest: z.string(), execution: executionRefSchema }).parse(local); + if ( + old.grantDigest !== assertion.grantDigest || + !sameExecution(old.execution, inspected.execution) + ) + throw Error("Durable duplicate admission conflict"); + } else writePrivateRecord(peer.installation, "adoptions", name, pending, true); + const adopted = await peer.installation.leases.adopt({ + leaseId: assertion.leaseId, + key: inspected.delegationId, + execution: inspected.execution, + }); + writePrivateRecord(peer.installation, "adoptions", name, { ...pending, state: "ready" }, false); + return handleFor({ + backend: peer.installation.leases, + installation: peer.installation, + leaseId: assertion.leaseId, + node: inspected.delegationId, + owner: adopted.owner, + execution: inspected.execution, + serviceIds: inspected.serviceIds, + seen: adopted, + }) as BorrowedArtifactTargetLease; } -/** Reads a receiver's signed negative settlement from the exact original tree; - * no sibling private directory, admission or parsed caller proof is trusted. */ -export function readConfigurationPrepareNonAdmission(authority: VerifiedExecutionAuthority) { - const owner = executionAuthorityState(authority); - if ( - owner.kind !== "configurationMaintenance.prepare" || - owner.installation.installation.role !== "ops-agent" - ) - throw Error("Original maintenance owner required"); - const lease = restoreArtifactTargetLeaseForSettlement(authority), - s = stateOf(lease); - try { - return guarded(s, () => { - const t = tree(s), - children = [...t.children.values()], - child = children[0]; - if ( - t.grants.size !== 1 || - children.length !== 1 || - !child?.nonAdmission || - child.phase !== "settled" || - child.settlement?.terminalReceiptDigest !== controlDigest(child.nonAdmission.payload) - ) - throw Error("Original negative receiver settlement required"); - return child.nonAdmission; - }); - } finally { - closeArtifactTargetLeaseHandle(lease); +/** A fixed receiver supplies its own durable execution authority only after adoption. */ +export function bindBorrowedControlExecution( + lease: BorrowedArtifactTargetLease, + authority: VerifiedExecutionAuthority, +): void { + const s = stateOf(lease), + a = executionAuthorityState(authority); + if (!sameExecution(a.execution, lease.execution)) + throw Error("Borrower execution authority mismatch"); + s.authority = authority; +} + +export async function settleArtifactTargetLeaseChild( + lease: TargetLeaseAuthority, + evidence: VerifiedChildSettlement, +): Promise { + const s = stateOf(lease); + const proof = childSettlementState(evidence); + if (!sameExecution(proof.evidence.identity, s.execution)) + throw Error("Foreign settlement identity"); + if (s.node === null && s.authority) { + const owner = executionAuthorityState(s.authority); + if (owner.kind === "configurationMaintenance.prepare") + assertConfigurationMaintenanceRootClosure(s.authority, await observeClosure(s, s.authority)); } + remember( + s, + await s.backend.settle({ leaseId: s.leaseId, key: s.node, settlement: proof.evidence }), + ); } -/** Seal only this original failed producer before asking its recipient for a - * non-admission certificate. This alone is not containment or lease release. */ -export async function prepareFailedServiceLifecycleRecovery( - authority: VerifiedExecutionAuthority, - lease: ArtifactTargetLease, -) { - readFailedServiceHardlinksReceipt(authority); - const state = executionAuthorityState(authority), - s = stateOf(lease); - if (Date.now() < state.execution.deadlineAt) throw Error("Original lifecycle must be expired"); +export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Promise { + const s = stateOf(lease); + if (s.node !== null) throw Error("Borrower cannot release root"); if ( - s.node !== null || - !s.settlementOnly || - controlDigest(s.execution) !== controlDigest(state.execution) + s.authority && + executionAuthorityState(s.authority).kind === "configurationMaintenance.prepare" ) - throw Error("Original settlement-only host lease required"); - await guarded(s, () => { - const t = tree(s); - if ( - t.grants.size !== 1 || - [...t.grants.values()].some( - (g) => - g.parentNode !== null || - g.phase !== "hardlinks" || - g.childKind !== "generation-hardlinks" || - g.recipient !== "data-manager", - ) - ) - throw Error("Original single hardlink grant required"); - if (t.children.size) throw Error("An adopted or unresolved hardlink child remains fenced"); - if (t.root.phase === "settled") throw Error("Original lifecycle already settled"); - if (t.root.phase !== "cancel_pending") { - t.root.phase = "cancel_pending"; - t.root.sequence++; - writeNode(s, t.root); - } - }); - cancelControlExecution(authority); + assertConfigurationMaintenanceRootClosure(s.authority, await observeClosure(s, s.authority)); + await s.backend.release({ leaseId: s.leaseId }); + remember(s, undefined); + if (s.authority) rootReleasedAt(s.authority); } -/** Receiver-side observation of the original canceled, expired host grant. - * No peer, adoption, target handle or forward authority is returned. */ -export function observeUnadoptedServiceHardlinkGrant( - installation: import("./control-authority").ControlInstallation, - assertion: string, -) { - const local = installationState(installation), - parsed = parseControlAssertion(assertion); - const grant = inspectArtifactDelegation({ dataRoot: local.dataRoot, installation, assertion }); + +export async function cancelArtifactTargetLease( + lease: TargetLeaseAuthority, +): Promise { + const s = stateOf(lease); + s.canceled = true; + const record = await s.backend.cancel({ leaseId: s.leaseId, key: s.node }); + remember(s, record); + // These immutable attempted identities are the adapter's cancellation outbox. + return Object.freeze( + (record ? descendantsOf(record, s.node) : []).map((child) => + Object.freeze({ ...child.execution }), + ), + ); +} + +/** Observation/adoption of an old identity is not permission to start another forward effect. */ +export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): void { + const s = stateOf(lease); + if (s.settlementOnly || s.canceled) throw Error("Recovered lease permits settlement only"); + if (s.authority) assertControlExecutionForward(s.authority); + const record = current(s); if ( - installation.role !== "data-manager" || - local.fixedChildId || - grant.owner.executor !== "host-cli" || - grant.owner.principal !== "host-cli" || - grant.kind !== "generation-hardlinks" || - grant.phase !== "hardlinks" || - grant.recipient !== "data-manager" || - controlDigest(grant.parent) !== controlDigest(grant.owner) || - Date.now() < grant.owner.deadlineAt || - Date.now() < grant.execution.deadlineAt + !record || + !leaseNodeForward(record, s.node, Date.now()) || + Date.now() >= s.execution.deadlineAt ) - throw Error("Original expired host hardlink grant required"); - const s = openState(local.dataRoot, grant.leaseId, local); - try { - return guarded(s, () => { - const t = tree(s), - witness = readControlExecutionWitness( - local, - grant.owner, - local.publicKeys[grant.owner.executor], - ); - if ( - !["service.start", "service.update"].includes(witness.kind) || - t.root.phase !== "cancel_pending" || - t.grants.size !== 1 || - t.children.size || - t.root.grantDigests[grant.delegationId] !== parsed.grantDigest - ) - throw Error("Original canceled unadopted lifecycle grant required"); - return generationHardlinksNonAdmissionPayloadSchema.parse({ - version: 1, - kind: "generation-hardlinks-not-admitted", - installationId: installation.installationId, - origin: grant.origin, - leaseId: grant.leaseId, - certificateDigest: grant.ownerCertificateDigest, - delegationId: grant.delegationId, - grantDigest: parsed.grantDigest, - parent: grant.owner, - child: grant.execution, - canceledSequence: t.root.sequence, - }); - }); - } finally { - closeState(s); - } + throw Error("Target lease permits observation/settlement only"); +} +export function assertArtifactTargetLeaseExecution( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): void { + if (!sameExecution(stateOf(lease).execution, execution)) + throw Error("Lifecycle lease execution mismatch"); + assertArtifactTargetLeaseForward(lease); +} +/** Receipt-only evidence: the exact owner may record containment after cancellation. */ +export function assertArtifactTargetLeaseReceiptOwner( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): void { + const s = stateOf(lease); + if (!sameExecution(s.execution, execution)) throw Error("Receipt lease execution mismatch"); + const record = current(s); + const node = record && (s.node === null ? record : record.children.find((c) => c.key === s.node)); + if (!node || node.state === "settled") throw Error("Receipt owner is already settled"); +} +/** The owner that ran the effects; a lease restored for settlement never records them. */ +export function assertArtifactTargetLeaseContainmentOwner( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): void { + assertArtifactTargetLeaseReceiptOwner(lease, execution); + if (stateOf(lease).settlementOnly) throw Error("Recovered lease permits settlement only"); +} +/** Drops the in-process handle. The lease, its children and every unknown fence remain. */ +export function closeArtifactTargetLeaseHandle(lease: TargetLeaseAuthority): void { + if (!handles.delete(lease)) throw Error("Unverified target lease handle"); +} + +const executionLeaseSchema = z.strictObject({ + leaseId: uuid, + releasedAt: z.number().int().positive().safe().optional(), +}); +function executionLease(authority: VerifiedExecutionAuthority) { + const a = executionAuthorityState(authority); + return executionLeaseSchema.parse( + readPrivateRecord(a.installation, "execution-leases", `${a.execution.operationKey}.json`), + ); +} +function executionLeaseId(authority: VerifiedExecutionAuthority): string { + return executionLease(authority).leaseId; +} +/** The time the owner first saw its root released, recorded once in its own state. */ +function rootReleasedAt(authority: VerifiedExecutionAuthority): number { + const a = executionAuthorityState(authority), + saved = executionLease(authority); + if (saved.releasedAt) return saved.releasedAt; + const releasedAt = Date.now(); + writePrivateRecord( + a.installation, + "execution-leases", + `${a.execution.operationKey}.json`, + { leaseId: saved.leaseId, releasedAt }, + false, + ); + return releasedAt; } -/** Only the original failed host producer may revoke its one never-adopted - * hardlink grant. Adoption and revocation share the physical journal guard. - * No recipient key, completion, SQL terminal or successful update is invented. */ -export async function revokeUnadoptedServiceHardlinks( +/** Reopen the original owned lease for cancellation and settlement only. */ +export function restoreArtifactTargetLeaseForSettlement( authority: VerifiedExecutionAuthority, - lease: ArtifactTargetLease, - nonAdmission?: unknown, -): Promise { - if (nonAdmission === undefined) throw Error("Data Manager non-admission proof required"); - const receipt = readFailedServiceHardlinksReceipt(authority), - state = executionAuthorityState(authority), - s = stateOf(lease); - if (Date.now() < state.execution.deadlineAt) throw Error("Original lifecycle must be expired"); - if ( - s.node !== null || - controlDigest(s.execution) !== controlDigest(state.execution) || - !s.settlementOnly - ) - throw Error("Original settlement-only host lease required"); - await guarded(s, () => { - const t = tree(s); - if ( - t.grants.size !== 1 || - [...t.grants.values()].some( - (grant) => - grant.parentNode !== null || - grant.childKind !== "generation-hardlinks" || - grant.phase !== "hardlinks" || - grant.recipient !== "data-manager", - ) - ) - throw Error("Original single hardlink grant required"); - if ([...t.children.values()].some((child) => !child.unadoptedRevocation)) - throw Error("An adopted hardlink child remains unresolved"); - if (t.root.phase !== "cancel_pending") throw Error("Original lifecycle must be canceled first"); - const proof = verifyControl( - s.installation.publicKeys["data-manager"], - GENERATION_HARDLINKS_NON_ADMISSION_DOMAIN, - nonAdmission, - generationHardlinksNonAdmissionPayloadSchema, - ); - const [originalGrant] = t.grants.values(); - if ( - !originalGrant || - proof.installationId !== s.certificate.installationId || - proof.leaseId !== s.certificate.leaseId || - proof.certificateDigest !== controlDigest(s.certificateEnvelope) || - proof.delegationId !== originalGrant.delegationId || - proof.grantDigest !== t.root.grantDigests[originalGrant.delegationId] || - controlDigest(proof.parent) !== controlDigest(s.execution) || - controlDigest(proof.child) !== controlDigest(originalGrant.child) || - controlDigest(proof.origin) !== controlDigest(originalGrant.origin) || - proof.canceledSequence !== t.root.sequence - ) - throw Error("Data Manager non-admission proof does not match original grant"); - for (const [id, grant] of t.grants) { - if (t.children.has(id)) { - if (t.children.get(id)?.failedLifecycleReceiptDigest !== controlDigest(receipt)) - throw Error("Original failed lifecycle receipt changed"); - continue; - } - const revoked = unadoptedLifecycleSchema.parse({ - version: 1, - origin: grant.origin, - leaseId: s.certificate.leaseId, - certificateDigest: controlDigest(s.certificateEnvelope), - delegationId: id, - grantDigest: t.root.grantDigests[id], - parent: s.execution, - child: grant.child, - failedReceiptDigest: controlDigest(receipt), - canceledSequence: t.root.sequence, - nonAdmission, - }); - writeBounded( - s, - "children", - `${id}.json`, - { unadoptedRevocation: signControl(s.key, unadoptedLifecycleDomain, revoked) }, - true, - ); - } - tree(s); - }); +): ArtifactTargetLease { + const a = executionAuthorityState(authority); + return handleFor({ + backend: a.installation.leases, + installation: a.installation, + leaseId: executionLeaseId(authority), + node: null, + owner: a.execution, + execution: a.execution, + serviceIds: a.serviceIds, + authority, + settlementOnly: true, + }) as ArtifactTargetLease; } -/** The effect producer re-reads the actual signed failure and complete shared - * tree; a caller-provided HTTP status or observation is never containment. - * A failure whose tree holds no grant issued nothing that could outlive it. - * Once a grant exists, only its recorded unadopted revocation contains it. */ -export function verifyFailedServiceLifecycleContainment(authority: VerifiedExecutionAuthority) { - const state = executionAuthorityState(authority); - const reference = z - .strictObject({ leaseId: uuid, certificateDigest: digest }) - .parse( - readPrivateRecord( - state.installation, - "execution-leases", - `${state.execution.operationKey}.json`, - ), - ); - const s = openState(state.installation.dataRoot, reference.leaseId, state.installation, true); - try { - const t = tree(s); - if ( - controlDigest(s.certificateEnvelope) !== reference.certificateDigest || - controlDigest(s.execution) !== controlDigest(state.execution) - ) - throw Error("Original unadopted hardlink revocation required"); - if (t.grants.size === 0 && t.children.size === 0) { - const failure = readFailedServiceLifecycleReceipt(authority); - return { - failedReceiptDigest: controlDigest(failure), - revisionId: failure.finalRevisionId, - leaseId: reference.leaseId, - }; - } - const receipt = readFailedServiceHardlinksReceipt(authority), - failedReceiptDigest = controlDigest(receipt); - if ( - t.root.phase === "held" || - t.grants.size !== 1 || - t.children.size !== 1 || - [...t.children.values()].some( - (child) => - !child.unadoptedRevocation || child.failedLifecycleReceiptDigest !== failedReceiptDigest, - ) - ) - throw Error("Original unadopted hardlink revocation required"); - return { - failedReceiptDigest, - revisionId: receipt.finalRevisionId, - leaseId: reference.leaseId, - }; - } finally { - closeState(s); - } +/** Whether the execution's own root lease is gone from the store, after its own effect settled. */ +export async function verifyRetainedArtifactTargetRootRelease( + authority: VerifiedExecutionAuthority, +): Promise> { + const a = executionAuthorityState(authority); + const leaseId = executionLeaseId(authority); + childSettlementState(verifyControlEffectSettlement(a.installation.installation, a.execution)); + if (await a.installation.leases.read(leaseId)) throw Error("Original root release is not proved"); + return Object.freeze({ releasedAt: rootReleasedAt(authority) }); +} +/** + * The same without waiting: from the release this owner recorded after the + * store dropped its lease, or from the store for a role that reads it. + */ +export function verifyLocalArtifactTargetRootRelease( + authority: VerifiedExecutionAuthority, +): Readonly<{ releasedAt: number }> { + const a = executionAuthorityState(authority); + const leaseId = executionLeaseId(authority); + childSettlementState(verifyControlEffectSettlement(a.installation.installation, a.execution)); + const recorded = executionLease(authority).releasedAt; + if (recorded) return Object.freeze({ releasedAt: recorded }); + if (!a.installation.leases.readSync) throw Error("This role cannot read the lease store"); + if (a.installation.leases.readSync(leaseId)) throw Error("Original root release is not proved"); + return Object.freeze({ releasedAt: rootReleasedAt(authority) }); } + +/** Settles the owner from recovery evidence and releases it when every child is known. */ export async function reconcileArtifactTargetLease(options: { - dataRoot: string; leaseId: string; authority: VerifiedRecoveryAuthority; }): Promise<"released" | "fenced"> { const recovery = recoveryAuthorityState(options.authority); - const s = openState(options.dataRoot, options.leaseId, recovery.authority.installation, true); - try { - if (controlDigest(s.execution) !== controlDigest(recovery.authority.execution)) - throw Error("Foreign recovery execution"); - const saved = z - .object({ - certificate: signedCertificateSchema, - privateKey: z.string(), - executionReceiptDigest: digest, - }) - .parse(readPrivateRecord(s.installation, "leases", `${options.leaseId}.json`)); - if ( - controlDigest(saved.certificate) !== controlDigest(s.certificateEnvelope) || - saved.executionReceiptDigest !== recovery.authority.receiptDigest - ) - throw Error("Recovery certificate mismatch"); - s.key = createPrivateKey({ - key: Buffer.from(saved.privateKey, "base64url"), - type: "pkcs8", - format: "der", - }); - if (recovery.authority.kind === "configurationMaintenance.prepare") { - s.authority = recovery.authority; - s.verifiedAuthority = restoreAdmittedControlExecution( - s.installation.installation, - recovery.authority.execution, - ); - } - const t = tree(s); - if ( - !knownOutcome(recovery.evidence) || - [...t.grants.keys()].some( - (id) => - t.children.get(id)?.phase !== "settled" || !knownOutcome(t.children.get(id)?.settlement), - ) - ) - return "fenced"; - if (s.slots?.release) { - finishOwnedRelease(s); - return "released"; - } - await guarded(s, () => { - if (s.authority?.kind === "configurationMaintenance.prepare") { - if (!s.verifiedAuthority) throw Error("Original maintenance recovery authority required"); - assertMaintenanceClosureFromTree(s, s.verifiedAuthority); - } - const root = readRoot(s); - root.settlement = recovery.evidence; - root.phase = "settled"; - root.sequence++; - writeNode(s, root); - }); - const lease = Object.freeze({ - leaseId: options.leaseId, - owner: s.execution, - serviceIds: s.certificate.serviceIds, - mode: "owner" as const, - }); - handles.set(lease, s); - await releaseArtifactTargetLease(lease); - return "released"; - } finally { - closeState(s); + const backend = recovery.authority.installation.leases; + const lease = await backend.read(options.leaseId); + if (!lease) return "released"; + if (!sameExecution(lease.owner, recovery.authority.execution)) + throw Error("Foreign recovery execution"); + if (!knownSettlement(recovery.evidence) || lease.children.some((child) => !settledKnown(child))) + return "fenced"; + if (recovery.authority.kind === "configurationMaintenance.prepare") { + const verified = restoreAdmittedControlExecution( + recovery.authority.installation.installation, + recovery.authority.execution, + ); + assertConfigurationMaintenanceRootClosure(verified, closureFromRecord(lease, verified)); } + if (lease.state !== "settled") + await backend.settle({ leaseId: lease.leaseId, key: null, settlement: recovery.evidence }); + await backend.release({ leaseId: lease.leaseId }); + rootReleasedAt( + restoreAdmittedControlExecution( + recovery.authority.installation.installation, + recovery.authority.execution, + ), + ); + return "released"; } -/** Finish only this original unused root, including its existing partial - * release cursor. No acquisition, forward authority, new clock or caller ID. */ +/** Finish only this original unused root. No acquisition, forward authority, new clock or caller ID. */ export async function reconcileConfigurationPreEffectRoot( authority: VerifiedExecutionAuthority, ): Promise { inspectConfigurationPreEffectRefusal(authority); - const state = executionAuthorityState(authority), - name = `${state.execution.operationKey}.json`; - let raw: unknown; + const state = executionAuthorityState(authority); + let leaseId: string; try { - raw = readPrivateRecord(state.installation, "execution-leases", name); + leaseId = executionLeaseId(authority); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT" && state.kind === "service.pull") return; throw error; } - if (raw === null && state.kind === "service.pull") return; if (state.kind === "service.pull") - throw Error("Normal coordinator pull cannot own a physical target root"); - const reference = z.strictObject({ leaseId: uuid, certificateDigest: digest }).parse(raw); - const ownerName = `${reference.leaseId}.json`, - saved = readPrivateRecord(state.installation, "leases", ownerName); - if ( - controlDigest((saved as { certificate: unknown }).certificate) !== reference.certificateDigest - ) - throw Error("Original pre-effect root certificate changed"); - try { - verifyRetainedArtifactTargetRootRelease(authority); - return; - } catch { - /* Only the existing exact recovery path may finish a partial release. */ - } + throw Error("Normal coordinator pull cannot own a target lease"); if ( (await reconcileArtifactTargetLease({ - dataRoot: state.installation.dataRoot, - leaseId: reference.leaseId, + leaseId, authority: verifyControlRecoveryAuthority(authority), })) !== "released" ) throw Error("Original pre-effect root remains fenced"); - verifyRetainedArtifactTargetRootRelease(authority); } -/** Verification-only preview for fixed admission adapters. No adoption or effect authority. */ -export function inspectArtifactDelegation(options: { - dataRoot: string; - assertion: string; - installation: import("./control-authority").ControlInstallation; -}) { - const parsed = parseControlAssertion(options.assertion); - const s = openState(options.dataRoot, parsed.leaseId, installationState(options.installation)); - try { - const t = tree(s); - const grant = t.grants.get(parsed.delegationId); - if (!grant || parsed.ownerCertificateDigest !== controlDigest(s.certificateEnvelope)) - throw Error("Unanchored delegation"); - const parent = grant.parentNode - ? required(t.grants.get(grant.parentNode), "Missing parent grant") - : undefined; - const parentKey = parent?.childPublicKey ?? s.certificate.ownerPublicKey; - const { signature, ...fields } = parsed; - verifyControl( - parentKey, - "openmapx-lease-assertion-v1", - { payload: fields, signature }, - assertionSchema.omit({ signature: true }), + +/** Image pulls hold no lease, but never run while their target is held. */ +export function assertCoordinatorPullLeaseCompatibility( + authority: VerifiedExecutionAuthority, +): void { + assertImagePullTargetFree(authority, "service.pull"); +} +export function assertImageStageLeaseCompatibility(authority: VerifiedExecutionAuthority): void { + assertImagePullTargetFree(authority, "service.image.stage"); +} +function assertImagePullTargetFree( + authority: VerifiedExecutionAuthority, + kind: "service.pull" | "service.image.stage", +): void { + const state = executionAuthorityState(authority), + raw = readPrivateRecord( + state.installation, + "executions", + `${state.execution.operationKey}.json`, + 65536, + ), + admitted = verifyControl( + state.installation.installation.publicKey, + "openmapx-control-execution-v2", + raw, + z + .object({ + kind: z.literal(kind), + execution: executionRefSchema, + operation: opsOperationSchema, + }) + .passthrough(), ); - const parentNode = grant.parentNode - ? required(t.children.get(grant.parentNode), "Missing parent adoption") - : t.root; - if (parentNode.grantDigests[grant.delegationId] !== parsed.grantDigest) - throw Error("Grant digest mismatch"); - return Object.freeze({ - leaseId: parsed.leaseId, - ownerCertificateDigest: controlDigest(s.certificateEnvelope), - owner: s.certificate.owner, - origin: grant.origin, - parent: grant.parent, - execution: grant.child, - kind: grant.childKind, - serviceIds: Object.freeze([...grant.serviceIds]), - recipient: grant.recipient, - phase: grant.phase, - delegationId: grant.delegationId, - retained: !!t.children.get(grant.delegationId), - }); - } finally { - closeState(s); + if ( + state.installation.fixedChildId || + state.kind !== kind || + controlDigest(raw) !== state.receiptDigest || + !sameExecution(admitted.execution, state.execution) || + admitted.operation.kind !== kind || + !("serviceId" in admitted.operation) || + (admitted.operation.kind === "service.pull" && + (admitted.operation.serviceId !== "data-manager" || !admitted.operation.candidateId)) || + opsOperationFingerprint(admitted.operation) !== state.execution.operationFingerprint + ) + throw Error("Actual normal coordinator pull required"); + const target = admitted.operation.serviceId; + if (!state.installation.registeredTargets.has(target)) + throw Error("Registered image target required"); + const list = state.installation.leases.listSync; + if (!list) throw Error("Image pulls run in the agent"); + for (const lease of list()) + if (lease.targets.includes(target)) throw Error("Coordinator target is held"); +} + +/** Complete only the original terminal handoff preparer after its helper has committed and settled. */ +export async function settlePreparedConfigurationHandoff( + authority: VerifiedExecutionAuthority, +): Promise { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "configurationHandoff.prepare" || + owner.installation.fixedChildId || + owner.installation.installation.role !== "ops-agent" + ) + throw Error("Original handoff preparer required"); + const admitted = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: unknown } }, + operation = opsOperationSchema.parse(admitted.payload.operation); + if (operation.kind !== "configurationHandoff.prepare") + throw Error("Original handoff operation required"); + const source = readCompletedConfigurationHandoff( + owner.installation.installation, + operation.handoffId, + ); + const lease = await owner.installation.leases.read(source.intent.leaseId); + if (!lease) throw Error("Original prepared helper tree changed"); + const parent = lease.children.filter((child) => sameExecution(child.execution, owner.execution)), + helper = lease.children.find((child) => child.key === source.intent.delegationId); + if ( + parent.length !== 1 || + !sameExecution(source.intent.preparer, owner.execution) || + canonicalJson(source.intent.operation) !== canonicalJson(operation) || + source.intent.preparerAdmissionDigest !== owner.receiptDigest || + !sameExecution(source.intent.ownerPrepare, lease.owner) || + helper?.parentKey !== parent[0]!.key || + helper.recipient !== "configuration-helper" || + helper.phase !== "configuration-handoff" || + helper.kind !== "configuration-helper" || + !sameExecution(helper.execution, source.intent.helper) || + helper.state !== "settled" || + canonicalJson(helper.settlement) !== canonicalJson(source.settlement) + ) + throw Error("Original prepared helper tree changed"); + verifyControlPreparedTerminal(owner.installation, owner.execution); + const proof = childSettlementState( + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ).evidence; + if (proof.outcome !== "committed" || proof.publication !== "committed") + throw Error("Original prepared handoff effect required"); + const prior = parent[0]!; + if (prior.state === "settled") { + if (canonicalJson(prior.settlement) !== canonicalJson(proof)) + throw Error("Original handoff settlement changed"); + return; + } + await owner.installation.leases.settle({ + leaseId: lease.leaseId, + key: prior.key, + settlement: proof, + }); +} +/** Receipt-only settlement of the original completed helper. */ +export async function settleRetainedConfigurationHandoffHelper( + authority: VerifiedExecutionAuthority, +): Promise { + const owner = executionAuthorityState(authority); + if (owner.kind !== "configuration-helper" || !owner.installation.fixedChildId) + throw Error("Original completed fixed helper required"); + const raw = readPrivateRecord( + owner.installation, + "executions", + `${owner.execution.operationKey}.json`, + 65536, + ) as { payload: { operation: { handoffId?: unknown } } }, + source = readCompletedConfigurationHandoff( + owner.installation.installation, + uuid.parse(raw.payload.operation.handoffId), + ), + adoption = z + .object({ + state: z.literal("ready"), + execution: executionRefSchema, + assertion: z.string(), + grantDigest: z.string(), + }) + .parse( + readPrivateRecord( + owner.installation, + "adoptions", + `${source.intent.delegationId}.json`, + 8192, + ), + ), + assertion = parseControlAssertion(adoption.assertion); + if ( + !sameExecution(source.intent.helper, owner.execution) || + source.intent.delegationId !== owner.installation.fixedChildId || + adoption.assertion !== source.intent.assertion || + !sameExecution(adoption.execution, owner.execution) || + assertion.leaseId !== source.intent.leaseId || + assertion.delegationId !== source.intent.delegationId + ) + throw Error("Original helper ready adoption changed"); + const lease = await owner.installation.leases.read(source.intent.leaseId), + child = lease?.children.find((value) => value.key === source.intent.delegationId); + if ( + !lease || + !child || + child.recipient !== "configuration-helper" || + child.kind !== "configuration-helper" || + !sameExecution(child.execution, owner.execution) + ) + throw Error("Original retained helper tree changed"); + if (child.state === "settled") { + if (canonicalJson(child.settlement) !== canonicalJson(source.settlement)) + throw Error("Original helper settlement changed"); + return; } + verifyControlPreparedTerminal(owner.installation, source.intent.preparer); + await owner.installation.leases.settle({ + leaseId: lease.leaseId, + key: child.key, + settlement: childSettlementState( + verifyControlEffectSettlement(owner.installation.installation, owner.execution), + ).evidence, + }); } -/** The delegation's root lease is still held by its owner: not released and - * not settled. Observation only; it returns no lease or capability. */ -export function assertArtifactDelegationRootHeld( - installation: import("./control-authority").ControlInstallation, - assertion: string, -): void { - const local = installationState(installation), - parsed = parseControlAssertion(assertion), - state = openState(local.dataRoot, parsed.leaseId, local); - try { - guarded(state, () => { - if (tree(state).root.phase === "settled" || state.slots?.release) - throw Error("Original owner lease is no longer held"); - }); - } finally { - closeState(state); - } +function childSettlementOf( + record: LeaseRecord | undefined, + node: string | null, + execution: ExecutionRef, +): ChildSettlement | null { + if (!record) throw Error("Target lease is not held"); + const entries = leaseChildrenOf(record, node).filter((child) => + sameExecution(child.execution, execution), + ); + if (entries.length !== 1) throw Error("Exact retained child grant required"); + const child = entries[0]!; + return settledKnown(child) ? (child.settlement as ChildSettlement) : null; } -export async function cancelArtifactTargetLease( +/** The settlement of the exact child `execution` issued by this lease node, if it is known. */ +export async function inspectArtifactTargetLeaseChildSettlement( lease: TargetLeaseAuthority, -): Promise { + execution: ExecutionRef, +): Promise { const s = stateOf(lease); - return guarded(s, () => { - const t = tree(s); - const node = - s.node === null ? t.root : required(t.children.get(s.node), "Missing retained child node"); - if (node.phase !== "settled") { - node.phase = "cancel_pending"; - node.sequence++; - writeNode(s, node); - tree(s); - } - // These immutable attempted identities are the adapter's cancellation - // outbox, including lost admissions. No descendant evidence is released. - return Object.freeze( - [...t.grants.values()] - .filter((grant) => { - let parent = grant.parentNode; - if (s.node === null) return true; - while (parent !== null) { - if (parent === s.node) return true; - parent = t.grants.get(parent)?.parentNode ?? null; - } - return false; - }) - .map((grant) => Object.freeze({ ...grant.child })), - ); - }); + return childSettlementOf(await fresh(s), s.node, execution); } - -/** A helper can adopt before its preparer exits, but may not enter drain/writer work before terminal preparation. */ -export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLease): void { +/** The same, for a role that reads the store itself. */ +export function inspectLocalLeaseChildSettlement( + lease: TargetLeaseAuthority, + execution: ExecutionRef, +): ChildSettlement | null { const s = stateOf(lease); - const t = tree(s); - const grant = s.node === null ? undefined : t.grants.get(s.node); + if (!s.backend.readSync) throw Error("This role cannot read the lease store"); + return childSettlementOf(current(s), s.node, execution); +} +/** + * Settles the data-manager child `execution` of this lease node from the + * terminal status data-manager reported to this role over its authenticated + * route. Only a terminal status with a known publication settles it. + */ +export async function settleDataManagerLeaseChild( + lease: TargetLeaseAuthority, + execution: ExecutionRef, + status: import("../services/data-manager/contracts/executions").DataExecutionStatus, +): Promise { + const s = stateOf(lease); + const record = await fresh(s); + const child = record + ? leaseChildrenOf(record, s.node).find((value) => sameExecution(value.execution, execution)) + : undefined; + if (!record || !child) throw Error("Exact retained child grant required"); + if (settledKnown(child)) return child.settlement as ChildSettlement; if ( - grant?.recipient !== "configuration-helper" || - grant.phase !== "configuration-handoff" || - !forwardAncestry(t, s.node) || - Date.now() >= grant.child.deadlineAt + !["ok", "partial", "error", "canceled", "interrupted"].includes(status.status) || + status.publication === "unknown" ) - throw Error("Configuration helper cannot enter drain"); - verifyControlPreparedTerminal(s.installation, grant.parent); + return null; + const observed = recordDataControlTerminalObservation( + s.installation.installation, + execution, + status, + ); + return settleIssuedLeaseChild(lease, execution, { + outcome: + status.publication === "not_started" + ? "failed_without_effect" + : status.status === "ok" + ? "committed" + : "contained", + publication: status.publication, + terminalReceiptDigest: observed.terminalDigest, + }); } - -/** Observation/adoption of an old identity is not permission to start another forward effect. */ -export function assertArtifactTargetLeaseForward(lease: TargetLeaseAuthority): void { +/** + * Settles the child `execution` this lease node issued, from the answer its + * recipient gave to this role over an authenticated route. A recorded + * settlement never changes. + */ +export async function settleIssuedLeaseChild( + lease: TargetLeaseAuthority, + execution: ExecutionRef, + evidence: Omit, +): Promise { const s = stateOf(lease); - if (s.settlementOnly) throw Error("Recovered lease permits settlement only"); - if (s.verifiedAuthority) assertControlExecutionForward(s.verifiedAuthority); - const t = tree(s); - const node = s.node === null ? t.root : t.children.get(s.node); - if (!node || !forwardAncestry(t, s.node) || Date.now() >= s.execution.deadlineAt) - throw Error("Target lease permits observation/settlement only"); + const record = await fresh(s); + const child = record + ? leaseChildrenOf(record, s.node).find((value) => sameExecution(value.execution, execution)) + : undefined; + if (!record || !child) throw Error("Exact retained child grant required"); + if (child.settlement && child.state === "settled") { + if ( + child.settlement.outcome !== evidence.outcome || + child.settlement.publication !== evidence.publication + ) + throw Error("Original child settlement changed"); + return child.settlement as ChildSettlement; + } + const settlement: ChildSettlement = { identity: execution, ...evidence }; + remember(s, await s.backend.settle({ leaseId: s.leaseId, key: child.key, settlement })); + return settlement; +} +/** The settlement of an original owned child, read through the owner's lease reference. */ +export async function inspectRetainedOwnedArtifactTargetChildSettlement( + authority: VerifiedExecutionAuthority, + execution: ExecutionRef, +): Promise { + const lease = restoreArtifactTargetLeaseForSettlement(authority); + try { + return await inspectArtifactTargetLeaseChildSettlement(lease, execution).catch( + (error: unknown) => { + if ((error as Error).message === "Target lease is not held") return null; + throw error; + }, + ); + } finally { + closeArtifactTargetLeaseHandle(lease); + } } -/** Fixed catalog-store adapter; caller DTOs cannot construct physical ownership. */ +/** Terminal-only check for a recovered catalog activation: every child settled. */ +export async function assertRetainedCatalogActivationChildrenSettled( + authority: VerifiedExecutionAuthority, +): Promise { + const owner = executionAuthorityState(authority); + if (owner.kind !== "catalog.activate" || owner.installation.installation.role !== "data-manager") + throw Error("Original catalog activation owner required"); + const lease = await owner.installation.leases.read(executionLeaseId(authority)); + if (lease && lease.children.some((child) => !settledKnown(child))) + throw Error("Catalog child tree remains unsettled"); +} + +/** Holder of MOTIS for catalog pins: the agent or the standalone host CLI. */ export function catalogPinTargetOwner(lease: TargetLeaseAuthority) { assertArtifactTargetLeaseForward(lease); const s = stateOf(lease); - if (!s.certificate.serviceIds.includes("motis")) + if (!s.serviceIds.includes("motis")) throw Error("MOTIS target authority required for catalog pins"); if (!["ops-agent", "host-cli"].includes(s.installation.installation.role)) throw Error("Catalog pins are owned by the agent or standalone host CLI"); return { installation: s.installation.installation, execution: s.execution, - origin: s.certificate.origin, + origin: s.authority ? executionAuthorityState(s.authority).origin : undefined, }; } -export function assertArtifactTargetLeaseExecution( - lease: TargetLeaseAuthority, - execution: ExecutionRef, -): void { - const state = stateOf(lease); - if (controlDigest(state.execution) !== controlDigest(execution)) - throw Error("Lifecycle lease execution mismatch"); - assertArtifactTargetLeaseForward(lease); -} -/** Receipt-only evidence: the exact physical owner may record containment after cancellation. - * This does not authorize a forward effect, delegation, acquisition or release. */ -export function assertArtifactTargetLeaseReceiptOwner( - lease: TargetLeaseAuthority, - execution: ExecutionRef, -): void { +/** A helper can adopt before its preparer exits, but may not drain before terminal preparation. */ +export function verifyConfigurationHelperReady(lease: BorrowedArtifactTargetLease): void { const s = stateOf(lease); - if (controlDigest(s.execution) !== controlDigest(execution)) - throw Error("Receipt lease execution mismatch"); - const t = tree(s), - node = s.node === null ? t.root : t.children.get(s.node); - if (!node || node.phase === "settled") throw Error("Receipt owner is already settled"); + const record = current(s); + const grant = record?.children.find((child) => child.key === s.node); + if ( + !record || + grant?.recipient !== "configuration-helper" || + grant.phase !== "configuration-handoff" || + !leaseNodeForward(record, s.node, Date.now()) + ) + throw Error("Configuration helper cannot enter drain"); + verifyControlPreparedTerminal(s.installation, grant.parent); } - -/** The physical owner that ran the effects; a lease restored for settlement never records them. */ -export function assertArtifactTargetLeaseContainmentOwner( - lease: TargetLeaseAuthority, - execution: ExecutionRef, +/** The delegation's root lease is still held by its owner. Observation only. */ +export function assertArtifactDelegationRootHeld( + installation: import("./control-authority").ControlInstallation, + assertion: string, ): void { - assertArtifactTargetLeaseReceiptOwner(lease, execution); - if (stateOf(lease).settlementOnly) throw Error("Recovered lease permits settlement only"); -} - -/** Drop local descriptors only. Durable locks, ownership and all unknown fences remain untouched. */ -export function closeArtifactTargetLeaseHandle(lease: TargetLeaseAuthority): void { - const s = handles.get(lease); - if (!s) throw Error("Unverified target lease handle"); - closeState(s); - handles.delete(lease); + const local = installationState(installation), + parsed = parseControlAssertion(assertion), + lease = local.leases.readSync?.(parsed.leaseId); + if (!lease || lease.state === "settled") throw Error("Original owner lease is no longer held"); } - -/** Read-only terminal evidence after root release: never mints a reusable lease handle. */ +/** Read-only terminal evidence of a borrowed child after the agent restarted. */ export function verifyRetainedArtifactTargetSettlement( installation: import("./control-authority").ControlInstallation, adoption: import("./control-authority").ControlAdoption, execution: ExecutionRef, ): void { const local = installationState(installation); - const s = openState(local.dataRoot, adoption.leaseId, local, true); + const lease = local.leases.readSync?.(adoption.leaseId); + const child = lease?.children.find((value) => value.key === adoption.delegationId); + const proof = childSettlementState(verifyControlEffectSettlement(installation, execution)); + if ( + lease && + (!child || + !settledKnown(child) || + !sameExecution(child.execution, execution) || + canonicalJson(child.settlement) !== canonicalJson(proof.evidence)) + ) + throw Error("Terminal child settlement is not proved"); +} + +declare const maintenanceClosureBrand: unique symbol; +export type ConfigurationMaintenanceClosureObservation = Readonly<{ + [maintenanceClosureBrand]: true; +}>; +const maintenanceClosureObservations = new WeakMap< + ConfigurationMaintenanceClosureObservation, + { + authority: VerifiedExecutionAuthority; + leaseId: string; + children: Array<{ execution: ExecutionRef; settlement: ChildSettlement }>; + } +>(); +export function configurationMaintenanceClosureObservationState( + value: ConfigurationMaintenanceClosureObservation, + authority: VerifiedExecutionAuthority, +) { + const source = maintenanceClosureObservations.get(value); + if (!source || source.authority !== authority) + throw Error("Actual original target-tree closure observation required"); + return source; +} +function closureFromRecord( + lease: LeaseRecord, + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceClosureObservation { + const children = lease.children.map((child) => { + if (!settledKnown(child)) + throw Error("Every original maintenance descendant must be contained"); + return { execution: child.execution, settlement: child.settlement as ChildSettlement }; + }); + const observation = Object.freeze({}) as ConfigurationMaintenanceClosureObservation; + maintenanceClosureObservations.set(observation, { + authority, + leaseId: lease.leaseId, + children, + }); + return observation; +} +async function observeClosure(s: State, authority: VerifiedExecutionAuthority) { + const record = await fresh(s); + if (!record) throw Error("Original maintenance root is not held"); + return closureFromRecord(record, authority); +} +function maintenanceLease(authority: VerifiedExecutionAuthority): LeaseRecord { + const owner = executionAuthorityState(authority); + if ( + owner.kind !== "configurationMaintenance.prepare" || + owner.installation.installation.role !== "ops-agent" || + owner.installation.fixedChildId + ) + throw Error("Original maintenance runtime tree required"); + const lease = owner.installation.leases.readSync?.(executionLeaseId(authority)); + if (!lease || !sameExecution(lease.owner, owner.execution)) + throw Error("Original maintenance root is not held"); + return lease; +} +/** Read-only observation of the original maintenance tree; it creates no lease or proof. */ +export function observeConfigurationMaintenanceRuntimeTree( + authority: VerifiedExecutionAuthority, +): ConfigurationMaintenanceClosureObservation { + return closureFromRecord(maintenanceLease(authority), authority); +} +export function verifyConfigurationMaintenanceRootClosure( + authority: VerifiedExecutionAuthority, +): void { + // The store releases a maintenance root only after this check passed, and the + // owner records the release; the released tree is no longer kept. + const state = executionAuthorityState(authority); + if ( + !state.installation.leases.readSync?.(executionLeaseId(authority)) && + executionLease(authority).releasedAt + ) + return; + const observation = observeConfigurationMaintenanceRuntimeTree(authority); try { - const t = tree(s); - const child = t.children.get(adoption.delegationId); - const proof = childSettlementState(verifyControlEffectSettlement(installation, execution)); - if ( - child?.phase !== "settled" || - !knownOutcome(child.settlement) || - controlDigest(child.execution) !== controlDigest(execution) || - controlDigest(child.settlement) !== controlDigest(proof.evidence) - ) - throw Error("Terminal child settlement is not proved"); + assertConfigurationMaintenanceRootClosure(authority, observation); } finally { - closeState(s); + maintenanceClosureObservations.delete(observation); + } +} +/** Source-only observation of the original held maintenance root. */ +export function configurationMaintenanceTargetOwner(lease: ArtifactTargetLease) { + const s = stateOf(lease); + const record = current(s); + const owner = s.authority && executionAuthorityState(s.authority); + if ( + s.node !== null || + !record || + !owner || + s.installation.installation.role !== "ops-agent" || + owner.kind !== "configurationMaintenance.prepare" || + record.state === "settled" + ) + throw Error("Held original maintenance target owner required"); + return Object.freeze({ + installation: s.installation.installation, + execution: s.execution, + origin: owner.origin, + admissionDigest: owner.receiptDigest, + leaseId: s.leaseId, + serviceIds: record.targets, + }); +} +function maintenanceOwnerScope( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + message: string, +) { + const s = stateOf(lease), + owner = executionAuthorityState(authority); + if ( + s.node !== null || + owner.kind !== "configurationMaintenance.prepare" || + !sameExecution(owner.execution, s.execution) + ) + throw Error(message); + return { s, owner }; +} +export function withConfigurationMaintenanceOwnerInitialization( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const { s, owner } = maintenanceOwnerScope( + lease, + authority, + "Original live maintenance initialization required", + ); + if (s.authority !== authority || s.settlementOnly) + throw Error("Original live maintenance initialization required"); + return withControlMaintenanceInitialization(authority, () => withPrivateEffectWriter(owner, run)); +} +/** Publication of already proved guard observations remains possible after cancellation. */ +export function withConfigurationMaintenanceOwnerObservation( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const { s, owner } = maintenanceOwnerScope( + lease, + authority, + "Original configuration maintenance recovery owner required", + ); + const record = current(s); + if (!record || record.state === "settled") + throw Error("Maintenance observation owner already released"); + return withPrivateEffectWriter(owner, run); +} +export function withConfigurationMaintenanceDecisionPublication( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const { s, owner } = maintenanceOwnerScope( + lease, + authority, + "Original maintenance decision owner required", + ); + if (current(s)?.state !== "held") throw Error("Original prepared maintenance root is fenced"); + return withControlMaintenanceDecisionPublication(authority, () => + withPrivateEffectWriter(owner, run), + ); +} +export function withConfigurationMaintenancePhasePublication( + lease: ArtifactTargetLease, + authority: VerifiedExecutionAuthority, + phase: + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff", + run: () => T, +): T { + const { s, owner } = maintenanceOwnerScope( + lease, + authority, + "Original frozen maintenance issuer required", + ); + if (s.settlementOnly || s.authority !== authority) + throw Error("Original frozen maintenance issuer required"); + return withControlDelegationPublication(authority, phase, () => { + if (current(s)?.state !== "held") throw Error("Original maintenance phase is fenced"); + return withPrivateEffectWriter(owner, run); + }); +} +export function withConfigurationMaintenanceReservationPublication( + authority: VerifiedExecutionAuthority, + run: () => T, +): T { + const owner = executionAuthorityState(authority); + if (owner.kind !== "configurationMaintenance.prepare") + throw Error("Original maintenance reservation owner required"); + if (maintenanceLease(authority).state !== "held") + throw Error("Original pre-effect maintenance root required"); + return withControlDelegationPublication(authority, "configuration-fence", () => + withPrivateEffectWriter(owner, run), + ); +} +/** Reopen the prepared owner's already-authorized frozen phase issuer. */ +export function restoreConfigurationMaintenancePhaseIssuer( + authority: VerifiedExecutionAuthority, + phase: + | "configuration-commit" + | "maintenance-hardlinks" + | "maintenance-recreate" + | "maintenance-handoff", +): ArtifactTargetLease { + const owner = executionAuthorityState(authority); + if (owner.kind !== "configurationMaintenance.prepare") + throw Error("Original authorized maintenance phase issuer required"); + const lease = restoreArtifactTargetLeaseForSettlement(authority), + s = stateOf(lease); + try { + withControlDelegationPublication(authority, phase, () => { + if ( + current(s)?.state !== "held" || + !owner.reservations.some((value) => value.phase === phase) + ) + throw Error("Original frozen maintenance root is fenced"); + s.settlementOnly = false; + }); + return lease; + } catch (error) { + closeArtifactTargetLeaseHandle(lease); + throw error; } } -// Fixed closure and original children consume this module's one nominal tree observation. +// Fixed closure and original children consume this module's one tree observation. export function readOriginalMaintenanceFinalChildren( authority: VerifiedExecutionAuthority, observation: ConfigurationMaintenanceClosureObservation, @@ -3558,7 +1580,7 @@ export function readOriginalMaintenanceFinalChildren( if ( state.kind !== "configurationMaintenance.prepare" || !owner.preparedFence || - tree.certificateDigest !== owner.rootCertificateDigest || + tree.leaseId !== owner.leaseId || owner.admissionDigest !== state.receiptDigest || canonicalJson(owner.plan.prepare) !== canonicalJson(state.execution) ) @@ -3586,9 +1608,7 @@ export function readOriginalMaintenanceFinalChildren( ), authority, ), - physical = tree.children.find( - (value) => canonicalJson(value.execution) === canonicalJson(actual.execution), - ); + physical = tree.children.find((value) => sameExecution(value.execution, actual.execution)); if (!physical || canonicalJson(physical.settlement) !== canonicalJson(actual.settlement)) throw Error("Original child physical settlement changed"); allowed.add(controlDigest(actual.execution)); @@ -3609,12 +1629,12 @@ export function readOriginalMaintenanceFinalChildren( ), binding = lookupControlChildPhaseBinding(child, owner.plan.nestedHardlinks); if (!binding) throw Error("Original middle owner's DM leaf unavailable"); - const terminal = verifyDataControlTerminalWitness( + const terminal = readDataControlTerminalObservation( state.installation.installation, binding.child.execution, ), - physical = tree.children.find( - (value) => canonicalJson(value.execution) === canonicalJson(binding.child.execution), + physical = tree.children.find((value) => + sameExecution(value.execution, binding.child.execution), ); if ( !physical || @@ -3632,7 +1652,7 @@ export function readOriginalMaintenanceFinalChildren( planIndex: step.ordinal, execution: binding.child.execution, terminalDigest: terminal.terminalDigest, - terminalAt: terminal.terminalAt, + terminalAt: terminal.observedAt, effectSettlementDigest: controlDigest(physical.settlement), publication: "committed", }), @@ -3643,13 +1663,13 @@ export function readOriginalMaintenanceFinalChildren( state.installation.installation, owner.plan.handoff.handoffId, ), - physical = tree.children.find( - (value) => canonicalJson(value.execution) === canonicalJson(helper.intent.helper), + physical = tree.children.find((value) => + sameExecution(value.execution, helper.intent.helper), ); if ( canonicalJson(helper.intent.operation) !== canonicalJson(step.operation) || - canonicalJson(helper.intent.preparer) !== canonicalJson(actual.execution) || - helper.intent.ownerCertificateDigest !== owner.rootCertificateDigest || + !sameExecution(helper.intent.preparer, actual.execution) || + helper.intent.leaseId !== owner.leaseId || !physical || canonicalJson(physical.settlement) !== canonicalJson(helper.settlement) ) @@ -3700,8 +1720,7 @@ const same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b); function need(value: unknown, message: string): asserts value { if (!value) throw Error(message); } -/** The original owner's own final and inspection, after the data-manager - * confirmed the settle. Parsed prepared status alone cannot enter it. */ +/** The original owner's own final and inspection, after the data-manager confirmed the settle. */ function assertPreparedAndSettledClosure( authority: VerifiedExecutionAuthority, observation: ConfigurationMaintenanceClosureObservation, @@ -3745,7 +1764,7 @@ function assertPreparedAndSettledClosure( owner.admissionDigest === state.receiptDigest && same(owner.origin, state.origin) && same(owner.plan.prepare, state.execution) && - owner.rootCertificateDigest === tree.certificateDigest && + owner.leaseId === tree.leaseId && same(owner.plan.reservations, state.reservations), "Original prepared root identity changed", ); @@ -3781,9 +1800,8 @@ function assertPreparedAndSettledClosure( ); } } -/** The maintenance root may release once every descendant is contained and - * either its fence is settled after a final, or no fence was ever prepared - * and its own prepare recorded that it failed without effect. */ +/** The maintenance root may release once every descendant is contained and either + * its fence is settled after a final, or no fence was ever prepared. */ export function assertConfigurationMaintenanceRootClosure( authority: VerifiedExecutionAuthority, observed: ConfigurationMaintenanceClosureObservation, @@ -3822,7 +1840,7 @@ export function assertConfigurationMaintenanceRootClosure( owner.admissionDigest === state.receiptDigest && same(owner.origin, state.origin) && same(owner.plan.prepare, state.execution) && - owner.rootCertificateDigest === tree.certificateDigest && + owner.leaseId === tree.leaseId && tree.children.every((child) => child.settlement.outcome !== "committed"), "Never-prepared maintenance closure required", ); diff --git a/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts b/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts index 4e65270d1..eb9ce76a2 100644 --- a/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts +++ b/packages/core/src/build-artifacts/test-fixtures/artifact-publication-process.ts @@ -103,7 +103,7 @@ const authority = control.retainAdmittedControlExecution(installation, { requestedDeadlineAt: deadlineAt, serviceIds: ["osrm"], }), - lease = await acquireArtifactTargetLease({ dataRoot, serviceIds: ["osrm"], owner: authority }); + lease = await acquireArtifactTargetLease({ serviceIds: ["osrm"], owner: authority }); const nativeRename = fs.renameSync; let stopped = false; fs.renameSync = (from, to) => { diff --git a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts index 0b88f0e92..c4c89f66e 100644 --- a/packages/core/src/build-artifacts/test-fixtures/peer-process.ts +++ b/packages/core/src/build-artifacts/test-fixtures/peer-process.ts @@ -1,10 +1,8 @@ -import { existsSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; import { createInterface } from "node:readline"; import * as c from "../control-authority"; -import { AnchoredDirectory } from "../filesystem"; -import { inspectControlAdmission, verifiedParentForArtifactDelegation } from "../peer-admission"; +import { inspectControlAdmission } from "../peer-admission"; import * as l from "../target-lease"; +import { verifiedParentForArtifactDelegation } from "../target-lease"; for await (const line of createInterface({ input: process.stdin })) { try { @@ -35,7 +33,6 @@ for await (const line of createInterface({ input: process.stdin })) { serviceIds: ["osrm"], }); const lease = await l.acquireArtifactTargetLease({ - dataRoot: join(input.infraDir, "data"), serviceIds: ["osrm"], owner: authority, }); @@ -52,33 +49,6 @@ for await (const line of createInterface({ input: process.stdin })) { const origin = c.retainedExecutionOrigin(c.installationState(installation), input.execution); c.closeControlInstallation(installation); process.stdout.write(`${JSON.stringify({ ok: true, leaseId: lease.leaseId, origin })}\n`); - } else if (input.mode === "observe-interleaving") { - const installation = c.openControlInstallation({ - infraDir: input.infraDir, - role: "data-manager", - serviceIds: ["motis"], - }); - const read = AnchoredDirectory.prototype.read; - let paused = false; - AnchoredDirectory.prototype.read = function (name, max, privateMode) { - const bytes = read.call(this, name, max, privateMode); - if (!paused && this.path.endsWith("/observations")) { - paused = true; - writeFileSync(input.ready, "ready"); - const until = Date.now() + 10000; - while (!existsSync(input.resume)) { - if (Date.now() >= until) throw Error("Interleaving resume timed out"); - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); - } - } - return bytes; - }; - l.inspectArtifactDelegation({ - dataRoot: join(input.infraDir, "data"), - assertion: input.assertion, - installation, - }); - process.stdout.write(`${JSON.stringify({ ok: true })}\n`); } else if (input.mode === "owner") { const installation = c.openControlInstallation({ infraDir: input.infraDir, @@ -87,7 +57,6 @@ for await (const line of createInterface({ input: process.stdin })) { }); const authority = c.retainAdmittedControlExecution(installation, input.admitted); const lease = await l.acquireArtifactTargetLease({ - dataRoot: join(input.infraDir, "data"), serviceIds: input.admitted.serviceIds, owner: authority, }); @@ -114,7 +83,6 @@ for await (const line of createInterface({ input: process.stdin })) { recipient: "configuration-helper", }); const lease = await l.adoptArtifactTargetLease({ - dataRoot: join(input.infraDir, "data"), assertion: input.assertion, peer, execution: input.execution, @@ -139,7 +107,6 @@ for await (const line of createInterface({ input: process.stdin })) { input.execution, ); const lease = await l.adoptArtifactTargetLease({ - dataRoot: join(input.infraDir, "data"), assertion: input.assertion, peer, execution: input.execution, @@ -193,7 +160,6 @@ for await (const line of createInterface({ input: process.stdin })) { : (() => { // Primitive protocol fixture, not the future generation-bound DM HTTP adapter. const inspected = l.inspectArtifactDelegation({ - dataRoot: join(input.infraDir, "data"), assertion: input.assertion, installation, }); @@ -214,7 +180,6 @@ for await (const line of createInterface({ input: process.stdin })) { ), adopt: (execution: typeof input.execution) => l.adoptArtifactTargetLease({ - dataRoot: join(input.infraDir, "data"), assertion: input.assertion, peer, execution, diff --git a/packages/core/src/build-artifacts/test-fixtures/service-lifecycle-recovery.ts b/packages/core/src/build-artifacts/test-fixtures/service-lifecycle-recovery.ts deleted file mode 100644 index 743083fda..000000000 --- a/packages/core/src/build-artifacts/test-fixtures/service-lifecycle-recovery.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { randomBytes, randomUUID } from "node:crypto"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { - createControlRootOperationKey, - executionFingerprint, - opsOperationFingerprint, -} from "../../ops"; -import { - commitConfigurationGeneration, - withConfigurationGenerationWriter, -} from "../../services/configuration-generation"; -import { createDataManagerTransport } from "../../services/data-manager/transport"; -import { - openServiceLifecycleReceipt, - withServiceLifecycleReceiptAuthority, - withVerifiedLifecycleOwnership, -} from "../../services/service-lifecycle-receipt"; -import { bindControlChildPhase, issueControlChildPhase } from "../child-phase"; -import * as c from "../control-authority"; -import { createDataConfigurationFenceStore } from "../data-configuration-fence"; -import { - generationHardlinksReservation, - retainGenerationHardlinksOwnerPlan, -} from "../generation-hardlinks-owner"; -import { inspectDataExecutionAdmission } from "../peer-admission"; -import * as leases from "../target-lease"; - -export async function failedServiceLifecycle( - root: string, - adopt: boolean | "after-failure" = false, - issueChild = true, -) { - const installations: c.ControlInstallation[] = []; - const generation = { - infraDir: root, - services: [], - input: { - domain: "example.test", - selectedRoots: [], - serviceConfigs: [], - integrationConfigs: [], - serviceSecrets: [], - }, - }; - const seed = await commitConfigurationGeneration(generation); - const installation = c.openControlInstallation({ - infraDir: root, - role: "host-cli", - serviceIds: ["alpha"], - }); - installations.push(installation); - const operation = { - kind: "service.update" as const, - serviceId: "alpha", - expectedConfigurationRevisionId: seed.revisionId, - }; - const admittedAt = Date.now(), - deadlineAt = admittedAt + 60000, - operationFingerprint = opsOperationFingerprint(operation); - const execution = { - executor: "host-cli" as const, - principal: "host-cli" as const, - executionId: randomUUID(), - operationKey: createControlRootOperationKey({ notAfter: deadlineAt, nonce: randomBytes(16) }), - deadlineAt, - operationFingerprint, - fingerprint: executionFingerprint({ kind: operation.kind, operationFingerprint, deadlineAt }), - }; - const revisionId = `cfg1_${"b".repeat(43)}`; - const plan = { - prune: true as const, - plan: [ - { - source: "data/osm", - target: "data/alpha/osm", - consumerService: "alpha", - dataType: "osm-pbf", - }, - ], - }; - const reservation = generationHardlinksReservation({ - parent: execution, - revisionId, - coordinatorMode: "ensure-running", - plan, - serviceIds: ["alpha"], - })!; - const authority = c.retainAdmittedControlExecution(installation, { - execution, - operation, - kind: operation.kind, - admittedAt, - requestedDeadlineAt: deadlineAt, - serviceIds: ["alpha"], - reservations: [reservation], - }); - const lease = await leases.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), - serviceIds: ["alpha"], - owner: authority, - }); - let assertion = ""; - let child: Awaited>; - async function adoptOriginalChild() { - const dm = c.openControlInstallation({ - infraDir: root, - role: "data-manager", - serviceIds: ["alpha"], - }); - installations.push(dm); - const receiver = inspectDataExecutionAdmission(dm, { - rawHeaders: ["x-openmapx-control-delegation", assertion], - transportPrincipal: "host-cli", - input: { - kind: "generation-hardlinks", - revisionId, - coordinatorMode: "ensure-running", - planDigest: c.controlDigest(plan), - }, - operationKey: child.execution.operationKey, - operationFingerprint: reservation.operationFingerprint, - requestedDeadlineAt: child.execution.deadlineAt, - }); - const borrowed = await receiver!.adopt(child.execution); - if (borrowed) { - const childAuthority = c.retainAdmittedControlExecution(dm, { - execution: child.execution, - kind: "generation-hardlinks", - operation: { - kind: "generation-hardlinks", - revisionId, - coordinatorMode: "ensure-running", - planDigest: c.controlDigest(plan), - }, - parent: receiver!.parent, - admittedAt: Date.now(), - requestedDeadlineAt: child.execution.deadlineAt, - serviceIds: ["alpha"], - }); - leases.bindBorrowedControlExecution(borrowed, childAuthority); - leases.closeArtifactTargetLeaseHandle(borrowed); - } - } - try { - retainGenerationHardlinksOwnerPlan(authority, { - version: 1, - parent: execution, - revisionId, - coordinatorMode: "ensure-running", - plan, - reservation, - }); - const store = await withServiceLifecycleReceiptAuthority(authority, () => - openServiceLifecycleReceipt({ - dataRoot: join(root, "data"), - execution, - serviceIds: ["alpha"], - action: "update", - noDeps: false, - coordinatorMode: "ensure-running", - expectedRevisionId: seed.revisionId, - seedRevisionId: null, - }), - ); - await withConfigurationGenerationWriter(root, (writer) => - withVerifiedLifecycleOwnership({ writer, lease, authority }, async (ownership) => { - let receipt = await store.read(); - for (const phase of ["pins_validated", "pulled"] as const) { - receipt = { ...receipt, phase, sequence: receipt.sequence + 1 }; - await store.checkpoint(receipt.sequence - 1, receipt, ownership); - } - await writer.commit({ - ...generation, - revisionId, - expectedCurrentRevisionId: seed.revisionId, - }); - receipt = { - ...receipt, - phase: "generation_committed", - commit: "final_committed", - finalRevisionId: revisionId, - sequence: receipt.sequence + 1, - }; - await store.checkpoint(receipt.sequence - 1, receipt, ownership); - if (issueChild) { - child = await bindControlChildPhase(authority, { phase: "hardlinks", ordinal: 0 }); - const issued = await issueControlChildPhase(authority, { - phase: "hardlinks", - ordinal: 0, - childPublicKey: c.controlRolePublicKey(installation, "data-manager"), - lease, - }); - assertion = issued.assertion; - if (adopt === true) await adoptOriginalChild(); - } - await store.checkpoint( - receipt.sequence, - { - ...receipt, - phase: "failed", - followUp: "unknown", - failure: { reason: "hardlinks", failedServiceIds: ["alpha"] }, - sequence: receipt.sequence + 1, - }, - ownership, - ); - if (adopt === "after-failure") await adoptOriginalChild(); - }), - ); - } finally { - leases.closeArtifactTargetLeaseHandle(lease); - } - const recoveryDm = c.openControlInstallation({ - infraDir: root, - role: "data-manager", - serviceIds: ["alpha"], - }); - installations.push(recoveryDm); - const unsafe = async (query: string) => - query.includes("pg_try_advisory_xact_lock") ? [{ held: true }] : []; - const recoveryFence = createDataConfigurationFenceStore(recoveryDm, { - unsafe, - transaction: async (run) => run({ unsafe }), - }); - const nonAdmission = () => - recoveryFence.attestUnadoptedGenerationHardlinks( - { - operationKey: child!.execution.operationKey, - requestedDeadlineAt: child!.execution.deadlineAt, - input: { - kind: "generation-hardlinks", - revisionId, - coordinatorMode: "ensure-running", - planDigest: c.controlDigest(plan), - }, - plan, - }, - ["x-openmapx-control-delegation", assertion], - ); - const recoveryRequests: Array<{ url: string; method: string | undefined }> = []; - const recoveryFetch: typeof fetch = async (url, options) => { - recoveryRequests.push({ url: String(url), method: options?.method }); - if ( - String(url) !== "http://127.0.0.1:4301/executions/generation-hardlinks/recovery" || - options?.method !== "POST" - ) - throw Error("Unexpected recovery request"); - const headers = new Headers(options?.headers), - rawHeaders = [...headers.entries()].flat(); - try { - return Response.json( - await recoveryFence.attestUnadoptedGenerationHardlinks( - JSON.parse(String(options?.body)), - rawHeaders, - ), - ); - } catch (error) { - return Response.json( - { error: error instanceof Error ? error.message : "Recovery unavailable" }, - { status: 409 }, - ); - } - }; - const transport = createDataManagerTransport({ - baseUrl: "http://127.0.0.1:4301", - authToken: "d".repeat(32), - fetch: recoveryFetch, - }); - const path = join(root, "data/ops-agent/lifecycle", `${execution.operationKey}.json`); - return { - installation, - authority, - execution, - path, - receiptBytes: readFileSync(path), - revisionId, - assertion, - child: child!, - plan, - reservation, - installations, - leaseId: lease.leaseId, - transport, - recoveryFetch, - recoveryRequests, - nonAdmission, - }; -} diff --git a/packages/core/src/catalog-pins/bootstrap.ts b/packages/core/src/catalog-pins/bootstrap.ts index b9b83004b..3050c6c09 100644 --- a/packages/core/src/catalog-pins/bootstrap.ts +++ b/packages/core/src/catalog-pins/bootstrap.ts @@ -3,7 +3,6 @@ import { beginControlEffect, controlDigest, executionAuthorityState, - installationState, retainAdmittedControlExecution, retainControlEffectReceipt, verifyControlEffectSettlement, @@ -60,11 +59,7 @@ export async function initializeManagedCatalogPins(options: { }); let lease: ArtifactTargetLease | undefined; try { - lease = await acquireArtifactTargetLease({ - dataRoot: installationState(options.installation).dataRoot, - serviceIds: ["motis"], - owner: authority, - }); + lease = await acquireArtifactTargetLease({ serviceIds: ["motis"], owner: authority }); beginControlEffect(authority); const pins = await initializeCatalogPinSet({ rootDir: options.rootDir, lease }); retainControlEffectReceipt(authority, { diff --git a/packages/core/src/catalog-pins/store.test.ts b/packages/core/src/catalog-pins/store.test.ts index 175e77d57..a11560313 100644 --- a/packages/core/src/catalog-pins/store.test.ts +++ b/packages/core/src/catalog-pins/store.test.ts @@ -27,6 +27,7 @@ import { provisionControlInstallation, retainAdmittedControlExecution, } from "../build-artifacts/control-authority"; +import { heldTargets } from "../build-artifacts/lease-store"; import { type ArtifactTargetLease, acquireArtifactTargetLease, @@ -124,7 +125,6 @@ beforeEach(async () => { serviceIds: ["motis"], }); lease = await acquireArtifactTargetLease({ - dataRoot: join(docker, "data"), serviceIds: ["motis"], owner: authority, }); @@ -348,7 +348,7 @@ it("bootstraps an actual local initialization root and releases its MOTIS owners const { initializeManagedCatalogPins } = await import("./bootstrap"); const pins = await initializeManagedCatalogPins({ rootDir: other, installation: local }); expect(pins).toEqual(createCatalogPinSet(transitous, gbfs)); - expect(existsSync(join(infra, "data/.admin-target-control/locks/motis"))).toBe(false); + expect(heldTargets(infra).includes("motis")).toBe(false); expect(await initializeManagedCatalogPins({ rootDir: other, installation: local })).toEqual( pins, ); @@ -398,7 +398,7 @@ it("does not acquire a startup target root for a malformed legacy pair", async ( await expect( initializeManagedCatalogPins({ rootDir: other, installation: local }), ).rejects.toThrow(); - expect(existsSync(join(infra, "data/.admin-target-control/locks/motis"))).toBe(false); + expect(heldTargets(infra).includes("motis")).toBe(false); } finally { closeControlInstallation(local); rmSync(other, { recursive: true, force: true }); diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index 9b27ee56f..dd94a8bbe 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -494,6 +494,26 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { }, "gbfsCatalogLock.inspect": { kind: "gbfsCatalogLock.inspect" }, "transitousPin.inspect": { kind: "transitousPin.inspect" }, + "lease.acquire": { + kind: "lease.acquire", + owner: { + executor: "data-manager", + principal: "data-manager", + executionId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + operationKey: "opk1_AAAAAAAAAAAAAAAAAAAA", + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: 1_900_000_000_000, + }, + ownerKind: "data.update", + targets: ["motis"], + }, + "lease.read": { kind: "lease.read", leaseId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22" }, + "lease.update": { + kind: "lease.update", + leaseId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + action: { action: "release" }, + }, }; it("compares projections independent of object ordering while preserving array and value identity", () => { diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index 3904cd4c2..cfde602bb 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -46,6 +46,7 @@ import { import { configurationApplyResultSchema } from "./configuration-result-records"; import { dataInspectionResultSchema } from "./data-inventory-records"; import { dataWorkflowReceiptSchema } from "./job-evidence"; +import { leaseOperationSchemas, leaseResultSchemas } from "./lease-records"; import { standaloneIntegrationOperationSchemas, standaloneIntegrationResultSchemas, @@ -197,11 +198,14 @@ export const OPS_OPERATION_KINDS = [ "transitousLock.approve", "gbfsCatalogLock.inspect", "transitousPin.inspect", + "lease.acquire", + "lease.read", + "lease.update", ] as const; export type OpsOperationKind = (typeof OPS_OPERATION_KINDS)[number]; -export type OpsRole = "api" | "data-manager"; +export type OpsRole = "api" | "data-manager" | "host-cli"; export type OpsExecutionMode = "sync" | "async"; export interface OpsKindPolicy { @@ -343,6 +347,10 @@ export const OPS_KIND_POLICIES = { "gbfsCatalogLock.inspect": sync("data-manager", 30_000), // Read-only, for the admin panel: the pins, never a proposal or a change. "transitousPin.inspect": sync("api", 30_000), + // Data-manager and the host CLI claim targets through the agent; see authorizeOpsOperation. + "lease.acquire": sync("data-manager", 15_000, 256 * 1024), + "lease.read": sync("data-manager", 15_000, 256 * 1024), + "lease.update": sync("data-manager", 15_000, 256 * 1024), } as const satisfies Record; /** Read streams/diagnostics do not acquire control-root authority or a retention dependency. */ @@ -698,6 +706,7 @@ export const opsOperationSchema = z.discriminatedUnion("kind", [ pairedPinApprovalInputSchema, kindOnly("gbfsCatalogLock.inspect"), kindOnly("transitousPin.inspect"), + ...leaseOperationSchemas, ]); export type OpsOperation = z.infer; export type TrustedConfigurationApplyOperation = Extract< @@ -739,6 +748,8 @@ export type OpsRequest = z.infer; export function authorizeOpsOperation(role: OpsRole, kind: OpsOperationKind): boolean { // Decode retained historical records, but never admit a new remote file writer. if (kind === "data.generateApiKeys") return false; + // The host CLI reaches the agent only to claim targets for its own commands. + if (role === "host-cli") return kind.startsWith("lease."); return ( OPS_KIND_POLICIES[kind].role === role || (role === "data-manager" && kind === "catalogPins.inspect") @@ -1425,6 +1436,7 @@ export const OPS_RESULT_SCHEMAS = { lockedAt: timestampSchema, lockedBy: actorLabelSchema, }), + ...leaseResultSchemas, } as const satisfies Record; export type OpsResultFor = z.infer<(typeof OPS_RESULT_SCHEMAS)[K]>; diff --git a/packages/core/src/ops/lease-records.ts b/packages/core/src/ops/lease-records.ts new file mode 100644 index 000000000..73e9dcbdf --- /dev/null +++ b/packages/core/src/ops/lease-records.ts @@ -0,0 +1,101 @@ +import z from "zod/v4"; +import { executionRefSchema } from "./execution-ref"; + +/** Target leases as the agent's store records them; plain data only. */ +export const leaseTargetsSchema = z + .array(z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/)) + .min(1) + .max(256) + .refine( + (targets) => targets.every((id, index) => index === 0 || (targets[index - 1] ?? id) < id), + "Targets must be sorted and unique", + ); +export const leaseSettlementSchema = z.strictObject({ + identity: executionRefSchema, + outcome: z.enum(["contained", "committed", "failed_without_effect", "unknown"]), + publication: z.enum(["not_started", "committed", "unknown"]), + terminalReceiptDigest: z.string().regex(/^[a-f0-9]{64}$/), +}); +export type LeaseSettlement = z.infer; +const nodeStateSchema = z.enum(["held", "cancel_pending", "recovery_required", "settled"]); +/** A child as its issuer records it, before it starts. */ +export const leaseChildIntentSchema = z.strictObject({ + key: z.uuid(), + /** The child that issued this one, or null for the owner. */ + parentKey: z.uuid().nullable(), + parent: executionRefSchema, + execution: executionRefSchema, + kind: z.string().min(1).max(128), + phase: z.string().min(1).max(64), + recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), + targets: leaseTargetsSchema, + permittedFurtherEdges: z.array(z.string().max(64)).max(1), + depth: z.number().int().min(1).max(2), + /** + * What the issuer prepared for the recipient to act on, such as the transit + * candidate data-manager staged for the agent. The recipient re-checks it + * against the files before any effect. + */ + evidence: z + .record(z.string(), z.unknown()) + .refine((value) => JSON.stringify(value).length <= 8192, "Evidence too large") + .optional(), +}); +export const leaseChildSchema = leaseChildIntentSchema.extend({ + /** Set when the recipient took the child over; an unadopted child never started. */ + adopted: z.boolean(), + state: nodeStateSchema, + settlement: leaseSettlementSchema.optional(), +}); +export type LeaseChild = z.infer; +export const leaseRecordSchema = z.strictObject({ + leaseId: z.uuid(), + owner: executionRefSchema, + ownerKind: z.string().min(1).max(128), + targets: leaseTargetsSchema, + state: nodeStateSchema, + settlement: leaseSettlementSchema.optional(), + children: z.array(leaseChildSchema).max(64), + reason: z.string().max(512).optional(), + createdAt: z.number().int().nonnegative().safe(), +}); +export type LeaseRecord = z.infer; +const holderSchema = leaseRecordSchema.pick({ + leaseId: true, + owner: true, + ownerKind: true, + state: true, +}); + +/** What data-manager and the host CLI ask the agent to do with their leases. */ +export const leaseOperationSchemas = [ + z.strictObject({ + kind: z.literal("lease.acquire"), + owner: executionRefSchema, + ownerKind: z.string().min(1).max(128), + targets: leaseTargetsSchema, + }), + z.strictObject({ kind: z.literal("lease.read"), leaseId: z.uuid() }), + z.strictObject({ + kind: z.literal("lease.update"), + leaseId: z.uuid(), + action: z.discriminatedUnion("action", [ + z.strictObject({ action: z.literal("delegate"), child: leaseChildIntentSchema }), + z.strictObject({ + action: z.literal("settle"), + key: z.uuid().nullable(), + settlement: leaseSettlementSchema, + }), + z.strictObject({ action: z.literal("cancel"), key: z.uuid().nullable() }), + z.strictObject({ action: z.literal("release") }), + ]), + }), +] as const; +export const leaseResultSchemas = { + "lease.acquire": z.discriminatedUnion("outcome", [ + z.strictObject({ outcome: z.literal("acquired"), lease: leaseRecordSchema }), + z.strictObject({ outcome: z.literal("busy"), target: z.string(), holder: holderSchema }), + ]), + "lease.read": z.strictObject({ lease: leaseRecordSchema.nullable() }), + "lease.update": z.strictObject({ lease: leaseRecordSchema.nullable() }), +} as const; diff --git a/packages/core/src/services/__tests__/configuration-generation-readers.test.ts b/packages/core/src/services/__tests__/configuration-generation-readers.test.ts index 4f1561da8..63fc08055 100644 --- a/packages/core/src/services/__tests__/configuration-generation-readers.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation-readers.test.ts @@ -179,7 +179,7 @@ async function fixture(reader: "selection" | "digest") { assertRetained: () => { expect(currentConfigurationGeneration(infraDir)).toBe(previous.revisionId); // The writer released its lock. - expect(existsSync(join(infraDir, CONFIGURATION_DIRECTORY, ".writer"))).toBe(false); + expect(existsSync(join(infraDir, CONFIGURATION_DIRECTORY, "writer.lock"))).toBe(false); expect( readdirSync(join(infraDir, CONFIGURATION_GENERATIONS)).every( (name) => !name.startsWith("."), diff --git a/packages/core/src/services/__tests__/infra-state.test.ts b/packages/core/src/services/__tests__/infra-state.test.ts index 68eacc4ee..5a1d5b8e9 100644 --- a/packages/core/src/services/__tests__/infra-state.test.ts +++ b/packages/core/src/services/__tests__/infra-state.test.ts @@ -178,9 +178,11 @@ describe("the writer lock", () => { it("is taken in the agent's state directory and released", async () => { const { infraDir } = checkout(); const release = await acquireConfigurationOwnership(infraDir); - expect(existsSync(join(infraDir, "state", "ops-agent", "configuration", ".writer"))).toBe(true); + expect(existsSync(join(infraDir, "state", "ops-agent", "configuration", "writer.lock"))).toBe( + true, + ); release(); - expect(existsSync(join(infraDir, "state", "ops-agent", "configuration", ".writer"))).toBe( + expect(existsSync(join(infraDir, "state", "ops-agent", "configuration", "writer.lock"))).toBe( false, ); }); diff --git a/packages/core/src/services/__tests__/recoverable-lock.test.ts b/packages/core/src/services/__tests__/recoverable-lock.test.ts index 556fbd1e4..36a5701da 100644 --- a/packages/core/src/services/__tests__/recoverable-lock.test.ts +++ b/packages/core/src/services/__tests__/recoverable-lock.test.ts @@ -106,6 +106,32 @@ describe("recoverable lock", () => { expect(asked).toEqual(["a".repeat(64)]); }); + it("reclaims a holder of this container from before the container restarted", async () => { + const container = "b".repeat(64); + const self = { ...currentLockHolder("self"), bootId: null, containerId: container }; + const before = foreign({ bootId: null, containerId: container, pidNamespace: "pid:[old]" }); + expect(await lockHolderGone(before, { ...self, pidNamespace: "pid:[new]" }, neverAsked)).toBe( + true, + ); + // The same namespace keeps judging the holder's own process. + expect( + await lockHolderGone( + { ...before, pidNamespace: "pid:[new]", pid: process.pid }, + { ...self, pidNamespace: "pid:[new]" }, + neverAsked, + ), + ).toBe(false); + }); + + it("refuses a lock path that is not a lock file at once", async () => { + const path = lockPath(); + const { mkdirSync } = await import("node:fs"); + mkdirSync(path); + await expect(acquireRecoverableLock(path, { purpose: "next", waitMs: 60_000 })).rejects.toThrow( + "is not a lock file", + ); + }); + it("never reclaims a holder that cannot be proved gone", async () => { const path = lockPath(); writeFileSync( diff --git a/packages/core/src/services/agent-state.ts b/packages/core/src/services/agent-state.ts index dbc365746..0df6b3f27 100644 --- a/packages/core/src/services/agent-state.ts +++ b/packages/core/src/services/agent-state.ts @@ -1,7 +1,28 @@ -import { chmodSync, lstatSync, mkdirSync, renameSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { chmodSync, lstatSync, mkdirSync, readFileSync, renameSync } from "node:fs"; import { dirname, join } from "node:path"; import { atomicWrite } from "../release-files"; +/** + * The token the host CLI presents to the agent. The agent creates it at + * startup in its own state; only the deployment user, through `docker compose + * exec ops-agent`, can read it. + */ +export function hostCliTokenPath(infraDir: string): string { + return join(infraDir, "state", "ops-agent", "host-cli.token"); +} +export function ensureHostCliToken(infraDir: string): string { + const path = hostCliTokenPath(infraDir); + try { + const existing = readFileSync(path, "utf8").trim(); + if (/^[A-Za-z0-9_-]{43}$/.test(existing)) return existing; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const token = randomBytes(32).toString("base64url"); + writeAgentStateFile(infraDir, "host-cli.token", `${token}\n`); + return token; +} /** Replaces one of the agent's state files atomically, creating its directory (0700) first. */ export function writeAgentStateFile(infraDir: string, name: string, contents: string): void { const directory = join(infraDir, "state", "ops-agent"); diff --git a/packages/core/src/services/configuration-ownership.ts b/packages/core/src/services/configuration-ownership.ts index 1e7330ab9..68ae82348 100644 --- a/packages/core/src/services/configuration-ownership.ts +++ b/packages/core/src/services/configuration-ownership.ts @@ -1,16 +1,15 @@ -import { lstatSync, mkdirSync, rmdirSync, type Stats } from "node:fs"; +import { lstatSync, mkdirSync } from "node:fs"; import { join } from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; +import { acquireRecoverableLock, LockBusyError } from "./recoverable-lock"; -/** The writer lock, in the agent's state: `state/ops-agent/configuration/.writer`. */ +/** The writer lock, in the agent's state: `state/ops-agent/configuration/writer.lock`. */ const CONFIGURATION_DIRECTORY = ["state", "ops-agent", "configuration"] as const; -const LOCK_NAME = ".writer"; -const WAIT_MS = 5_000; +const LOCK_NAME = "writer.lock"; export class ConfigurationWriterBusyError extends Error { - constructor() { + constructor(cause?: LockBusyError) { super( - "Configuration writer is busy: another configuration change is running. Try again when it finishes.", + `Configuration writer is busy: another configuration change is running. Try again when it finishes.${cause?.holder ? ` ${cause.message}` : ""}`, ); } } @@ -37,64 +36,12 @@ function assertOwnDirectory(path: string): void { } } -async function acquireLockDirectory(path: string, signal?: AbortSignal): Promise<() => void> { - const deadline = performance.now() + WAIT_MS; - for (;;) { - signal?.throwIfAborted(); - if (performance.now() >= deadline) throw new ConfigurationWriterBusyError(); - try { - mkdirSync(path, { mode: 0o700 }); - break; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - let existing: Stats; - try { - existing = lstatSync(path); - } catch (inspectionError) { - if ((inspectionError as NodeJS.ErrnoException).code === "ENOENT") continue; - throw inspectionError; - } - if ( - !existing.isDirectory() || - existing.isSymbolicLink() || - existing.uid !== owner() || - (existing.mode & 0o777) !== 0o700 - ) { - throw new Error("Unsafe configuration writer ownership"); - } - const remaining = deadline - performance.now(); - if (remaining <= 0) throw new ConfigurationWriterBusyError(); - await delay(Math.min(25, remaining), undefined, { signal }); - } - } - const held = lstatSync(path); - let released = false; - return () => { - if (released) throw new Error("Configuration ownership already released"); - const current = lstatSync(path); - if ( - current.dev !== held.dev || - current.ino !== held.ino || - !current.isDirectory() || - current.isSymbolicLink() - ) { - throw new Error("Configuration writer ownership changed"); - } - // No recursive deletion: unexpected contents fail closed. - rmdirSync(path); - released = true; - }; -} - /** - * POSIX mkdir provides exclusive ownership across CLI and host processes on the - * same local filesystem. Never remove an existing lock, including after a crash: - * time, a PID and a failed liveness check are not authority to steal ownership. - * Recovery requires stopping ALL writers before an operator removes the lock. - * - * The lock and everything it guards live in the agent's state directory, which - * the host CLI shares as the same user; `infra/docker`'s own modes decide - * nothing. + * Exclusive configuration writer for the agent, its helper and the host CLI. + * The lock file records its holder, and another writer reclaims it only when + * that holder is provably gone; it is never held across a Docker effect. + * The lock and everything it guards live in the agent's state directory, + * which the host CLI shares as the same user. */ export async function acquireConfigurationOwnership( infraDir: string, @@ -108,5 +55,18 @@ export async function acquireConfigurationOwnership( mkdirSync(directory, { recursive: true, mode: 0o700 }); assertOwnDirectory(state); assertOwnDirectory(directory); - return acquireLockDirectory(join(directory, LOCK_NAME), signal); + try { + return await acquireRecoverableLock(join(directory, LOCK_NAME), { + purpose: "configuration writer", + signal, + }); + } catch (error) { + if (error instanceof LockBusyError) throw new ConfigurationWriterBusyError(error); + throw error; + } +} + +/** The writer lock's path, for the operator exit. */ +export function configurationWriterLockPath(infraDir: string): string { + return join(infraDir, ...CONFIGURATION_DIRECTORY, LOCK_NAME); } diff --git a/packages/core/src/services/data-manager/configuration-maintenance.ts b/packages/core/src/services/data-manager/configuration-maintenance.ts index d51a3cedd..33310d963 100644 --- a/packages/core/src/services/data-manager/configuration-maintenance.ts +++ b/packages/core/src/services/data-manager/configuration-maintenance.ts @@ -1,4 +1,6 @@ import { + type ConfigurationPrepareNonAdmission, + configurationPrepareNonAdmissionSchema, type DataConfigurationFencePrepare, type DataConfigurationFenceReceipt, type DataConfigurationFenceSettle, @@ -40,15 +42,34 @@ export async function prepareDataConfigurationFence( } export async function inspectDataConfigurationFence( maintenanceId: string, - options: { transport: DataManagerTransport; witness: string; signal?: AbortSignal }, + options: { + transport: DataManagerTransport; + candidateId: string; + ownerLeaseId: string; + signal?: AbortSignal; + }, ): Promise { - const result = await sdk.inspectDataConfigurationFence(maintenanceId, { - transport: options.transport, - signal: options.signal, - headers: { "x-openmapx-configuration-fence-witness": boundedHeader(options.witness) }, - }); + const result = await sdk.inspectDataConfigurationFence( + maintenanceId, + { candidateId: options.candidateId, ownerLeaseId: options.ownerLeaseId }, + { transport: options.transport, signal: options.signal }, + ); return dataConfigurationFenceReceiptSchema.parse(unwrapDataManagerResponse(result)); } +export async function recoverUnadmittedDataConfigurationFence( + input: DataConfigurationFencePrepare, + options: ControlOptions, +): Promise { + const result = await sdk.recoverUnadmittedConfigurationPreparation( + boundedBody(dataConfigurationFencePrepareSchema.parse(input)), + { + transport: options.transport, + signal: options.signal, + headers: { "x-openmapx-control-delegation": boundedHeader(options.controlAssertion) }, + }, + ); + return configurationPrepareNonAdmissionSchema.parse(unwrapDataManagerResponse(result)); +} export async function settleDataConfigurationFence( input: DataConfigurationFenceSettle, options: ControlOptions, diff --git a/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json b/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json index d0d047a53..25bfe1ce5 100644 --- a/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json +++ b/packages/core/src/services/data-manager/contracts/__fixtures__/reviewed-surface.json @@ -1,7 +1,7 @@ { "revision": "42a7964f9b8eaf28a6c2749ab8608e3b1d1f8e19", - "explicitOperations": 53, - "note": "Static inventory. Expands two literal ambient actions and imported relay constant. Excludes Fastify automatic HEAD aliases; binary HEAD routes explicitly included. No route handler was invoked. Updated by explicit review for the eight previously installed durable execution/fence routes and the new reversible source-disable route; baseline revision precedes the latter. Task05.2 explicitly adds the country-only server preflight route; parser, paging and metadata changes retain existing route identities. Merge review explicitly retains the installed Overture recovery inspection and main read-only transit pin inspection; no additional mutation is admitted. Recovery review adds the service-auth generation-hardlinks non-admission attestation route; it observes the original canceled expired grant and locked receiver state, without admitting or replaying a job. Application maintenance recovery adds its service-auth non-admission route for the original canceled expired preparation, requiring locked SQL and original private adoption evidence; it never admits or replays PREPARE. Removal review drops the nine uncalled NDJSON progress routes and the uncalled font download route; their job kinds remain admitted only through durable executions.", + "explicitOperations": 52, + "note": "Static inventory. Expands two literal ambient actions and imported relay constant. Excludes Fastify automatic HEAD aliases; binary HEAD routes explicitly included. No route handler was invoked. Updated by explicit review for the eight previously installed durable execution/fence routes and the new reversible source-disable route; baseline revision precedes the latter. Task05.2 explicitly adds the country-only server preflight route; parser, paging and metadata changes retain existing route identities. Merge review explicitly retains the installed Overture recovery inspection and main read-only transit pin inspection; no additional mutation is admitted. Agent lease review drops the generation-hardlinks non-admission route: the hardlink leaf holds no lease of its own. Application maintenance recovery adds its service-auth non-admission route for the original canceled expired preparation, requiring locked SQL and original private adoption evidence; it never admits or replays PREPARE. Removal review drops the nine uncalled NDJSON progress routes and the uncalled font download route; their job kinds remain admitted only through durable executions.", "operations": [ { "method": "POST", @@ -75,10 +75,6 @@ "method": "POST", "path": "/executions/generation-hardlinks" }, - { - "method": "POST", - "path": "/executions/generation-hardlinks/recovery" - }, { "method": "GET", "path": "/internal/transit/operator-feed/:handle" diff --git a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts index 5b7700854..53a4f7e9d 100644 --- a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts +++ b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts @@ -5,7 +5,6 @@ import { dataEpochSchema, dataFingerprintSchema, dataRevisionSchema, - generationHardlinksNonAdmissionPayloadSchema, } from "./executions"; const binding = { @@ -25,27 +24,23 @@ export const dataConfigurationFencePrepareSchema = z.strictObject({ .refine((values) => values.every((value, i) => i === 0 || (values[i - 1] ?? value) < value)), coordinatorMode: dataCoordinatorModeSchema, }); -export const CONFIGURATION_PREPARE_NON_ADMISSION_DOMAIN = - "openmapx-configuration-prepare-non-admission-v1"; -/** A receiver observation, never an admission or forward execution. */ -export const configurationPrepareNonAdmissionPayloadSchema = z.strictObject({ - version: z.literal(1), +/** The data-manager's answer that an expired preparation was never admitted. It + * can no longer be admitted, so the answer is final. */ +export const configurationPrepareNonAdmissionSchema = z.strictObject({ + version: z.literal(2), kind: z.literal("configuration-prepare-not-admitted"), - installationId: z.uuid(), - origin: generationHardlinksNonAdmissionPayloadSchema.shape.origin, leaseId: z.uuid(), - certificateDigest: dataFingerprintSchema, delegationId: z.uuid(), - grantDigest: dataFingerprintSchema, child: executionRefSchema, input: dataConfigurationFencePrepareSchema, - adoptionDigest: dataFingerprintSchema, - canceledSequence: z.number().int().positive().safe(), observedAt: dataEpochSchema, }); -export const configurationPrepareNonAdmissionSchema = z.strictObject({ - payload: configurationPrepareNonAdmissionPayloadSchema, - signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/), +export type ConfigurationPrepareNonAdmission = z.infer< + typeof configurationPrepareNonAdmissionSchema +>; +export const dataConfigurationFenceInspectQuerySchema = z.strictObject({ + candidateId: z.uuid(), + ownerLeaseId: z.uuid(), }); export const dataConfigurationFenceReceiptSchema = z.strictObject({ ...binding, diff --git a/packages/core/src/services/data-manager/contracts/executions.ts b/packages/core/src/services/data-manager/contracts/executions.ts index 5b861ad8e..b0690b8ac 100644 --- a/packages/core/src/services/data-manager/contracts/executions.ts +++ b/packages/core/src/services/data-manager/contracts/executions.ts @@ -11,30 +11,6 @@ export const dataFingerprintSchema = z.string().regex(/^[a-f0-9]{64}$/); export const dataEpochSchema = z.number().int().nonnegative().safe(); export const dataRevisionSchema = z.string().regex(/^cfg1_[A-Za-z0-9_-]{43}$/); export const dataCoordinatorModeSchema = z.enum(["ensure-running", "require-running"]); -export const GENERATION_HARDLINKS_NON_ADMISSION_DOMAIN = - "openmapx-generation-hardlinks-non-admission-v1"; -export const generationHardlinksNonAdmissionPayloadSchema = z.strictObject({ - version: z.literal(1), - kind: z.literal("generation-hardlinks-not-admitted"), - installationId: z.uuid(), - origin: z.strictObject({ - version: z.literal(1), - rootKey: dataOperationKeySchema, - issuer: z.enum(["host-cli", "data-manager", "ops-agent"]), - notAfter: dataEpochSchema, - }), - leaseId: z.uuid(), - certificateDigest: dataFingerprintSchema, - delegationId: z.uuid(), - grantDigest: dataFingerprintSchema, - parent: executionRefSchema, - child: executionRefSchema, - canceledSequence: z.number().int().positive().safe(), -}); -export const generationHardlinksNonAdmissionSchema = z.strictObject({ - payload: generationHardlinksNonAdmissionPayloadSchema, - signature: z.string().min(1).max(256), -}); const regional = (kind: K) => z.strictObject({ kind: z.literal(kind), region: regionSchema }); const countries = z.array(z.string().regex(/^[A-Za-z]{2,3}$/)).max(256); diff --git a/packages/core/src/services/data-manager/contracts/operations.ts b/packages/core/src/services/data-manager/contracts/operations.ts index 97c9e1b4d..77b0403fc 100644 --- a/packages/core/src/services/data-manager/contracts/operations.ts +++ b/packages/core/src/services/data-manager/contracts/operations.ts @@ -92,19 +92,6 @@ export const dataManagerOperations = { 503: json(httpErrorSchema), }, }, - recoverUnadmittedGenerationHardlinks: { - method: "POST", - path: "/executions/generation-hardlinks/recovery", - auth: service, - request: { body: e.startGenerationHardlinksExecutionSchema }, - responses: { - ...serviceErrors, - 200: json(e.generationHardlinksNonAdmissionSchema), - 403: json(httpErrorSchema), - 409: json(httpErrorSchema), - 503: json(httpErrorSchema), - }, - }, prepareDataConfigurationFence: { method: "POST", path: "/executions/configuration-maintenance/prepare", @@ -135,7 +122,10 @@ export const dataManagerOperations = { method: "GET", path: "/executions/configuration-maintenance/:maintenanceId", auth: service, - request: { params: z.strictObject({ maintenanceId: z.uuid() }) }, + request: { + params: z.strictObject({ maintenanceId: z.uuid() }), + query: cm.dataConfigurationFenceInspectQuerySchema, + }, responses: { ...serviceErrors, 200: json(cm.dataConfigurationFenceReceiptSchema), diff --git a/packages/core/src/services/data-manager/generated/data-manager.generated.ts b/packages/core/src/services/data-manager/generated/data-manager.generated.ts index 6dffc0b9b..3dc331383 100644 --- a/packages/core/src/services/data-manager/generated/data-manager.generated.ts +++ b/packages/core/src/services/data-manager/generated/data-manager.generated.ts @@ -320,6 +320,7 @@ import type { InspectDataConfigurationFence429, InspectDataConfigurationFence500, InspectDataConfigurationFence503, + InspectDataConfigurationFenceParams, InspectTransitSyncPreflight200, InspectTransitSyncPreflight400, InspectTransitSyncPreflight401, @@ -414,17 +415,6 @@ import type { RecoverUnadmittedConfigurationPreparation500, RecoverUnadmittedConfigurationPreparation503, RecoverUnadmittedConfigurationPreparationBody, - RecoverUnadmittedGenerationHardlinks200, - RecoverUnadmittedGenerationHardlinks400, - RecoverUnadmittedGenerationHardlinks401, - RecoverUnadmittedGenerationHardlinks403, - RecoverUnadmittedGenerationHardlinks409, - RecoverUnadmittedGenerationHardlinks413, - RecoverUnadmittedGenerationHardlinks415, - RecoverUnadmittedGenerationHardlinks429, - RecoverUnadmittedGenerationHardlinks500, - RecoverUnadmittedGenerationHardlinks503, - RecoverUnadmittedGenerationHardlinksBody, ReloadDatasets200, ReloadDatasets400, ReloadDatasets401, @@ -579,7 +569,6 @@ import { prepareOfflinePackageRequest, proposeTransitBumpRequest, recoverUnadmittedConfigurationPreparationRequest, - recoverUnadmittedGenerationHardlinksRequest, reloadDatasetsRequest, removeTransitSourceRequest, restartMotisRequest, @@ -807,116 +796,6 @@ export const startGenerationHardlinksExecution = async ( ); }; -export type recoverUnadmittedGenerationHardlinksResponse200 = { - data: RecoverUnadmittedGenerationHardlinks200; - status: 200; -}; - -export type recoverUnadmittedGenerationHardlinksResponse400 = { - data: RecoverUnadmittedGenerationHardlinks400; - status: 400; -}; - -export type recoverUnadmittedGenerationHardlinksResponse401 = { - data: RecoverUnadmittedGenerationHardlinks401; - status: 401; -}; - -export type recoverUnadmittedGenerationHardlinksResponse403 = { - data: RecoverUnadmittedGenerationHardlinks403; - status: 403; -}; - -export type recoverUnadmittedGenerationHardlinksResponse409 = { - data: RecoverUnadmittedGenerationHardlinks409; - status: 409; -}; - -export type recoverUnadmittedGenerationHardlinksResponse413 = { - data: RecoverUnadmittedGenerationHardlinks413; - status: 413; -}; - -export type recoverUnadmittedGenerationHardlinksResponse415 = { - data: RecoverUnadmittedGenerationHardlinks415; - status: 415; -}; - -export type recoverUnadmittedGenerationHardlinksResponse429 = { - data: RecoverUnadmittedGenerationHardlinks429; - status: 429; -}; - -export type recoverUnadmittedGenerationHardlinksResponse500 = { - data: RecoverUnadmittedGenerationHardlinks500; - status: 500; -}; - -export type recoverUnadmittedGenerationHardlinksResponse503 = { - data: RecoverUnadmittedGenerationHardlinks503; - status: 503; -}; - -export type recoverUnadmittedGenerationHardlinksResponseSuccess = - recoverUnadmittedGenerationHardlinksResponse200 & { - headers: Headers; - }; -export type recoverUnadmittedGenerationHardlinksResponseError = ( - | recoverUnadmittedGenerationHardlinksResponse400 - | recoverUnadmittedGenerationHardlinksResponse401 - | recoverUnadmittedGenerationHardlinksResponse403 - | recoverUnadmittedGenerationHardlinksResponse409 - | recoverUnadmittedGenerationHardlinksResponse413 - | recoverUnadmittedGenerationHardlinksResponse415 - | recoverUnadmittedGenerationHardlinksResponse429 - | recoverUnadmittedGenerationHardlinksResponse500 - | recoverUnadmittedGenerationHardlinksResponse503 -) & { - headers: Headers; -}; - -export type recoverUnadmittedGenerationHardlinksResponse = - | recoverUnadmittedGenerationHardlinksResponseSuccess - | recoverUnadmittedGenerationHardlinksResponseError; - -export const getRecoverUnadmittedGenerationHardlinksUrl = () => { - return `/executions/generation-hardlinks/recovery`; -}; - -export const recoverUnadmittedGenerationHardlinks = async ( - recoverUnadmittedGenerationHardlinksBody: RecoverUnadmittedGenerationHardlinksBody, - options?: Parameters[1], -): Promise => { - const getHeaders = ( - h?: NonNullable, - ): Record => { - if (!h) return {}; - if (h instanceof Headers) return Object.fromEntries(h.entries()); - if (Symbol.iterator in h) { - return Object.fromEntries( - Array.from( - h as Iterable>, - (entry) => Array.from(entry) as [string, string], - ), - ); - } - const headers: Record = {}; - for (const [name, value] of Object.entries(h)) { - if (value !== undefined) headers[name] = value; - } - return headers; - }; - return recoverUnadmittedGenerationHardlinksRequest( - getRecoverUnadmittedGenerationHardlinksUrl(), - { - ...options, - method: "POST", - headers: { "Content-Type": "application/json", ...getHeaders(options?.headers) }, - body: JSON.stringify(recoverUnadmittedGenerationHardlinksBody), - }, - ); -}; - export type prepareDataConfigurationFenceResponse200 = { data: PrepareDataConfigurationFence200; status: 200; @@ -1209,16 +1088,32 @@ export type inspectDataConfigurationFenceResponse = | inspectDataConfigurationFenceResponseSuccess | inspectDataConfigurationFenceResponseError; -export const getInspectDataConfigurationFenceUrl = (maintenanceId: string) => { - return `/executions/configuration-maintenance/${encodeURIComponent(String(maintenanceId))}`; +export const getInspectDataConfigurationFenceUrl = ( + maintenanceId: string, + params: InspectDataConfigurationFenceParams, +) => { + const normalizedParams = new URLSearchParams(); + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? "null" : String(value)); + } + }); + + const stringifiedParams = normalizedParams.toString(); + + return stringifiedParams.length > 0 + ? `/executions/configuration-maintenance/${encodeURIComponent(String(maintenanceId))}?${stringifiedParams}` + : `/executions/configuration-maintenance/${encodeURIComponent(String(maintenanceId))}`; }; export const inspectDataConfigurationFence = async ( maintenanceId: string, + params: InspectDataConfigurationFenceParams, options?: Parameters[1], ): Promise => { return inspectDataConfigurationFenceRequest( - getInspectDataConfigurationFenceUrl(maintenanceId), + getInspectDataConfigurationFenceUrl(maintenanceId, params), { ...options, method: "GET", diff --git a/packages/core/src/services/data-manager/generated/models/index.ts b/packages/core/src/services/data-manager/generated/models/index.ts index 48004cdc1..203a9b36c 100644 --- a/packages/core/src/services/data-manager/generated/models/index.ts +++ b/packages/core/src/services/data-manager/generated/models/index.ts @@ -503,6 +503,7 @@ export * from "./inspectDataConfigurationFence415"; export * from "./inspectDataConfigurationFence429"; export * from "./inspectDataConfigurationFence500"; export * from "./inspectDataConfigurationFence503"; +export * from "./inspectDataConfigurationFenceParams"; export * from "./inspectTransitSyncPreflight200"; export * from "./inspectTransitSyncPreflight200Result"; export * from "./inspectTransitSyncPreflight200ResultCapacity"; @@ -676,20 +677,16 @@ export * from "./proposeTransitBump502"; export * from "./proposeTransitBump503"; export * from "./proposeTransitBumpBody"; export * from "./recoverUnadmittedConfigurationPreparation200"; -export * from "./recoverUnadmittedConfigurationPreparation200Payload"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadChild"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadChildExecutor"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadChildPrincipal"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadInput"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadKind"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadOrigin"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadOriginIssuer"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadOriginVersion"; -export * from "./recoverUnadmittedConfigurationPreparation200PayloadVersion"; +export * from "./recoverUnadmittedConfigurationPreparation200Child"; +export * from "./recoverUnadmittedConfigurationPreparation200ChildExecutor"; +export * from "./recoverUnadmittedConfigurationPreparation200ChildPrincipal"; +export * from "./recoverUnadmittedConfigurationPreparation200Input"; +export * from "./recoverUnadmittedConfigurationPreparation200InputCoordinatorMode"; +export * from "./recoverUnadmittedConfigurationPreparation200InputOwnerPrepare"; +export * from "./recoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor"; +export * from "./recoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal"; +export * from "./recoverUnadmittedConfigurationPreparation200Kind"; +export * from "./recoverUnadmittedConfigurationPreparation200Version"; export * from "./recoverUnadmittedConfigurationPreparation400"; export * from "./recoverUnadmittedConfigurationPreparation401"; export * from "./recoverUnadmittedConfigurationPreparation403"; @@ -704,34 +701,6 @@ export * from "./recoverUnadmittedConfigurationPreparationBodyCoordinatorMode"; export * from "./recoverUnadmittedConfigurationPreparationBodyOwnerPrepare"; export * from "./recoverUnadmittedConfigurationPreparationBodyOwnerPrepareExecutor"; export * from "./recoverUnadmittedConfigurationPreparationBodyOwnerPreparePrincipal"; -export * from "./recoverUnadmittedGenerationHardlinks200"; -export * from "./recoverUnadmittedGenerationHardlinks200Payload"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadChild"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadChildExecutor"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadChildPrincipal"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadKind"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadOrigin"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadOriginIssuer"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadOriginVersion"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadParent"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadParentExecutor"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadParentPrincipal"; -export * from "./recoverUnadmittedGenerationHardlinks200PayloadVersion"; -export * from "./recoverUnadmittedGenerationHardlinks400"; -export * from "./recoverUnadmittedGenerationHardlinks401"; -export * from "./recoverUnadmittedGenerationHardlinks403"; -export * from "./recoverUnadmittedGenerationHardlinks409"; -export * from "./recoverUnadmittedGenerationHardlinks413"; -export * from "./recoverUnadmittedGenerationHardlinks415"; -export * from "./recoverUnadmittedGenerationHardlinks429"; -export * from "./recoverUnadmittedGenerationHardlinks500"; -export * from "./recoverUnadmittedGenerationHardlinks503"; -export * from "./recoverUnadmittedGenerationHardlinksBody"; -export * from "./recoverUnadmittedGenerationHardlinksBodyInput"; -export * from "./recoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode"; -export * from "./recoverUnadmittedGenerationHardlinksBodyInputKind"; -export * from "./recoverUnadmittedGenerationHardlinksBodyPlan"; -export * from "./recoverUnadmittedGenerationHardlinksBodyPlanPlanItem"; export * from "./reloadDatasets200"; export * from "./reloadDatasets400"; export * from "./reloadDatasets401"; diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts new file mode 100644 index 000000000..5ad8bf9e7 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type InspectDataConfigurationFenceParams = { + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + candidateId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + ownerLeaseId: string; +}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200.ts index 7a4708a28..bf71664a6 100644 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200.ts +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200.ts @@ -1,9 +1,22 @@ // biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. // Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedConfigurationPreparation200Payload } from "./recoverUnadmittedConfigurationPreparation200Payload"; +import type { RecoverUnadmittedConfigurationPreparation200Child } from "./recoverUnadmittedConfigurationPreparation200Child"; +import type { RecoverUnadmittedConfigurationPreparation200Input } from "./recoverUnadmittedConfigurationPreparation200Input"; +import type { RecoverUnadmittedConfigurationPreparation200Kind } from "./recoverUnadmittedConfigurationPreparation200Kind"; +import type { RecoverUnadmittedConfigurationPreparation200Version } from "./recoverUnadmittedConfigurationPreparation200Version"; export type RecoverUnadmittedConfigurationPreparation200 = { - payload: RecoverUnadmittedConfigurationPreparation200Payload; - /** @pattern ^[A-Za-z0-9_-]{86}$ */ - signature: string; + version: RecoverUnadmittedConfigurationPreparation200Version; + kind: RecoverUnadmittedConfigurationPreparation200Kind; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + leaseId: string; + /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ + delegationId: string; + child: RecoverUnadmittedConfigurationPreparation200Child; + input: RecoverUnadmittedConfigurationPreparation200Input; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + observedAt: number; }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Child.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Child.ts new file mode 100644 index 000000000..b86a59d81 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Child.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { RecoverUnadmittedConfigurationPreparation200ChildExecutor } from "./recoverUnadmittedConfigurationPreparation200ChildExecutor"; +import type { RecoverUnadmittedConfigurationPreparation200ChildPrincipal } from "./recoverUnadmittedConfigurationPreparation200ChildPrincipal"; + +export type RecoverUnadmittedConfigurationPreparation200Child = { + executor: RecoverUnadmittedConfigurationPreparation200ChildExecutor; + principal: RecoverUnadmittedConfigurationPreparation200ChildPrincipal; + executionId: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + operationFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildExecutor.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildExecutor.ts new file mode 100644 index 000000000..44839bab5 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildExecutor.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200ChildExecutor = + (typeof RecoverUnadmittedConfigurationPreparation200ChildExecutor)[keyof typeof RecoverUnadmittedConfigurationPreparation200ChildExecutor]; + +export const RecoverUnadmittedConfigurationPreparation200ChildExecutor = { + "ops-agent": "ops-agent", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildPrincipal.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildPrincipal.ts new file mode 100644 index 000000000..a4ba92728 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200ChildPrincipal.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200ChildPrincipal = + (typeof RecoverUnadmittedConfigurationPreparation200ChildPrincipal)[keyof typeof RecoverUnadmittedConfigurationPreparation200ChildPrincipal]; + +export const RecoverUnadmittedConfigurationPreparation200ChildPrincipal = { + api: "api", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInput.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Input.ts similarity index 59% rename from packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInput.ts rename to packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Input.ts index 1f360ff1e..86045a0df 100644 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInput.ts +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Input.ts @@ -1,14 +1,14 @@ // biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. // Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode } from "./recoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare } from "./recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare"; +import type { RecoverUnadmittedConfigurationPreparation200InputCoordinatorMode } from "./recoverUnadmittedConfigurationPreparation200InputCoordinatorMode"; +import type { RecoverUnadmittedConfigurationPreparation200InputOwnerPrepare } from "./recoverUnadmittedConfigurationPreparation200InputOwnerPrepare"; -export type RecoverUnadmittedConfigurationPreparation200PayloadInput = { +export type RecoverUnadmittedConfigurationPreparation200Input = { /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ maintenanceId: string; /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ candidateId: string; - ownerPrepare: RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare; + ownerPrepare: RecoverUnadmittedConfigurationPreparation200InputOwnerPrepare; /** @pattern ^[a-f0-9]{64}$ */ planDigest: string; /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ @@ -20,5 +20,5 @@ export type RecoverUnadmittedConfigurationPreparation200PayloadInput = { * @items.pattern ^[a-z0-9][a-z0-9-]{0,63}$ */ serviceIds: string[]; - coordinatorMode: RecoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode; + coordinatorMode: RecoverUnadmittedConfigurationPreparation200InputCoordinatorMode; }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputCoordinatorMode.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputCoordinatorMode.ts new file mode 100644 index 000000000..e29cc4cba --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputCoordinatorMode.ts @@ -0,0 +1,10 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200InputCoordinatorMode = + (typeof RecoverUnadmittedConfigurationPreparation200InputCoordinatorMode)[keyof typeof RecoverUnadmittedConfigurationPreparation200InputCoordinatorMode]; + +export const RecoverUnadmittedConfigurationPreparation200InputCoordinatorMode = { + "ensure-running": "ensure-running", + "require-running": "require-running", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepare.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepare.ts new file mode 100644 index 000000000..6e1b607d3 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepare.ts @@ -0,0 +1,21 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. +import type { RecoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor } from "./recoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor"; +import type { RecoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal } from "./recoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal"; + +export type RecoverUnadmittedConfigurationPreparation200InputOwnerPrepare = { + executor: RecoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor; + principal: RecoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal; + executionId: string; + /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ + operationKey: string; + /** @pattern ^[a-f0-9]{64}$ */ + operationFingerprint: string; + /** @pattern ^[a-f0-9]{64}$ */ + fingerprint: string; + /** + * @minimum 0 + * @maximum 9007199254740991 + */ + deadlineAt: number; +}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor.ts new file mode 100644 index 000000000..8b585657b --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor = + (typeof RecoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor)[keyof typeof RecoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor]; + +export const RecoverUnadmittedConfigurationPreparation200InputOwnerPrepareExecutor = { + "ops-agent": "ops-agent", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal.ts new file mode 100644 index 000000000..6e14f5d26 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal.ts @@ -0,0 +1,11 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal = + (typeof RecoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal)[keyof typeof RecoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal]; + +export const RecoverUnadmittedConfigurationPreparation200InputOwnerPreparePrincipal = { + api: "api", + "data-manager": "data-manager", + "host-cli": "host-cli", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Kind.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Kind.ts new file mode 100644 index 000000000..fee8cace4 --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Kind.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200Kind = + (typeof RecoverUnadmittedConfigurationPreparation200Kind)[keyof typeof RecoverUnadmittedConfigurationPreparation200Kind]; + +export const RecoverUnadmittedConfigurationPreparation200Kind = { + "configuration-prepare-not-admitted": "configuration-prepare-not-admitted", +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Payload.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Payload.ts deleted file mode 100644 index 0694ce181..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Payload.ts +++ /dev/null @@ -1,37 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedConfigurationPreparation200PayloadChild } from "./recoverUnadmittedConfigurationPreparation200PayloadChild"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadInput } from "./recoverUnadmittedConfigurationPreparation200PayloadInput"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadKind } from "./recoverUnadmittedConfigurationPreparation200PayloadKind"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadOrigin } from "./recoverUnadmittedConfigurationPreparation200PayloadOrigin"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadVersion } from "./recoverUnadmittedConfigurationPreparation200PayloadVersion"; - -export type RecoverUnadmittedConfigurationPreparation200Payload = { - version: RecoverUnadmittedConfigurationPreparation200PayloadVersion; - kind: RecoverUnadmittedConfigurationPreparation200PayloadKind; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - installationId: string; - origin: RecoverUnadmittedConfigurationPreparation200PayloadOrigin; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - leaseId: string; - /** @pattern ^[a-f0-9]{64}$ */ - certificateDigest: string; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - delegationId: string; - /** @pattern ^[a-f0-9]{64}$ */ - grantDigest: string; - child: RecoverUnadmittedConfigurationPreparation200PayloadChild; - input: RecoverUnadmittedConfigurationPreparation200PayloadInput; - /** @pattern ^[a-f0-9]{64}$ */ - adoptionDigest: string; - /** - * @maximum 9007199254740991 - * @exclusiveMinimum 0 - */ - canceledSequence: number; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - observedAt: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChild.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChild.ts deleted file mode 100644 index b82d4e31b..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChild.ts +++ /dev/null @@ -1,21 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedConfigurationPreparation200PayloadChildExecutor } from "./recoverUnadmittedConfigurationPreparation200PayloadChildExecutor"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadChildPrincipal } from "./recoverUnadmittedConfigurationPreparation200PayloadChildPrincipal"; - -export type RecoverUnadmittedConfigurationPreparation200PayloadChild = { - executor: RecoverUnadmittedConfigurationPreparation200PayloadChildExecutor; - principal: RecoverUnadmittedConfigurationPreparation200PayloadChildPrincipal; - executionId: string; - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - operationKey: string; - /** @pattern ^[a-f0-9]{64}$ */ - operationFingerprint: string; - /** @pattern ^[a-f0-9]{64}$ */ - fingerprint: string; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - deadlineAt: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildExecutor.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildExecutor.ts deleted file mode 100644 index f607e1660..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildExecutor.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadChildExecutor = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadChildExecutor)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadChildExecutor]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadChildExecutor = { - "ops-agent": "ops-agent", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildPrincipal.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildPrincipal.ts deleted file mode 100644 index 7fdcfea0e..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadChildPrincipal.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadChildPrincipal = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadChildPrincipal)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadChildPrincipal]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadChildPrincipal = { - api: "api", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode.ts deleted file mode 100644 index 5e04e8e30..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode.ts +++ /dev/null @@ -1,10 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadInputCoordinatorMode = { - "ensure-running": "ensure-running", - "require-running": "require-running", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare.ts deleted file mode 100644 index f1793596a..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare.ts +++ /dev/null @@ -1,21 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor } from "./recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal } from "./recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal"; - -export type RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepare = { - executor: RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor; - principal: RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal; - executionId: string; - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - operationKey: string; - /** @pattern ^[a-f0-9]{64}$ */ - operationFingerprint: string; - /** @pattern ^[a-f0-9]{64}$ */ - fingerprint: string; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - deadlineAt: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor.ts deleted file mode 100644 index 9d649c09b..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPrepareExecutor = { - "ops-agent": "ops-agent", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal.ts deleted file mode 100644 index eb55ded6b..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadInputOwnerPreparePrincipal = { - api: "api", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadKind.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadKind.ts deleted file mode 100644 index cc176aa22..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadKind.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadKind = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadKind)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadKind]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadKind = { - "configuration-prepare-not-admitted": "configuration-prepare-not-admitted", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOrigin.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOrigin.ts deleted file mode 100644 index 67eccff17..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOrigin.ts +++ /dev/null @@ -1,16 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedConfigurationPreparation200PayloadOriginIssuer } from "./recoverUnadmittedConfigurationPreparation200PayloadOriginIssuer"; -import type { RecoverUnadmittedConfigurationPreparation200PayloadOriginVersion } from "./recoverUnadmittedConfigurationPreparation200PayloadOriginVersion"; - -export type RecoverUnadmittedConfigurationPreparation200PayloadOrigin = { - version: RecoverUnadmittedConfigurationPreparation200PayloadOriginVersion; - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - rootKey: string; - issuer: RecoverUnadmittedConfigurationPreparation200PayloadOriginIssuer; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - notAfter: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginIssuer.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginIssuer.ts deleted file mode 100644 index 52e5ecd93..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginIssuer.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadOriginIssuer = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadOriginIssuer)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadOriginIssuer]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadOriginIssuer = { - "host-cli": "host-cli", - "data-manager": "data-manager", - "ops-agent": "ops-agent", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginVersion.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginVersion.ts deleted file mode 100644 index 9a379c947..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadOriginVersion.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadOriginVersion = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadOriginVersion)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadOriginVersion]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadOriginVersion = { - NUMBER_1: 1, -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadVersion.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadVersion.ts deleted file mode 100644 index baa4999a9..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200PayloadVersion.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedConfigurationPreparation200PayloadVersion = - (typeof RecoverUnadmittedConfigurationPreparation200PayloadVersion)[keyof typeof RecoverUnadmittedConfigurationPreparation200PayloadVersion]; - -export const RecoverUnadmittedConfigurationPreparation200PayloadVersion = { - NUMBER_1: 1, -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Version.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Version.ts new file mode 100644 index 000000000..ed101ff8e --- /dev/null +++ b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedConfigurationPreparation200Version.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. +// Generated by pnpm data-manager:sdk:generate. Do not edit. + +export type RecoverUnadmittedConfigurationPreparation200Version = + (typeof RecoverUnadmittedConfigurationPreparation200Version)[keyof typeof RecoverUnadmittedConfigurationPreparation200Version]; + +export const RecoverUnadmittedConfigurationPreparation200Version = { + NUMBER_2: 2, +} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200.ts deleted file mode 100644 index 7b00b165f..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200.ts +++ /dev/null @@ -1,12 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinks200Payload } from "./recoverUnadmittedGenerationHardlinks200Payload"; - -export type RecoverUnadmittedGenerationHardlinks200 = { - payload: RecoverUnadmittedGenerationHardlinks200Payload; - /** - * @minLength 1 - * @maxLength 256 - */ - signature: string; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200Payload.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200Payload.ts deleted file mode 100644 index b346d8433..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200Payload.ts +++ /dev/null @@ -1,30 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinks200PayloadChild } from "./recoverUnadmittedGenerationHardlinks200PayloadChild"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadKind } from "./recoverUnadmittedGenerationHardlinks200PayloadKind"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadOrigin } from "./recoverUnadmittedGenerationHardlinks200PayloadOrigin"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadParent } from "./recoverUnadmittedGenerationHardlinks200PayloadParent"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadVersion } from "./recoverUnadmittedGenerationHardlinks200PayloadVersion"; - -export type RecoverUnadmittedGenerationHardlinks200Payload = { - version: RecoverUnadmittedGenerationHardlinks200PayloadVersion; - kind: RecoverUnadmittedGenerationHardlinks200PayloadKind; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - installationId: string; - origin: RecoverUnadmittedGenerationHardlinks200PayloadOrigin; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - leaseId: string; - /** @pattern ^[a-f0-9]{64}$ */ - certificateDigest: string; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - delegationId: string; - /** @pattern ^[a-f0-9]{64}$ */ - grantDigest: string; - parent: RecoverUnadmittedGenerationHardlinks200PayloadParent; - child: RecoverUnadmittedGenerationHardlinks200PayloadChild; - /** - * @maximum 9007199254740991 - * @exclusiveMinimum 0 - */ - canceledSequence: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChild.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChild.ts deleted file mode 100644 index 0aebc140e..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChild.ts +++ /dev/null @@ -1,21 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinks200PayloadChildExecutor } from "./recoverUnadmittedGenerationHardlinks200PayloadChildExecutor"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadChildPrincipal } from "./recoverUnadmittedGenerationHardlinks200PayloadChildPrincipal"; - -export type RecoverUnadmittedGenerationHardlinks200PayloadChild = { - executor: RecoverUnadmittedGenerationHardlinks200PayloadChildExecutor; - principal: RecoverUnadmittedGenerationHardlinks200PayloadChildPrincipal; - executionId: string; - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - operationKey: string; - /** @pattern ^[a-f0-9]{64}$ */ - operationFingerprint: string; - /** @pattern ^[a-f0-9]{64}$ */ - fingerprint: string; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - deadlineAt: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildExecutor.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildExecutor.ts deleted file mode 100644 index 866c876f4..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildExecutor.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadChildExecutor = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadChildExecutor)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadChildExecutor]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadChildExecutor = { - "ops-agent": "ops-agent", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildPrincipal.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildPrincipal.ts deleted file mode 100644 index 05c4585c8..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadChildPrincipal.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadChildPrincipal = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadChildPrincipal)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadChildPrincipal]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadChildPrincipal = { - api: "api", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadKind.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadKind.ts deleted file mode 100644 index 29e01b69e..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadKind.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadKind = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadKind)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadKind]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadKind = { - "generation-hardlinks-not-admitted": "generation-hardlinks-not-admitted", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOrigin.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOrigin.ts deleted file mode 100644 index f8f987dde..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOrigin.ts +++ /dev/null @@ -1,16 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinks200PayloadOriginIssuer } from "./recoverUnadmittedGenerationHardlinks200PayloadOriginIssuer"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadOriginVersion } from "./recoverUnadmittedGenerationHardlinks200PayloadOriginVersion"; - -export type RecoverUnadmittedGenerationHardlinks200PayloadOrigin = { - version: RecoverUnadmittedGenerationHardlinks200PayloadOriginVersion; - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - rootKey: string; - issuer: RecoverUnadmittedGenerationHardlinks200PayloadOriginIssuer; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - notAfter: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginIssuer.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginIssuer.ts deleted file mode 100644 index 0871733de..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginIssuer.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadOriginIssuer = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadOriginIssuer)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadOriginIssuer]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadOriginIssuer = { - "host-cli": "host-cli", - "data-manager": "data-manager", - "ops-agent": "ops-agent", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginVersion.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginVersion.ts deleted file mode 100644 index 238394eef..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadOriginVersion.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadOriginVersion = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadOriginVersion)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadOriginVersion]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadOriginVersion = { - NUMBER_1: 1, -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParent.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParent.ts deleted file mode 100644 index 2e722a779..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParent.ts +++ /dev/null @@ -1,21 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinks200PayloadParentExecutor } from "./recoverUnadmittedGenerationHardlinks200PayloadParentExecutor"; -import type { RecoverUnadmittedGenerationHardlinks200PayloadParentPrincipal } from "./recoverUnadmittedGenerationHardlinks200PayloadParentPrincipal"; - -export type RecoverUnadmittedGenerationHardlinks200PayloadParent = { - executor: RecoverUnadmittedGenerationHardlinks200PayloadParentExecutor; - principal: RecoverUnadmittedGenerationHardlinks200PayloadParentPrincipal; - executionId: string; - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - operationKey: string; - /** @pattern ^[a-f0-9]{64}$ */ - operationFingerprint: string; - /** @pattern ^[a-f0-9]{64}$ */ - fingerprint: string; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - deadlineAt: number; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentExecutor.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentExecutor.ts deleted file mode 100644 index b004e9e3e..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentExecutor.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadParentExecutor = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadParentExecutor)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadParentExecutor]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadParentExecutor = { - "ops-agent": "ops-agent", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentPrincipal.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentPrincipal.ts deleted file mode 100644 index 09934abf9..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadParentPrincipal.ts +++ /dev/null @@ -1,11 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadParentPrincipal = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadParentPrincipal)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadParentPrincipal]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadParentPrincipal = { - api: "api", - "data-manager": "data-manager", - "host-cli": "host-cli", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadVersion.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadVersion.ts deleted file mode 100644 index 276583574..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks200PayloadVersion.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks200PayloadVersion = - (typeof RecoverUnadmittedGenerationHardlinks200PayloadVersion)[keyof typeof RecoverUnadmittedGenerationHardlinks200PayloadVersion]; - -export const RecoverUnadmittedGenerationHardlinks200PayloadVersion = { - NUMBER_1: 1, -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks400.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks400.ts deleted file mode 100644 index 279cd293b..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks400.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks400 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks401.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks401.ts deleted file mode 100644 index 9ccd027ff..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks401.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks401 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks403.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks403.ts deleted file mode 100644 index cfa6c030e..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks403.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks403 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks409.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks409.ts deleted file mode 100644 index 99a0263c8..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks409.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks409 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks413.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks413.ts deleted file mode 100644 index 6a1effe22..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks413.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks413 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks415.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks415.ts deleted file mode 100644 index 573272e57..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks415.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks415 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks429.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks429.ts deleted file mode 100644 index df9efd480..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks429.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks429 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks500.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks500.ts deleted file mode 100644 index 02c603201..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks500.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks500 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks503.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks503.ts deleted file mode 100644 index 63308a05b..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinks503.ts +++ /dev/null @@ -1,20 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinks503 = - | { - error: string; - ok?: false; - message?: string; - jobId?: string; - } - | { - /** - * @minimum 400 - * @maximum 599 - */ - statusCode: number; - error: string; - message: string; - code?: string; - }; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBody.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBody.ts deleted file mode 100644 index ebb0a442e..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBody.ts +++ /dev/null @@ -1,16 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinksBodyInput } from "./recoverUnadmittedGenerationHardlinksBodyInput"; -import type { RecoverUnadmittedGenerationHardlinksBodyPlan } from "./recoverUnadmittedGenerationHardlinksBodyPlan"; - -export type RecoverUnadmittedGenerationHardlinksBody = { - /** @pattern ^opk1_[A-Za-z0-9_-]{16,64}$ */ - operationKey: string; - /** - * @minimum 0 - * @maximum 9007199254740991 - */ - requestedDeadlineAt: number; - input: RecoverUnadmittedGenerationHardlinksBodyInput; - plan: RecoverUnadmittedGenerationHardlinksBodyPlan; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInput.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInput.ts deleted file mode 100644 index fe8e8a930..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInput.ts +++ /dev/null @@ -1,13 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode } from "./recoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode"; -import type { RecoverUnadmittedGenerationHardlinksBodyInputKind } from "./recoverUnadmittedGenerationHardlinksBodyInputKind"; - -export type RecoverUnadmittedGenerationHardlinksBodyInput = { - kind: RecoverUnadmittedGenerationHardlinksBodyInputKind; - /** @pattern ^cfg1_[A-Za-z0-9_-]{43}$ */ - revisionId: string; - /** @pattern ^[a-f0-9]{64}$ */ - planDigest: string; - coordinatorMode: RecoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode.ts deleted file mode 100644 index 506a13048..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode.ts +++ /dev/null @@ -1,10 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode = - (typeof RecoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode)[keyof typeof RecoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode]; - -export const RecoverUnadmittedGenerationHardlinksBodyInputCoordinatorMode = { - "ensure-running": "ensure-running", - "require-running": "require-running", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputKind.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputKind.ts deleted file mode 100644 index 2f9e2c6e0..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyInputKind.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinksBodyInputKind = - (typeof RecoverUnadmittedGenerationHardlinksBodyInputKind)[keyof typeof RecoverUnadmittedGenerationHardlinksBodyInputKind]; - -export const RecoverUnadmittedGenerationHardlinksBodyInputKind = { - "generation-hardlinks": "generation-hardlinks", -} as const; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlan.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlan.ts deleted file mode 100644 index ba39e78ce..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlan.ts +++ /dev/null @@ -1,9 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. -import type { RecoverUnadmittedGenerationHardlinksBodyPlanPlanItem } from "./recoverUnadmittedGenerationHardlinksBodyPlanPlanItem"; - -export type RecoverUnadmittedGenerationHardlinksBodyPlan = { - /** @maxItems 256 */ - plan: RecoverUnadmittedGenerationHardlinksBodyPlanPlanItem[]; - prune: true; -}; diff --git a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlanPlanItem.ts b/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlanPlanItem.ts deleted file mode 100644 index f777fb6cc..000000000 --- a/packages/core/src/services/data-manager/generated/models/recoverUnadmittedGenerationHardlinksBodyPlanPlanItem.ts +++ /dev/null @@ -1,23 +0,0 @@ -// biome-ignore-all lint: Generated by Orval; edit data-manager contracts and regenerate. -// Generated by pnpm data-manager:sdk:generate. Do not edit. - -export type RecoverUnadmittedGenerationHardlinksBodyPlanPlanItem = { - /** - * @minLength 1 - * @maxLength 1024 - */ - source: string; - /** - * @minLength 1 - * @maxLength 1024 - */ - target: string; - /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ - consumerService: string; - /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ - dataType: string; - /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ - instance?: string; - /** @pattern ^[A-Za-z0-9][A-Za-z0-9._-]*$ */ - targetFilename?: string; -}; diff --git a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts index a45d3da00..59dbed3c4 100644 --- a/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts +++ b/packages/core/src/services/data-manager/generated/operation-bindings.generated.ts @@ -299,15 +299,6 @@ export async function recoverUnadmittedConfigurationPreparationRequest( ): Promise { return recoverUnadmittedConfigurationPreparationBound(url, options); } -const recoverUnadmittedGenerationHardlinksBound = bindDataManagerOperation( - "recoverUnadmittedGenerationHardlinks", -); -export async function recoverUnadmittedGenerationHardlinksRequest( - url: string, - options?: DataManagerRequestOptions, -): Promise { - return recoverUnadmittedGenerationHardlinksBound(url, options); -} const reloadDatasetsBound = bindDataManagerOperation("reloadDatasets"); export async function reloadDatasetsRequest( url: string, diff --git a/packages/core/src/services/data-manager/index.ts b/packages/core/src/services/data-manager/index.ts index 543a145c2..f6fecae94 100644 --- a/packages/core/src/services/data-manager/index.ts +++ b/packages/core/src/services/data-manager/index.ts @@ -1,6 +1,7 @@ export { inspectDataConfigurationFence, prepareDataConfigurationFence, + recoverUnadmittedDataConfigurationFence, settleDataConfigurationFence, } from "./configuration-maintenance"; export type { diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 78c31b0b4..ef8bc1472 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -2,6 +2,8 @@ export { openAdminDataWorkflow } from "./admin-data-workflow"; export { type AgentStatePaths, agentStatePaths, + ensureHostCliToken, + hostCliTokenPath, prepareAgentStateDirectories, } from "./agent-state"; export { prepareAgentStores } from "./agent-stores"; @@ -97,7 +99,10 @@ export { renderConfiguration, withConfigurationGenerationWriter, } from "./configuration-generation"; -export { acquireConfigurationOwnership } from "./configuration-ownership"; +export { + acquireConfigurationOwnership, + configurationWriterLockPath, +} from "./configuration-ownership"; export { assertTrustedConfigurationSchema, validateTrustedConfigurationValues, @@ -234,6 +239,13 @@ export { type PlatformConfigurationPreview, } from "./platform-preview"; export { expandPresets, PRESETS, type PresetName, UnknownPresetError } from "./presets"; +export { + acquireRecoverableLock, + clearRecoverableLock, + LockBusyError, + readLockHolder, + withRecoverableLock, +} from "./recoverable-lock"; export { ServiceRegistry, type ServiceRegistryOptions } from "./registry"; export { type BuiltInServiceReadOptions, diff --git a/packages/core/src/services/recoverable-lock.ts b/packages/core/src/services/recoverable-lock.ts index 597cde8a7..dabf77f7d 100644 --- a/packages/core/src/services/recoverable-lock.ts +++ b/packages/core/src/services/recoverable-lock.ts @@ -136,6 +136,16 @@ export async function lockHolderGone( containerRunning: ContainerRunning = dockerContainerRunning, ): Promise { if (holder.bootId && self.bootId && holder.bootId !== self.bootId) return true; + // Every process of a container shares its PID namespace, so the same container + // in another namespace has restarted since and the holder ended with it. + if ( + holder.containerId && + holder.containerId === self.containerId && + holder.pidNamespace && + self.pidNamespace && + holder.pidNamespace !== self.pidNamespace + ) + return true; if (holder.pidNamespace && holder.pidNamespace === self.pidNamespace) { if (!processAlive(holder.pid)) return true; const startTime = processStartTime(holder.pid); @@ -149,14 +159,17 @@ export async function lockHolderGone( function readHolderFile(path: string): { holder?: LockHolder; ino: number } | undefined { let descriptor: number; try { - descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + // Never block on a FIFO someone placed at the lock path. + descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; } try { const stat = fstatSync(descriptor); - if (!stat.isFile() || stat.size > 4096) return { ino: stat.ino }; + // Waiting never turns another kind of file into a lock; leave it to the operator. + if (!stat.isFile()) throw new Error(`${path} is not a lock file; remove it by hand`); + if (stat.size > 4096) return { ino: stat.ino }; const buffer = Buffer.alloc(stat.size); readSync(descriptor, buffer, 0, buffer.length, 0); try { @@ -217,7 +230,11 @@ export async function acquireRecoverableLock( try { descriptor = openSync( path, - constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, 0o600, ); } catch (error) { diff --git a/packages/core/src/services/service-lifecycle-receipt.test.ts b/packages/core/src/services/service-lifecycle-receipt.test.ts index 856077ad6..f48bdc74a 100644 --- a/packages/core/src/services/service-lifecycle-receipt.test.ts +++ b/packages/core/src/services/service-lifecycle-receipt.test.ts @@ -12,6 +12,7 @@ import { retainAdmittedControlExecution, } from "../build-artifacts/control-authority"; import { AnchoredDirectory } from "../build-artifacts/filesystem"; +import { leaseStoreDirectory, openLeaseStore } from "../build-artifacts/lease-store"; import { acquireArtifactTargetLease, closeArtifactTargetLeaseHandle, @@ -91,7 +92,6 @@ it("signs exact seed and checkpoints only with the live writer and physical targ let lease: Awaited> | undefined; try { lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), owner: o.authority, serviceIds: ["alpha", "beta"], }); @@ -112,7 +112,7 @@ it("signs exact seed and checkpoints only with the live writer and physical targ ), ); expect((await readServiceLifecycleReceipt(options(o.execution))).phase).toBe("pins_validated"); - const path = join(root, "data/ops-agent/lifecycle", `${o.execution.operationKey}.json`), + const path = join(root, "state/ops-agent/lifecycle", `${o.execution.operationKey}.json`), original = readFileSync(path); const tampered = JSON.parse(original.toString()); tampered.current.payload.receipt.finalRevisionId = revision; @@ -155,7 +155,6 @@ it("refuses a final revision that the owned writer did not commit", async () => let lease: Awaited> | undefined; try { lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), owner: o.authority, serviceIds: ["alpha", "beta"], }); @@ -202,7 +201,6 @@ async function withPinsValidatedReceipt( let lease: Awaited> | undefined; try { const held = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), owner: o.authority, serviceIds: ["alpha", "beta"], }); @@ -342,7 +340,6 @@ it.each([false, true])( let restarted: ReturnType | undefined; try { lease = await acquireArtifactTargetLease({ - dataRoot: join(root, "data"), owner: o.authority, serviceIds: ["alpha", "beta"], }); @@ -415,7 +412,11 @@ it.each([false, true])( sequence: 1, }); } else await expect(recovered).rejects.toThrow(); - expect(existsSync(join(root, "data/.admin-target-control/locks/alpha"))).toBe(true); + expect( + openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }) + .list() + .flatMap((value) => value.targets), + ).toContain("alpha"); } finally { if (lease) closeArtifactTargetLeaseHandle(lease); if (restarted) closeControlInstallation(restarted); diff --git a/packages/core/src/services/service-lifecycle-receipt.ts b/packages/core/src/services/service-lifecycle-receipt.ts index 733e50086..87b5af0e8 100644 --- a/packages/core/src/services/service-lifecycle-receipt.ts +++ b/packages/core/src/services/service-lifecycle-receipt.ts @@ -1,5 +1,5 @@ import { AsyncLocalStorage } from "node:async_hooks"; -import { dirname } from "node:path"; +import { dirname, join } from "node:path"; import { z } from "zod"; // biome-ignore-start lint/suspicious/noImportCycles: Recovery reads original receipts only after initialization; nominal writer/lease checks already depend on this control graph. import { @@ -185,8 +185,9 @@ function assertDataRoot(dataRoot: string, trusted: string) { expected.close(); } } +/** Receipts live in the agent's own state, which data-manager does not mount. */ function directories(dataRoot: string) { - const root = new AnchoredDirectory(dataRoot); + const root = new AnchoredDirectory(join(dirname(dataRoot), "state")); const agent = root.directory("ops-agent", true, true), directory = agent.directory("lifecycle", true, true); return { diff --git a/packages/db-schema/src/data-manager.ts b/packages/db-schema/src/data-manager.ts index 96fb18d80..ccc5adb71 100644 --- a/packages/db-schema/src/data-manager.ts +++ b/packages/db-schema/src/data-manager.ts @@ -86,7 +86,8 @@ export const configurationFence = dataManager.table( operationKey: text("operation_key").notNull().unique(), candidateId: uuid("candidate_id").notNull(), planDigest: text("plan_digest").notNull(), - ownerCertificateDigest: text("owner_certificate_digest").notNull(), + /** The agent's maintenance lease the fence was prepared for. */ + ownerLeaseId: uuid("owner_lease_id").notNull(), /** The prepare request body, compared on a repeated prepare. */ input: jsonb("input").notNull(), /** "prepared" | "settled" */ diff --git a/scripts/check-ops-authority.ts b/scripts/check-ops-authority.ts index 45a8b8f12..458fdb2b6 100644 --- a/scripts/check-ops-authority.ts +++ b/scripts/check-ops-authority.ts @@ -180,6 +180,35 @@ const PRODUCTION_SOURCE_ROOTS = [ const DOCKER_INVOCATION = /\b(?:execa|execFile|execFileSync|spawn|spawnSync|exec)\s*\(\s*(['"`])docker\1/; +/** + * Exclusive access to targets is held in the agent's lease store only. The + * shared admission directory on the data volume, which every role could write, + * must not come back in code, tests, manifests or scripts. + */ +const SHARED_ADMISSION_DIRECTORY = [".admin", "target", "control"].join("-"); +const ADMISSION_SCAN_ROOTS = ["apps", "services", "packages", "scripts"]; +const ADMISSION_SCAN_EXTENSIONS = [".ts", ".tsx", ".mts", ".js", ".mjs", ".json", ".sh", ".yml"]; + +function walkAll(directory: string): string[] { + const files: string[] = []; + let entries: string[]; + try { + entries = readdirSync(directory); + } catch { + return files; + } + for (const name of entries) { + const path = join(directory, name); + if (statSync(path).isDirectory()) { + if (["node_modules", "dist", ".turbo", ".next", "coverage"].includes(name)) continue; + files.push(...walkAll(path)); + } else if (ADMISSION_SCAN_EXTENSIONS.some((extension) => name.endsWith(extension))) { + files.push(path); + } + } + return files; +} + export interface AuthorityViolation { file: string; detail: string; @@ -399,6 +428,17 @@ export function findAuthorityViolations(root: string = ROOT): AuthorityViolation } } + for (const scanRoot of ADMISSION_SCAN_ROOTS) { + for (const file of walkAll(join(root, scanRoot))) { + if (readFileSync(file, "utf8").includes(SHARED_ADMISSION_DIRECTORY)) { + violations.push({ + file: relative(root, file), + detail: `names the removed shared admission directory ${SHARED_ADMISSION_DIRECTORY}`, + }); + } + } + } + return violations; } diff --git a/services/data-manager/__tests__/auth.test.ts b/services/data-manager/__tests__/auth.test.ts index 602648170..d898869c6 100644 --- a/services/data-manager/__tests__/auth.test.ts +++ b/services/data-manager/__tests__/auth.test.ts @@ -355,7 +355,6 @@ describe("per-caller route allowlist", () => { "inspectDataConfigurationFence", "settleDataConfigurationFence", "startGenerationHardlinksExecution", - "recoverUnadmittedGenerationHardlinks", ] as const) expect(DATA_MANAGER_API_ROUTES[operationId], operationId).toBe(false); const { app } = await contractApp(); diff --git a/services/data-manager/__tests__/transitous/catalog-review.test.ts b/services/data-manager/__tests__/transitous/catalog-review.test.ts index 7fe84adc9..360b83b08 100644 --- a/services/data-manager/__tests__/transitous/catalog-review.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-review.test.ts @@ -184,7 +184,7 @@ it.skipIf(!enabled)( tokenFile = join(root, "dm-token"); writeFileSync(tokenFile, token, { mode: 0o600 }); agent = buildOpsAgentServer({ - tokens: { api: "a".repeat(43), "data-manager": token }, + tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, runtime: createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: root }), resourcePolicy: createRegistryResourcePolicy({ serviceIds: ["motis"], diff --git a/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts b/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts index 690e14086..aa68c9a8d 100644 --- a/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-validation-current.test.ts @@ -67,7 +67,6 @@ async function fixture(kind: "catalog.validate" | "transit-sync" | "download-fon serviceIds: ids, }), lease = await acquireArtifactTargetLease({ - dataRoot: dataDir, owner: authority, serviceIds: ids, }); diff --git a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts index 72d67211b..acc3ef789 100644 --- a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts @@ -12,7 +12,7 @@ import { writeFileSync, } from "node:fs"; import { createRequire } from "node:module"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { createInterface } from "node:readline"; import { promisify } from "node:util"; import * as control from "@openmapx/core/build-artifacts"; @@ -765,7 +765,7 @@ it const [row] = await state.db .sql`SELECT finished_at FROM data_manager.jobs WHERE id=${validation.id}`; expect(row!.finished_at).toBeNull(); - expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); return; } expect( @@ -785,7 +785,7 @@ it await vi.waitFor( () => expect(() => - control.verifyDataControlTerminalWitness(installation!, proof.sourceExecution), + control.readDataControlTerminalObservation(installation!, proof.sourceExecution), ).not.toThrow(), { timeout: 10000, interval: 100 }, ); @@ -798,16 +798,20 @@ it proof.sourceExecution, ); expect( - control.inspectRetainedOwnedArtifactTargetChildSettlement(originalOwner, child.execution) - ?.outcome, + ( + await control.inspectRetainedOwnedArtifactTargetChildSettlement( + originalOwner, + child.execution, + ) + )?.outcome, ).toBe("committed"); - expect(() => - control.inspectRetainedOwnedArtifactTargetChildSettlement(originalOwner, { + await expect( + await control.inspectRetainedOwnedArtifactTargetChildSettlement(originalOwner, { ...child.execution, executionId: randomUUID(), }), - ).toThrow(/original child/); - expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(false); + ).rejects.toThrow(/original child/); + expect(control.heldTargets(dirname(data)).includes("motis")).toBe(false); const validationBytes = readFileSync( join(data, "motis/proposals", proposalId, `validation-${validation.id}.json`), ); @@ -869,7 +873,7 @@ it ); expect(await cancelResponse).toBe(true); expect(lostCancellationResponse).toBe(true); - expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); } if (nativeDm) { // Freeze between lifecycle writes before injecting the pin-pointer crash. @@ -886,8 +890,8 @@ it expect( existsSync( join( - data, - ".admin-target-control/executions/data-manager", + dirname(data), + "control-private/data-manager/execution-witnesses/data-manager", `${activation.operationKey}.writer`, ), ), @@ -935,7 +939,7 @@ it const [row] = await state.db .sql`SELECT finished_at FROM data_manager.jobs WHERE id=${activation.id}`; expect(row!.finished_at).toBeNull(); - expect(existsSync(join(data, ".admin-target-control/locks/motis"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); const next = await sdk.startDataExecution( { input: { kind: "catalog.validate", proposalId, countries: ["de"] }, diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 1cfd884b0..9f5eac1cc 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -3599,913 +3599,6 @@ } } }, - "/executions/generation-hardlinks/recovery": { - "post": { - "operationId": "recoverUnadmittedGenerationHardlinks", - "tags": [ - "data-manager" - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "operationKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "requestedDeadlineAt": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "input": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "generation-hardlinks" - ] - }, - "revisionId": { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - "planDigest": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "coordinatorMode": { - "type": "string", - "enum": [ - "ensure-running", - "require-running" - ] - } - }, - "required": [ - "kind", - "revisionId", - "planDigest", - "coordinatorMode" - ], - "additionalProperties": false - }, - "plan": { - "type": "object", - "properties": { - "plan": { - "maxItems": 256, - "type": "array", - "items": { - "type": "object", - "properties": { - "source": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "target": { - "type": "string", - "minLength": 1, - "maxLength": 1024 - }, - "consumerService": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" - }, - "dataType": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" - }, - "instance": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" - }, - "targetFilename": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" - } - }, - "required": [ - "source", - "target", - "consumerService", - "dataType" - ], - "additionalProperties": false - } - }, - "prune": { - "type": "boolean", - "enum": [ - true - ] - } - }, - "required": [ - "plan", - "prune" - ], - "additionalProperties": false - } - }, - "required": [ - "operationKey", - "requestedDeadlineAt", - "input", - "plan" - ], - "additionalProperties": false - } - } - } - }, - "security": [ - { - "serviceBearer": [] - }, - { - "serviceToken": [] - } - ], - "x-openmapx-contract-id": "recoverUnadmittedGenerationHardlinks", - "x-openmapx-auth": "service", - "x-openmapx-api-caller": false, - "responses": { - "200": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "payload": { - "type": "object", - "properties": { - "version": { - "type": "number", - "enum": [ - 1 - ] - }, - "kind": { - "type": "string", - "enum": [ - "generation-hardlinks-not-admitted" - ] - }, - "installationId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "origin": { - "type": "object", - "properties": { - "version": { - "type": "number", - "enum": [ - 1 - ] - }, - "rootKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "issuer": { - "type": "string", - "enum": [ - "host-cli", - "data-manager", - "ops-agent" - ] - }, - "notAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "version", - "rootKey", - "issuer", - "notAfter" - ], - "additionalProperties": false - }, - "leaseId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "certificateDigest": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "delegationId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "grantDigest": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "parent": { - "type": "object", - "properties": { - "executor": { - "type": "string", - "enum": [ - "ops-agent", - "data-manager", - "host-cli" - ] - }, - "principal": { - "type": "string", - "enum": [ - "api", - "data-manager", - "host-cli" - ] - }, - "executionId": { - "type": "string" - }, - "operationKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "operationFingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "fingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "deadlineAt": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "executor", - "principal", - "executionId", - "operationKey", - "operationFingerprint", - "fingerprint", - "deadlineAt" - ], - "additionalProperties": false - }, - "child": { - "type": "object", - "properties": { - "executor": { - "type": "string", - "enum": [ - "ops-agent", - "data-manager", - "host-cli" - ] - }, - "principal": { - "type": "string", - "enum": [ - "api", - "data-manager", - "host-cli" - ] - }, - "executionId": { - "type": "string" - }, - "operationKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "operationFingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "fingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "deadlineAt": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "executor", - "principal", - "executionId", - "operationKey", - "operationFingerprint", - "fingerprint", - "deadlineAt" - ], - "additionalProperties": false - }, - "canceledSequence": { - "type": "integer", - "exclusiveMinimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "version", - "kind", - "installationId", - "origin", - "leaseId", - "certificateDigest", - "delegationId", - "grantDigest", - "parent", - "child", - "canceledSequence" - ], - "additionalProperties": false - }, - "signature": { - "type": "string", - "minLength": 1, - "maxLength": 256 - } - }, - "required": [ - "payload", - "signature" - ], - "additionalProperties": false - } - } - } - }, - "400": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "401": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "403": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "409": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "413": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "415": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "429": { - "description": "Response", - "headers": { - "retry-after": { - "schema": { - "type": "string", - "pattern": "^[1-9]\\d*$" - }, - "description": "Required response header" - } - }, - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "500": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - }, - "503": { - "description": "Response", - "content": { - "application/json": { - "schema": { - "anyOf": [ - { - "type": "object", - "properties": { - "error": { - "type": "string" - }, - "ok": { - "type": "boolean", - "enum": [ - false - ] - }, - "message": { - "type": "string" - }, - "jobId": { - "type": "string" - } - }, - "required": [ - "error" - ], - "additionalProperties": false - }, - { - "type": "object", - "properties": { - "statusCode": { - "type": "integer", - "minimum": 400, - "maximum": 599 - }, - "error": { - "type": "string" - }, - "message": { - "type": "string" - }, - "code": { - "type": "string" - } - }, - "required": [ - "statusCode", - "error", - "message" - ], - "additionalProperties": false - } - ] - } - } - } - } - } - } - }, "/executions/configuration-maintenance/prepare": { "post": { "operationId": "prepareDataConfigurationFence", @@ -5460,80 +4553,93 @@ "schema": { "type": "object", "properties": { - "payload": { + "version": { + "type": "number", + "enum": [ + 2 + ] + }, + "kind": { + "type": "string", + "enum": [ + "configuration-prepare-not-admitted" + ] + }, + "leaseId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "delegationId": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "child": { "type": "object", "properties": { - "version": { - "type": "number", + "executor": { + "type": "string", "enum": [ - 1 + "ops-agent", + "data-manager", + "host-cli" ] }, - "kind": { + "principal": { "type": "string", "enum": [ - "configuration-prepare-not-admitted" + "api", + "data-manager", + "host-cli" ] }, - "installationId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + "executionId": { + "type": "string" }, - "origin": { - "type": "object", - "properties": { - "version": { - "type": "number", - "enum": [ - 1 - ] - }, - "rootKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "issuer": { - "type": "string", - "enum": [ - "host-cli", - "data-manager", - "ops-agent" - ] - }, - "notAfter": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "version", - "rootKey", - "issuer", - "notAfter" - ], - "additionalProperties": false + "operationKey": { + "type": "string", + "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" }, - "leaseId": { + "operationFingerprint": { "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + "pattern": "^[a-f0-9]{64}$" }, - "certificateDigest": { + "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, - "delegationId": { + "deadlineAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "executor", + "principal", + "executionId", + "operationKey", + "operationFingerprint", + "fingerprint", + "deadlineAt" + ], + "additionalProperties": false + }, + "input": { + "type": "object", + "properties": { + "maintenanceId": { "type": "string", "format": "uuid", "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "grantDigest": { + "candidateId": { "type": "string", - "pattern": "^[a-f0-9]{64}$" + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" }, - "child": { + "ownerPrepare": { "type": "object", "properties": { "executor": { @@ -5584,158 +4690,68 @@ ], "additionalProperties": false }, - "input": { - "type": "object", - "properties": { - "maintenanceId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "candidateId": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "ownerPrepare": { - "type": "object", - "properties": { - "executor": { - "type": "string", - "enum": [ - "ops-agent", - "data-manager", - "host-cli" - ] - }, - "principal": { - "type": "string", - "enum": [ - "api", - "data-manager", - "host-cli" - ] - }, - "executionId": { - "type": "string" - }, - "operationKey": { - "type": "string", - "pattern": "^opk1_[A-Za-z0-9_-]{16,64}$" - }, - "operationFingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "fingerprint": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - "deadlineAt": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "required": [ - "executor", - "principal", - "executionId", - "operationKey", - "operationFingerprint", - "fingerprint", - "deadlineAt" - ], - "additionalProperties": false - }, - "planDigest": { + "planDigest": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revisionId": { + "type": "string", + "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" + }, + "hardlinkPlanDigest": { + "anyOf": [ + { "type": "string", "pattern": "^[a-f0-9]{64}$" }, - "revisionId": { - "type": "string", - "pattern": "^cfg1_[A-Za-z0-9_-]{43}$" - }, - "hardlinkPlanDigest": { - "anyOf": [ - { - "type": "string", - "pattern": "^[a-f0-9]{64}$" - }, - { - "type": "null" - } - ] - }, - "serviceIds": { - "minItems": 1, - "maxItems": 256, - "type": "array", - "items": { - "type": "string", - "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" - } - }, - "coordinatorMode": { - "type": "string", - "enum": [ - "ensure-running", - "require-running" - ] + { + "type": "null" } - }, - "required": [ - "maintenanceId", - "candidateId", - "ownerPrepare", - "planDigest", - "revisionId", - "hardlinkPlanDigest", - "serviceIds", - "coordinatorMode" - ], - "additionalProperties": false - }, - "adoptionDigest": { - "type": "string", - "pattern": "^[a-f0-9]{64}$" + ] }, - "canceledSequence": { - "type": "integer", - "exclusiveMinimum": 0, - "maximum": 9007199254740991 + "serviceIds": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + } }, - "observedAt": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "coordinatorMode": { + "type": "string", + "enum": [ + "ensure-running", + "require-running" + ] } }, "required": [ - "version", - "kind", - "installationId", - "origin", - "leaseId", - "certificateDigest", - "delegationId", - "grantDigest", - "child", - "input", - "adoptionDigest", - "canceledSequence", - "observedAt" - ], - "additionalProperties": false - }, - "signature": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{86}$" + "maintenanceId", + "candidateId", + "ownerPrepare", + "planDigest", + "revisionId", + "hardlinkPlanDigest", + "serviceIds", + "coordinatorMode" + ], + "additionalProperties": false + }, + "observedAt": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "required": [ - "payload", - "signature" + "version", + "kind", + "leaseId", + "delegationId", + "child", + "input", + "observedAt" ], "additionalProperties": false } @@ -6301,6 +5317,26 @@ "data-manager" ], "parameters": [ + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "query", + "name": "candidateId", + "required": true + }, + { + "schema": { + "type": "string", + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" + }, + "in": "query", + "name": "ownerLeaseId", + "required": true + }, { "schema": { "type": "string", diff --git a/services/data-manager/src/jobs/execution/api.test.ts b/services/data-manager/src/jobs/execution/api.test.ts index 0848b9f24..c87f9e5c8 100644 --- a/services/data-manager/src/jobs/execution/api.test.ts +++ b/services/data-manager/src/jobs/execution/api.test.ts @@ -24,21 +24,6 @@ it("keeps observation and mutation routes closed without a durable receiver", as url: "/executions", payload: { operationKey: "opk1_abcdefghijklmnop", input: { kind: "download-fonts" } }, }, - { - method: "POST" as const, - url: "/executions/generation-hardlinks/recovery", - payload: { - operationKey: "opk1_abcdefghijklmnop", - requestedDeadlineAt: Date.now() + 60000, - input: { - kind: "generation-hardlinks", - revisionId: `cfg1_${"a".repeat(43)}`, - planDigest: "a".repeat(64), - coordinatorMode: "ensure-running", - }, - plan: { plan: [], prune: true }, - }, - }, { method: "GET" as const, url: `/executions/${id}` }, { method: "GET" as const, url: "/executions/by-key/opk1_abcdefghijklmnop" }, { @@ -46,7 +31,10 @@ it("keeps observation and mutation routes closed without a durable receiver", as url: `/executions/${id}/cancel`, payload: { operationKey: "opk1_abcdefghijklmnop", fingerprint: "a".repeat(64) }, }, - { method: "GET" as const, url: `/executions/configuration-maintenance/${id}` }, + { + method: "GET" as const, + url: `/executions/configuration-maintenance/${id}?candidateId=${id}&ownerLeaseId=${id}`, + }, ]) { const result = await app.inject({ ...request, headers }); expect(result.statusCode, result.body).toBe(503); diff --git a/services/data-manager/src/jobs/execution/api.ts b/services/data-manager/src/jobs/execution/api.ts index 2e09a1a24..34f466902 100644 --- a/services/data-manager/src/jobs/execution/api.ts +++ b/services/data-manager/src/jobs/execution/api.ts @@ -6,7 +6,6 @@ import type { DataExecutionIdentity, DataExecutionInput, DataExecutionStatus, - generationHardlinksNonAdmissionSchema, startGenerationHardlinksExecutionSchema, } from "@openmapx/core/data-manager-contracts"; import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; @@ -15,17 +14,14 @@ import { requestPrincipal } from "../../auth.js"; import type { DataManagerPrincipal } from "../../principals.js"; import { registerDataManagerRoute } from "../../route-contract.js"; -/** The receiver performs signed admission/adoption, separately from bearer +/** The receiver checks the delegation or parent a caller names, after bearer * authentication. A missing receiver never authorizes a compatibility worker. */ export interface DataExecutionReceiver { recoverUnadmittedPreparation?( request: DataConfigurationFencePrepare, rawHeaders: readonly string[], + transportPrincipal: DataManagerPrincipal, ): Promise>; - attestUnadoptedHardlinks?( - request: z.infer, - rawHeaders: readonly string[], - ): Promise>; observeReviewedTransit?(request: { key: string; countries: string[]; @@ -67,7 +63,8 @@ export interface DataExecutionReceiver { ): Promise; inspectFence( maintenanceId: string, - rawHeaders: readonly string[], + expected: { candidateId: string; ownerLeaseId: string }, + transportPrincipal: DataManagerPrincipal, ): Promise; settleFence( input: DataConfigurationFenceSettle, @@ -107,10 +104,6 @@ export function registerDataExecutionApi( await receiver.admitHardlinks(request.body, request.raw.rawHeaders, principal(request)), ); }); - registerDataManagerRoute(app, "recoverUnadmittedGenerationHardlinks", async (request, reply) => { - if (!receiver?.attestUnadoptedHardlinks) return unavailable(reply); - return receiver.attestUnadoptedHardlinks(request.body, request.raw.rawHeaders); - }); registerDataManagerRoute(app, "prepareDataConfigurationFence", async (request, reply) => { if (!receiver) return unavailable(reply); return receiver.prepareFence(request.body, request.raw.rawHeaders, principal(request)); @@ -120,14 +113,19 @@ export function registerDataExecutionApi( "recoverUnadmittedConfigurationPreparation", async (request, reply) => { if (!receiver?.recoverUnadmittedPreparation) return unavailable(reply); - return receiver.recoverUnadmittedPreparation(request.body, request.raw.rawHeaders); + return receiver.recoverUnadmittedPreparation( + request.body, + request.raw.rawHeaders, + principal(request), + ); }, ); registerDataManagerRoute(app, "inspectDataConfigurationFence", async (request, reply) => { if (!receiver) return unavailable(reply); const receipt = await receiver.inspectFence( request.params.maintenanceId, - request.raw.rawHeaders, + request.query, + principal(request), ); return ( receipt ?? reply.code(404).send({ error: "Configuration maintenance identity is unknown" }) diff --git a/services/data-manager/src/jobs/execution/dispatch.ts b/services/data-manager/src/jobs/execution/dispatch.ts index f89b076ed..9954d97f9 100644 --- a/services/data-manager/src/jobs/execution/dispatch.ts +++ b/services/data-manager/src/jobs/execution/dispatch.ts @@ -55,6 +55,7 @@ import { buildJobContext, runTransitousPipeline } from "../transitous/pipeline.j import type { SingleFlightController } from "../transitous/single-flight.js"; import { dataExecutionDiagnostic } from "./diagnostics.js"; import { runOwnedDataOpsOperation } from "./ops-children.js"; +import { ownedLease } from "./owned-lease.js"; import type { DataExecutionContext } from "./runtime.js"; import { publishOwnedTransitSourceMutation } from "./source-mutation.js"; import { @@ -177,7 +178,7 @@ export function createDataExecutionDispatcher(deps: { ); try { await publishDataChange(() => - publishCatalogValidationCurrent(context.authority, context.targetAuthority), + publishCatalogValidationCurrent(context.authority, ownedLease(context)), ); await isolateOwnedTransitProducer( context, diff --git a/services/data-manager/src/jobs/execution/fixed-cli.test.ts b/services/data-manager/src/jobs/execution/fixed-cli.test.ts index d33bfdc4c..3b0a47597 100644 --- a/services/data-manager/src/jobs/execution/fixed-cli.test.ts +++ b/services/data-manager/src/jobs/execution/fixed-cli.test.ts @@ -11,7 +11,7 @@ import { } from "node:fs"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey, @@ -184,7 +184,7 @@ it.skipIf( runtime: agentRuntime, journal: agentJournal, controlInstallation: agentInstallation, - tokens: { api: token, "data-manager": "d".repeat(43) }, + tokens: { api: token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, }); await agent.listen({ host: "127.0.0.1", port: 0 }); @@ -240,12 +240,12 @@ it.skipIf( expect(rows[0]?.status.status).toBe("ok"); expect(rows[0]?.parent?.issuer).toBe("host-cli"); expect(rows[0]?.execution.principal).toBe("host-cli"); - control.verifyDataControlTerminalWitness( + control.readDataControlTerminalObservation( agentInstallation, rows[0]!.execution, rows[0]!.status, ); - expect(existsSync(join(data, ".admin-target-control/locks/data-manager"))).toBe(false); + expect(control.heldTargets(dirname(data)).includes("data-manager")).toBe(false); } finally { await runtime.close(); agent.server.closeAllConnections(); diff --git a/services/data-manager/src/jobs/execution/full-update.test.ts b/services/data-manager/src/jobs/execution/full-update.test.ts index 97b518c8f..1d7438d84 100644 --- a/services/data-manager/src/jobs/execution/full-update.test.ts +++ b/services/data-manager/src/jobs/execution/full-update.test.ts @@ -13,7 +13,7 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey, OpsClient, type OpsOperation } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; @@ -121,8 +121,8 @@ it.skipIf( journal = control.createDataExecutionControlJournal(dm, sql), fence = createDataConfigurationMaintenance(sql, dm); const publicationFailures: Record = {}; - const publishContained = journal.publishContainedTerminal; - journal.publishContainedTerminal = async (execution) => { + const publishContained = journal.recordContainedTerminal; + journal.recordContainedTerminal = async (execution) => { try { return await publishContained(execution); } catch (error) { @@ -249,7 +249,7 @@ it.skipIf( runtime: host.runtime, journal: agentJournal, controlInstallation: agentInstallation, - tokens: { api: token, "data-manager": "d".repeat(43) }, + tokens: { api: token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, resourceClaimer: createProductionRegistryResourceClaimer({ services: host.loaded.map((s) => ({ serviceId: s.manifest.id, @@ -322,7 +322,7 @@ it.skipIf( expect(existsSync(commands) ? readFileSync(commands, "utf8") : "").not.toContain( "osrm-extract", ); - expect(existsSync(join(data, ".admin-target-control/locks/osrm"))).toBe(false); + expect(control.heldTargets(dirname(data)).includes("osrm")).toBe(false); releaseTransit(); const until = Date.now() + 90000; let terminal = await client.status(operation.kind, operationId!); @@ -353,7 +353,7 @@ it.skipIf( expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe( host.initial.revisionId, ); - expect(existsSync(join(data, ".admin-target-control/locks/osrm"))).toBe(false); + expect(control.heldTargets(dirname(data)).includes("osrm")).toBe(false); } catch (error) { const witnessProblems: Record = {}; for (const row of await store.listUnsettled()) { @@ -368,7 +368,7 @@ it.skipIf( ) continue; try { - control.verifyDataControlTerminalWitness(agentInstallation, row.execution, row.status); + control.readDataControlTerminalObservation(agentInstallation, row.execution, row.status); witnessProblems[row.input.kind] = "verified"; } catch (problem) { witnessProblems[row.input.kind] = diff --git a/services/data-manager/src/jobs/execution/hardlinks.test.ts b/services/data-manager/src/jobs/execution/hardlinks.test.ts index be21efc32..2750f3a0b 100644 --- a/services/data-manager/src/jobs/execution/hardlinks.test.ts +++ b/services/data-manager/src/jobs/execution/hardlinks.test.ts @@ -10,7 +10,7 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { createDataManagerTransport, @@ -135,7 +135,6 @@ it plan: [slot], }), lease = await control.acquireArtifactTargetLease({ - dataRoot: data, serviceIds: services, owner: authority, }), @@ -246,9 +245,8 @@ it .toBe("unknown"); await runtime.close(); const recovered = await journal.restoreCompletion(child); - expect(recovered.lease.mode).toBe("borrowed"); - expect(() => control.assertArtifactTargetLeaseForward(recovered.lease)).toThrow(); - control.closeArtifactTargetLeaseHandle(recovered.lease); + // The hardlink leaf holds no lease of its own; its parent's lease fences the target. + expect(recovered.lease).toBeUndefined(); restarted = createDataExecutionRuntime({ store, dispatch: async () => { @@ -261,7 +259,7 @@ it timeout: 15000, }) .toBe("ok"); - expect(existsSync(join(data, ".admin-target-control/locks/data-manager"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("data-manager")).toBe(true); } const terminal = await observeDataExecution( { id: admitted.id, operationKey: child.operationKey, fingerprint: child.fingerprint }, @@ -271,11 +269,11 @@ it status: "ok", result: { kind: "generation-hardlinks", hardlinks: { linked: 1, via: "data-manager" } }, }); - await expect - .poll(() => control.inspectArtifactTargetLeaseChildSettlement(lease, child)?.publication, { - timeout: 10000, - }) - .toBe("committed"); + // The parent settles its data-manager child from the authenticated status. + await control.settleDataManagerLeaseChild(lease, child, terminal); + expect( + (await control.inspectArtifactTargetLeaseChildSettlement(lease, child))?.publication, + ).toBe("committed"); const inode = statSync(join(data, "consumer/fixture")).ino; unlinkSync(join(data, "src/fixture")); const observed = unwrapDataManagerResponse( diff --git a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts index 02f0895b0..d8d2e4900 100644 --- a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts +++ b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts @@ -4,10 +4,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { createCatalogPinSet } from "@openmapx/core/catalog-pins"; -import { generationHardlinksNonAdmissionSchema } from "@openmapx/core/data-manager-contracts"; import { OpsClient } from "@openmapx/core/ops"; import { afterAll, beforeAll, expect, it, vi } from "vitest"; -import { failedServiceLifecycle } from "../../../../../packages/core/src/build-artifacts/test-fixtures/service-lifecycle-recovery"; import { setupCron } from "../../cron"; import { db, rawSql } from "../../db"; import { resetOpsClient, runOpsOperation } from "../../ops-client"; @@ -303,98 +301,6 @@ it.skipIf(!enabled)("leaves rows that carry no operation ledger or are still run for (const id of owned) await rawSql`DELETE FROM data_manager.jobs WHERE id=${id}`; } }); -it.skipIf(!enabled)( - "attests non-admission from the original canceled grant and actual locked SQL, never from a missing shared file alone", - async () => { - const fixture = await failedServiceLifecycle(root); - const owner = control.restoreArtifactTargetLeaseForSettlement(fixture.authority); - const clock = vi.spyOn(Date, "now").mockReturnValue(fixture.execution.deadlineAt + 1); - const request = { - operationKey: fixture.child.execution.operationKey, - requestedDeadlineAt: fixture.child.execution.deadlineAt, - input: { - kind: "generation-hardlinks" as const, - revisionId: fixture.revisionId, - coordinatorMode: "ensure-running" as const, - planDigest: control.controlDigest(fixture.plan), - }, - plan: fixture.plan, - }; - const rawHeaders = ["x-openmapx-control-delegation", fixture.assertion]; - let rowId: string | undefined; - try { - await expect(fence.attestUnadoptedGenerationHardlinks(request, rawHeaders)).rejects.toThrow(); - await control.prepareFailedServiceLifecycleRecovery(fixture.authority, owner); - const proof = await fence.attestUnadoptedGenerationHardlinks(request, rawHeaders); - expect(proof.payload.child).toEqual(fixture.child.execution); - expect(proof.payload.parent).toEqual(fixture.execution); - expect(proof.payload.kind).toBe("generation-hardlinks-not-admitted"); - const routed = await legacyFixture(); - try { - const url = "/executions/generation-hardlinks/recovery"; - expect( - (await routed.app.inject({ method: "POST", url, payload: request })).statusCode, - ).toBe(401); - const response = await routed.app.inject({ - method: "POST", - url, - payload: request, - headers: { ...headers, "x-openmapx-control-delegation": fixture.assertion }, - }); - expect(response.statusCode).toBe(200); - expect(generationHardlinksNonAdmissionSchema.parse(response.json())).toEqual(proof); - expect(routed.dispatches()).toBe(0); - } finally { - await routed.close(); - } - await expect( - fence.attestUnadoptedGenerationHardlinks( - { ...request, requestedDeadlineAt: request.requestedDeadlineAt + 1 }, - rawHeaders, - ), - ).rejects.toThrow("Original hardlink request changed"); - const blocker = await rawSql.reserve(); - try { - await blocker.unsafe("SELECT pg_advisory_lock($1,$2)", [0x4f4d58, 2]); - await expect( - fence.attestUnadoptedGenerationHardlinks(request, rawHeaders), - ).rejects.toMatchObject({ statusCode: 409 }); - } finally { - await blocker.unsafe("SELECT pg_advisory_unlock($1,$2)", [0x4f4d58, 2]); - blocker.release(); - } - const privatePath = join( - root, - "control-private/data-manager/executions", - `${request.operationKey}.json`, - ); - mkdirSync(join(root, "control-private/data-manager/executions"), { - recursive: true, - mode: 0o700, - }); - writeFileSync(privatePath, "{}", { mode: 0o600 }); - try { - await expect(fence.attestUnadoptedGenerationHardlinks(request, rawHeaders)).rejects.toThrow( - "Original receiver admission or adoption requires recovery", - ); - } finally { - rmSync(privatePath); - } - const [row] = - await rawSql`INSERT INTO data_manager.jobs(id,kind,status,operation_key,finished_at) VALUES(${fixture.child.execution.executionId},'generation-hardlinks','error',${fixture.child.execution.operationKey},now()) RETURNING id`; - rowId = String(row!.id); - await expect(fence.attestUnadoptedGenerationHardlinks(request, rawHeaders)).rejects.toThrow( - /admitted|busy/, - ); - } finally { - clock.mockRestore(); - if (rowId) await rawSql`DELETE FROM data_manager.jobs WHERE id=${rowId}`; - control.closeArtifactTargetLeaseHandle(owner); - for (const i of fixture.installations) control.closeControlInstallation(i); - } - }, - 30000, -); it.skipIf(!enabled)( "I4 records actual missing-catalog preflight as terminal error and permits the first subsequent admission", async () => { @@ -1121,7 +1027,11 @@ it.skipIf(!enabled)( old = envNames.map((name) => process.env[name]); const token = Buffer.alloc(32, 5).toString("base64url"); const server = buildOpsAgentServer({ - tokens: { api: Buffer.alloc(32, 6).toString("base64url"), "data-manager": token }, + tokens: { + api: Buffer.alloc(32, 6).toString("base64url"), + "data-manager": token, + "host-cli": Buffer.alloc(32, 7).toString("base64url"), + }, controlInstallation: agent, runtime: createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: repository }), resourcePolicy: createRegistryResourcePolicy({ @@ -1216,7 +1126,7 @@ it.skipIf(!enabled)( expect((await f.runtime.inspect(id))?.publication).toBe("not_started"); // Terminal status is published before asynchronous lease settlement finishes. await expect - .poll(() => fs.existsSync(join(root, "data/.admin-target-control/locks/motis")), { + .poll(() => control.heldTargets(root).includes("motis"), { timeout: 10000, }) .toBe(false); diff --git a/services/data-manager/src/jobs/execution/ops-children.test.ts b/services/data-manager/src/jobs/execution/ops-children.test.ts index 0225ac6f7..f34b1eec9 100644 --- a/services/data-manager/src/jobs/execution/ops-children.test.ts +++ b/services/data-manager/src/jobs/execution/ops-children.test.ts @@ -2,7 +2,7 @@ import { spawn } from "node:child_process"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { createInterface } from "node:readline"; import * as control from "@openmapx/core/build-artifacts"; import { @@ -18,6 +18,14 @@ import { build } from "esbuild"; import { afterAll, beforeAll, expect, it, vi } from "vitest"; import { transitChildCandidateId } from "./ops-children.js"; +/** The lease holding `target` in the installation whose data root is `data`. */ +function targetLease(data: string, target: string): string | undefined { + return control + .openLeaseStore({ directory: control.leaseStoreDirectory(dirname(data)), purpose: "test" }) + .list() + .find((lease) => lease.targets.includes(target))?.leaseId; +} + let bundle: string, bundleRoot: string; beforeAll(async () => { bundleRoot = mkdtempSync(join(tmpdir(), "omx-data-agent-bundle-")); @@ -111,7 +119,6 @@ it.each(["promotion", "reload"] as const)( plan: [plan], }); const lease = await control.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds, owner: authority, }); @@ -135,6 +142,7 @@ it.each(["promotion", "reload"] as const)( const assertion = await control.delegateArtifactTargetLease(lease, { ...plan, childPublicKey: agentVerifier.publicKey, + evidence: control.publishedDataTransitCandidate(authority, { phase, ordinal: 0 }), }); const token = "b".repeat(43), tokenFile = join(root, "token"); @@ -438,7 +446,7 @@ it.skipIf( const pending = await store.inspect(id); expect(pending?.status.status).toBe("cancel_pending"); expect(pending?.finishedAt).toBeNull(); - expect(existsSync(join(root, "data/.admin-target-control/locks/motis"))).toBe(true); + expect(control.heldTargets(root).includes("motis")).toBe(true); send({ type: "probe" }); expect(await next()).toMatchObject({ forward: false, @@ -456,7 +464,7 @@ it.skipIf( changed: true, }); await expect - .poll(() => existsSync(join(root, "data/.admin-target-control/locks/motis")), { + .poll(() => control.heldTargets(root).includes("motis"), { timeout: 10000, }) .toBe(false); @@ -764,8 +772,9 @@ it { timeout: 15000 }, ) .toBe("bound"); - const lock = join(data, ".admin-target-control/locks/motis-feed-proxy"), - inode = statSync(lock).ino; + const lease = () => targetLease(data, "motis-feed-proxy"), + held = lease(); + expect(held).toBeDefined(); await expect( journal.assertOwnedProxyRecoverySettled((await store.inspect(id))!.execution), ).rejects.toThrow(); @@ -783,7 +792,7 @@ it await restarted.reconcile(); expect((await store.inspect(id))?.finishedAt).toBeNull(); expect((await store.inspect(id))?.status.publication).toBe("unknown"); - expect(statSync(lock).ino).toBe(inode); + expect(lease()).toBe(held); expect(posts).toBe(1); await restarted.close(); return; @@ -801,8 +810,9 @@ it { timeout: 15000 }, ) .toBe(2); - const lock = join(data, ".admin-target-control/locks/motis-feed-proxy"), - inode = statSync(lock).ino; + const lease = () => targetLease(data, "motis-feed-proxy"), + held = lease(); + expect(held).toBeDefined(); await expect .poll( () => @@ -840,7 +850,7 @@ it await restarted.reconcile(); expect((await store.inspect(id))?.finishedAt).toBeNull(); expect((await receiver.inspect(id))?.publication).toBe("unknown"); - expect(statSync(lock).ino).toBe(inode); + expect(lease()).toBe(held); await expect( journal.assertOwnedProxyRecoverySettled((await store.inspect(id))!.execution), ).rejects.toThrow(); @@ -848,7 +858,7 @@ it try { expect(await control.pruneControlRecords(dm)).toEqual([]); expect((await store.inspect(id))?.finishedAt).toBeNull(); - expect(statSync(lock).ino).toBe(inode); + expect(lease()).toBe(held); } finally { clock.mockRestore(); } @@ -884,18 +894,15 @@ it kind === "promotion" ? 600000 : 120000, ); expect(record.status.deadlineAt).toBe(deadline); - const lock = join( - data, - ".admin-target-control/locks", - kind === "promotion" ? "motis" : "motis-feed-proxy", - ), - inode = statSync(lock).ino, + const lease = () => targetLease(data, kind === "promotion" ? "motis" : "motis-feed-proxy"), + held = lease(), rootReceipt = join(root, "control-private/data-manager/executions", `${key}.json`), original = readFileSync(rootReceipt); await receiver.requestCancel({ id, operationKey: key, fingerprint: admitted.fingerprint }); expect((await next()).type).toBe("aborted"); + expect(held).toBeDefined(); expect((await store.inspect(id))?.finishedAt).toBeNull(); - expect(statSync(lock).ino).toBe(inode); + expect(lease()).toBe(held); expect((await receiver.inspect(id))?.status).toBe("cancel_pending"); send({ type: "probe" }); expect(await next()).toMatchObject({ @@ -909,7 +916,7 @@ it .poll(async () => (await receiver.inspect(id!))?.status, { timeout: 15000 }) .toBe("canceled"); expect((await receiver.inspect(id))?.publication).toBe("committed"); - await expect.poll(() => existsSync(lock), { timeout: 10000 }).toBe(false); + await expect.poll(lease, { timeout: 10000 }).toBeUndefined(); expect(readFileSync(rootReceipt)).toEqual(original); expect(posts).toBe(1); expect(cancels).toBe(1); @@ -1035,7 +1042,6 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( reservations: [reservation], }); const lease = await control.acquireArtifactTargetLease({ - dataRoot: data, serviceIds: ["motis-feed-proxy"], owner: authority, }); @@ -1045,6 +1051,7 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( ...reservation, lease, childPublicKey: agentVerifier.publicKey, + evidence: control.publishedDataTransitCandidate(authority, reservation), }); const token = "e".repeat(43), tokenFile = join(root, "token"); @@ -1115,25 +1122,26 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( if (crash === "after-terminal") { expect(status.state).toBe("failed"); expect(status.errorClass).toBe("runtime"); - expect(control.verifyFeedProxyCanonicalTerminal(authority, bound.execution).state).toBe( - "failed", - ); - expect( - control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution), - ).toMatchObject({ outcome: "contained", publication: "not_started" }); + // data-manager records the agent's terminal status it observed. + control.recordOpsControlTerminalObservation(dm, bound.execution, status); + expect(control.readOpsControlTerminalObservation(dm, bound.execution).state).toBe("failed"); + await expect( + await control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution), + ).resolves.toMatchObject({ outcome: "contained", publication: "not_started" }); const repaired = JSON.parse(readFileSync(terminalPath, "utf8")).payload; expect(repaired.terminalDigest).toBe(originalTerminal.terminalDigest); expect(repaired.terminalAt).toBe(originalTerminal.terminalAt); } else { expect(status.errorClass).toBe("recovery_required"); + // A status that is not terminal is never recorded. expect(() => - control.verifyFeedProxyCanonicalTerminal(authority, bound.execution), + control.recordOpsControlTerminalObservation(dm, bound.execution, status), ).toThrow(); } const jobs = JSON.parse(readFileSync(join(root, "state/ops-agent/jobs-v1.json"), "utf8")); expect(JSON.stringify(jobs)).toContain(bound.execution.operationKey); await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); - expect(existsSync(join(data, ".admin-target-control/locks/motis-feed-proxy"))).toBe(true); + expect(control.heldTargets(dirname(data)).includes("motis-feed-proxy")).toBe(true); // No POST is sent to the restarted process; original admission remains. } finally { await first?.close(); diff --git a/services/data-manager/src/jobs/execution/ops-children.ts b/services/data-manager/src/jobs/execution/ops-children.ts index 62c388ac6..70b5f1219 100644 --- a/services/data-manager/src/jobs/execution/ops-children.ts +++ b/services/data-manager/src/jobs/execution/ops-children.ts @@ -390,10 +390,12 @@ import { issueControlChildPhase, lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding, - verifyFeedProxyCanonicalTerminal, + publishedDataTransitCandidate, + recordOpsControlTerminalObservation, } from "@openmapx/core/build-artifacts"; import { type OpsJobStatus, type OpsResultFor, parseOpsResult } from "@openmapx/core/ops"; import { opsClient } from "../../ops-client.js"; +import { ownedLease } from "./owned-lease.js"; import type { DataExecutionContext } from "./runtime.js"; import type { FreshDataChildReservation } from "./store.js"; export class DataOpsObservationPending extends Error { @@ -471,10 +473,10 @@ async function observeChild( }; if (child.state !== "terminal") child = await checkpointCurrent(context, saved); if (["succeeded", "failed", "timed_out"].includes(status.state)) { - let evidence: ReturnType; + let evidence: Awaited>; try { - evidence = inspectArtifactTargetLeaseChildSettlement( - context.targetAuthority, + evidence = await inspectArtifactTargetLeaseChildSettlement( + ownedLease(context), child.execution, ); } catch { @@ -482,14 +484,8 @@ async function observeChild( continue; } if (evidence && evidence.outcome !== "unknown" && evidence.publication !== "unknown") { - if (child.operation.kind === "feedProxy.validateAndReload") { - try { - verifyFeedProxyCanonicalTerminal(context.authority, child.execution); - } catch { - await delay(100, undefined, { signal }); - continue; - } - } + // Recovery reads the terminal status back from this role's own record. + recordOpsControlTerminalObservation(context.installation, child.execution, status); child = await checkpointCurrent(context, { ...child, state: "terminal", @@ -599,8 +595,9 @@ export async function runOwnedDataOpsOperation( const issued = await issueControlChildPhase(context.authority, { phase: slot.phase, ordinal: slot.ordinal, - lease: context.targetAuthority, + lease: ownedLease(context), childPublicKey: controlRolePublicKey(context.installation, "ops-agent"), + evidence: publishedDataTransitCandidate(context.authority, slot), }); const reference = lookupControlChildPhaseAttempt(context.authority, slot); if (!reference?.leaseId) throw new DataOpsObservationPending(slot.operationKey); diff --git a/services/data-manager/src/jobs/execution/owned-lease.ts b/services/data-manager/src/jobs/execution/owned-lease.ts new file mode 100644 index 000000000..71288e528 --- /dev/null +++ b/services/data-manager/src/jobs/execution/owned-lease.ts @@ -0,0 +1,10 @@ +import type { ArtifactTargetLease } from "@openmapx/core/build-artifacts"; +import type { DataExecutionContext } from "./runtime.js"; + +/** The execution's own lease; only a hardlink leaf has none. */ +export function ownedLease( + context: Pick, +): ArtifactTargetLease { + if (!context.targetAuthority) throw Error("This execution holds no lease of its own"); + return context.targetAuthority; +} diff --git a/services/data-manager/src/jobs/execution/receiver.ts b/services/data-manager/src/jobs/execution/receiver.ts index 56cd7e1af..178626199 100644 --- a/services/data-manager/src/jobs/execution/receiver.ts +++ b/services/data-manager/src/jobs/execution/receiver.ts @@ -47,10 +47,8 @@ export function createDataExecutionReceiver(deps: { return input; }; return { - recoverUnadmittedPreparation: (request, rawHeaders) => - configurationFence.recoverUnadmittedPreparation(request, rawHeaders), - attestUnadoptedHardlinks: (request, rawHeaders) => - configurationFence.attestUnadoptedGenerationHardlinks(request, rawHeaders), + recoverUnadmittedPreparation: (request, rawHeaders, transportPrincipal) => + configurationFence.recoverUnadmittedPreparation(request, rawHeaders, transportPrincipal), async observeReviewedTransit(request) { const record = await runtime.lookupByIdempotencyKey?.(request.key); if (!record) return null; @@ -97,10 +95,11 @@ export function createDataExecutionReceiver(deps: { operationFingerprint: dataExecutionInputFingerprint(request.input), }); if (!verified || !controlHeaders(rawHeaders).delegation) - throw Object.assign(Error("Signed borrowed generation leaf required"), { statusCode: 403 }); + throw Object.assign(Error("Delegated generation leaf required"), { statusCode: 403 }); const preparedGeneration = await configurationFence.assertGenerationAdmission( request, rawHeaders, + transportPrincipal, verified.parent, ); return runtime.admit(request.input, { @@ -144,7 +143,8 @@ export function createDataExecutionReceiver(deps: { requestCancel: runtime.requestCancel, prepareFence: (input, headers, transportPrincipal) => configurationFence.prepare(input, headers, transportPrincipal), - inspectFence: (id, headers) => configurationFence.inspect(id, headers), + inspectFence: (id, expected, transportPrincipal) => + configurationFence.inspect(id, expected, transportPrincipal), settleFence: (input, headers, transportPrincipal) => configurationFence.settle(input, headers, transportPrincipal), subscribe: deps.dispatcher.subscribe, diff --git a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts index f3f1e907e..e22ed9b71 100644 --- a/services/data-manager/src/jobs/execution/runner-cancellation.test.ts +++ b/services/data-manager/src/jobs/execution/runner-cancellation.test.ts @@ -24,6 +24,14 @@ import { createTransitousScriptRunner } from "../transitous/script-runner"; import { createDataExecutionRuntime } from "./runtime"; import { createPostgresDataExecutionStore } from "./store"; +/** The lease that holds MOTIS in the installation at `infraDir`, if any. */ +function motisLease(infraDir: string): string | undefined { + return control + .openLeaseStore({ directory: control.leaseStoreDirectory(infraDir), purpose: "test" }) + .list() + .find((lease) => lease.targets.includes("motis"))?.leaseId; +} + let recoveryBundleRoot: string, dmBundle: string, agentBundle: string; beforeAll(async () => { recoveryBundleRoot = mkdtempSync(join(tmpdir(), "omx-dm-crash-bundle-")); @@ -224,8 +232,8 @@ it.skipIf( expect(posts).toBe(2); const [original] = await sql`SELECT * FROM data_manager.jobs WHERE id=${id}`; if (!original) throw Error("Original row missing"); - const lock = join(root, "data/.admin-target-control/locks/motis"), - inode = statSync(lock).ino; + const lease = motisLease(root); + expect(lease).toBeDefined(); const proof = join( root, `control-private/data-manager/observations/worker-${original.operation_key}.json`, @@ -297,7 +305,7 @@ it.skipIf( "triggered_by", ]) expect(current?.[key]).toEqual(original[key]); - expect(statSync(lock).ino).toBe(inode); + expect(motisLease(root)).toBe(lease); expect(existsSync(proof)).toBe(false); expect(posts).toBe(2); for (const [name, controlDigest] of controlledDigests) @@ -431,8 +439,8 @@ it const [original] = await sql`SELECT * FROM data_manager.jobs WHERE id=${id}`; if (!original) throw Error("Original row missing"); const operationKey = String(original.operation_key), - lock = join(root, "data/.admin-target-control/locks/motis"); - const inode = statSync(lock).ino; + lease = motisLease(root); + expect(lease).toBeDefined(); await dm!.kill(); dm = undefined; // SIGKILL occurred while dispatcher awaited the independent child. @@ -503,7 +511,7 @@ it "triggered_by", ]) expect(current?.[key]).toEqual(original[key]); - expect(statSync(lock).ino).toBe(inode); + expect(motisLease(root)).toBe(lease); expect(existsSync(join(observations, `worker-${operationKey}.json`))).toBe(false); dm!.send({ type: "probe" }); expect((await dm!.next("probe")).dispatches).toBe(0); @@ -647,24 +655,27 @@ it.skipIf( .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) .toBe("canceled"); await expect.poll(() => terminalPublished, { timeout: 15000 }).toBe(true); - const terminal = await store.inspect(id), - lock = join(root, "data/.admin-target-control/locks/motis"); + const terminal = await store.inspect(id); if (!terminal) throw Error("Original terminal row missing"); - expect(existsSync(lock)).toBe(true); + expect(motisLease(root)).toBeDefined(); expect(() => - control.verifyDataControlTerminalWitness(installation, terminal.execution, terminal.status), + control.readDataControlTerminalObservation( + installation, + terminal.execution, + terminal.status, + ), ).toThrow(/ENOENT/); expect(terminal.status.publication).toBe("not_started"); expect(terminal.finishedAt).not.toBeNull(); expect(posts).toBe(1); expect(cancels).toBeGreaterThanOrEqual(1); resumeCleanup(); - // The signed contained-terminal witness is published only after lease - // release; the canceled database row above is visible before that point. + // The contained terminal is recorded only after lease release; the + // canceled database row above is visible before that point. await expect .poll( () => - control.verifyDataControlTerminalWitness( + control.readDataControlTerminalObservation( installation, terminal.execution, terminal.status, @@ -672,7 +683,7 @@ it.skipIf( { timeout: 15000 }, ) .toEqual(terminal.status); - expect(existsSync(lock)).toBe(false); + expect(motisLease(root)).toBeUndefined(); } finally { resumeCleanup(); writeFileSync(released, "release"); diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts index d83f63db2..e39f161b6 100644 --- a/services/data-manager/src/jobs/execution/runtime.test.ts +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { mkdirSync, mkdtempSync, existsSync as requireExists, rmSync, statSync } from "node:fs"; import { connect, createServer } from "node:net"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey } from "@openmapx/core/ops"; import { drizzle } from "drizzle-orm/postgres-js"; @@ -261,14 +261,10 @@ describe.skipIf( }); expect((await store.inspect(id))?.finishedAt).not.toBeNull(); expect(dispatches).toBe(1); - expect( - (await import("node:fs")).existsSync( - join(root, "data/.admin-target-control/locks/data-manager"), - ), - ).toBe(false); + expect(control.heldTargets(root).includes("data-manager")).toBe(false); const recovered = await store.inspect(id); expect( - control.verifyDataControlTerminalWitness(installation, recovered!.execution).status, + control.readDataControlTerminalObservation(installation, recovered!.execution).status, ).toEqual(recovered!.status); await runtime.reconcile(); expect(dispatches).toBe(1); @@ -497,7 +493,7 @@ describe.skipIf( expect(retained?.triggeredBy).toBe("cron"); expect(retained?.finishedAt).not.toBeNull(); expect( - control.verifyDataControlTerminalWitness(installation, retained!.execution).status, + control.readDataControlTerminalObservation(installation, retained!.execution).status, ).toEqual(retained!.status); expect(dispatches).toBe(1); } finally { @@ -579,9 +575,7 @@ describe.skipIf( expect(forwards).toBe(0); expect((await store.inspect(id))?.finishedAt).toBeNull(); expect((await store.inspect(id))?.status.publication).toBe("unknown"); - expect( - (await import("node:fs")).existsSync(join(data, ".admin-target-control/locks/motis")), - ).toBe(true); + expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); } finally { await runtime.close(); if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; @@ -635,13 +629,9 @@ describe.skipIf( .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) .toBe("ok"); await expect - .poll( - async () => - !(await import("node:fs")).existsSync( - join(data, ".admin-target-control/locks/data-manager"), - ), - { timeout: 15000 }, - ) + .poll(async () => !control.heldTargets(dirname(data)).includes("data-manager"), { + timeout: 15000, + }) .toBe(true); await runtime.close(); const record = await store.inspect(id); @@ -816,8 +806,8 @@ describe.skipIf( ); id = admitted.id; await readyGate; - const lock = join(root, "data/.admin-target-control/locks/data-manager"), - inode = statSync(lock).ino; + const held = () => control.heldTargets(root).includes("data-manager"); + expect(held()).toBe(true); if (mode === "consume-after-abort") { start(); await expect.poll(active, { timeout: 5000 }).toBe(1); @@ -836,7 +826,7 @@ describe.skipIf( cancelPackets: proxy.packets(), status: row?.status.status, finished: row?.finishedAt !== null, - fence: requireExists(lock), + fence: held(), }; }, { timeout: 5000 }, @@ -849,7 +839,7 @@ describe.skipIf( finished: false, fence: true, }); - expect(statSync(lock).ino).toBe(inode); + expect(held()).toBe(true); } proxy.release(); await expect.poll(() => querySettled, { timeout: 5000 }).toBe(true); @@ -868,7 +858,7 @@ describe.skipIf( expect(callbacks).toBe(0); // Durable terminal SQL precedes asynchronous physical lease release. // The worker connection's barrier above does not await that release. - await expect.poll(() => requireExists(lock), { timeout: 5000 }).toBe(false); + await expect.poll(held, { timeout: 5000 }).toBe(false); } finally { proxy.release(); start(); diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index 42478d913..040959012 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -1,10 +1,9 @@ import { setTimeout as settleDelay } from "node:timers/promises"; import { + type ArtifactTargetLease, acquireArtifactTargetLease, assertArtifactTargetLeaseForward, assertControlExecutionForward, - type BorrowedArtifactTargetLease, - bindBorrowedControlExecution, ControlExecutionFencedError, type ControlInstallation, cancelArtifactTargetLease, @@ -13,10 +12,8 @@ import { releaseArtifactTargetLease, restoreAdmittedControlExecution, restoreArtifactTargetLeaseForSettlement, - restoreGenerationHardlinksLeaseForSettlement, retainAdmittedControlExecution, settleArtifactTargetLeaseChild, - type TargetLeaseAuthority, type VerifiedExecutionAuthority, type VerifiedPreparedGenerationAdmission, verifyControlEffectSettlement, @@ -38,6 +35,7 @@ import { type DataOpsChildAttempt, dataChildControlReservation, } from "./ops-children.js"; +import { ownedLease } from "./owned-lease.js"; import type { DataExecutionPatch, DataExecutionRecord, DataExecutionStore } from "./store.js"; import { withDataExecutionScope } from "./worker-scope.js"; export type DataExecutionProgressPatch = Partial< @@ -51,7 +49,9 @@ export interface DataExecutionContext { readonly execution: ExecutionRef; readonly executionVersion: number; readonly signal: AbortSignal; - readonly targetAuthority: TargetLeaseAuthority; + /** The execution's own lease in the agent's store. A hardlink leaf runs under + * its caller's lease and has none. */ + readonly targetAuthority?: ArtifactTargetLease; readonly authority: VerifiedExecutionAuthority; readonly installation: ControlInstallation; readonly store: DataExecutionStore; @@ -78,7 +78,8 @@ export interface DataExecutionRuntime { sourceJobId?: string; triggeredBy?: string; idempotencyKey?: string; - adopt?: (execution: ExecutionRef) => Promise; + /** Checks the normalized identity against what the caller named. */ + adopt?: (execution: ExecutionRef) => Promise; }, ): Promise; inspect(id: string): Promise; @@ -88,6 +89,25 @@ export interface DataExecutionRuntime { reconcile(): Promise; close(): Promise; } +/** Settles and releases the execution's own lease after its terminal record, + * then records the terminal for later steps that build on it. */ +async function releaseOwnLease( + journal: { + installation: ControlInstallation; + recordContainedTerminal(e: ExecutionRef): Promise; + }, + execution: ExecutionRef, + lease: ArtifactTargetLease | undefined, +) { + if (lease) { + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(journal.installation, execution), + ); + await releaseArtifactTargetLease(lease); + } + await journal.recordContainedTerminal(execution); +} export function dataExecutionTargetIds(input: DataExecutionInput): string[] { return ["transit-sync", "catalog.validate", "catalog.activate"].includes(input.kind) ? ["data-manager", "motis", "motis-feed-proxy", "motis-staging"] @@ -125,7 +145,7 @@ export function createDataExecutionRuntime(deps: { authority = restoreAdmittedControlExecution(owner.installation, record.execution), lease = record.input.kind === "generation-hardlinks" - ? restoreGenerationHardlinksLeaseForSettlement(authority) + ? undefined : restoreArtifactTargetLeaseForSettlement(authority); let tail: Promise = Promise.resolve(), version = record.version; @@ -197,7 +217,7 @@ export function createDataExecutionRuntime(deps: { executionCanceled = true; } catch {} try { - await cancelArtifactTargetLease(owned.targetAuthority); + if (owned.targetAuthority) await cancelArtifactTargetLease(owned.targetAuthority); leaseCanceled = true; } catch {} if (executionCanceled && leaseCanceled) return; @@ -215,7 +235,7 @@ export function createDataExecutionRuntime(deps: { } catch { /* Original identity/lease gaps retain the durable cancellation intent. */ } finally { - if (context) closeArtifactTargetLeaseHandle(context.targetAuthority); + if (context?.targetAuthority) closeArtifactTargetLeaseHandle(context.targetAuthority); } }) .finally(() => recovering.delete(record.status.id)); @@ -223,13 +243,13 @@ export function createDataExecutionRuntime(deps: { } async function start( record: DataExecutionRecord, - adopt?: (execution: ExecutionRef) => Promise, + adopt?: (execution: ExecutionRef) => Promise, parent?: VerifiedParentExecution, hardlinkPlan?: unknown, ) { const owner = requireJournal(), targets = dataExecutionTargetIds(record.input); - const borrowed = await adopt?.(record.execution); + await adopt?.(record.execution); const authority = retainAdmittedControlExecution(owner.installation, { execution: record.execution, kind: record.input.kind, @@ -239,22 +259,19 @@ export function createDataExecutionRuntime(deps: { ? { requestedDeadlineAt: record.requestedDeadlineAt } : {}), ...(parent ? { parent } : {}), - serviceIds: borrowed?.serviceIds ?? targets, + serviceIds: targets, reservations: record.reservations.map(dataChildControlReservation), }); - if (borrowed) bindBorrowedControlExecution(borrowed, authority); - if (record.input.kind === "generation-hardlinks") { - if (hardlinkPlan === undefined) throw Error("Signed durable generation plan required"); + const leaf = record.input.kind === "generation-hardlinks"; + if (leaf) { + if (hardlinkPlan === undefined) throw Error("Durable generation plan required"); await owner.retainHardlinksPlan(authority, hardlinkPlan); } await owner.begin(authority); - const lease = - borrowed ?? - (await acquireArtifactTargetLease({ - dataRoot: owner.dataRoot, - serviceIds: targets, - owner: authority, - })); + // A hardlink leaf runs under the lease of the agent or host CLI that issued it. + const lease = leaf + ? undefined + : await acquireArtifactTargetLease({ serviceIds: targets, owner: authority }); const controller = new AbortController(); let entry: LiveEntry; const context: DataExecutionContext = { @@ -299,7 +316,7 @@ export function createDataExecutionRuntime(deps: { assertForward() { controller.signal.throwIfAborted(); assertControlExecutionForward(authority); - assertArtifactTargetLeaseForward(lease); + if (lease) assertArtifactTargetLeaseForward(lease); }, }; entry = { record, controller, context, task: Promise.resolve(), tail: Promise.resolve() }; @@ -420,16 +437,7 @@ export function createDataExecutionRuntime(deps: { }); if (isDataExecutionTerminal(entry.record.status)) { await owner.assertOwnedProxyRecoverySettled(record.execution); - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(owner.installation, record.execution), - ); - if (lease.mode === "owner") { - await owner.assertOwnedProxyRecoverySettled(record.execution); - await releaseArtifactTargetLease(lease); - await owner.publishContainedTerminal(record.execution); - } else if (record.input.kind === "generation-hardlinks") - await owner.publishContainedTerminal(record.execution); + await releaseOwnLease(owner, record.execution, lease); } } catch (error) { // An I/O/CAS/containment gap is not a terminal receipt or permission to @@ -453,7 +461,7 @@ export function createDataExecutionRuntime(deps: { } finally { clearTimeout(timer); clearInterval(ancestry); - closeArtifactTargetLeaseHandle(lease); + if (lease) closeArtifactTargetLeaseHandle(lease); active.delete(record.status.id); } })(); @@ -474,11 +482,12 @@ export function createDataExecutionRuntime(deps: { restoreAdmittedControlExecution(owner.installation, saved.execution); if (entry) cancelControlExecution(authority); else cancelRecovered(saved); - if (entry) { + const own = entry?.context.targetAuthority; + if (own) { const until = now() + 15000; for (;;) { try { - await cancelArtifactTargetLease(entry.context.targetAuthority); + await cancelArtifactTargetLease(own); break; } catch (error) { if (now() >= until) throw error; @@ -529,27 +538,15 @@ export function createDataExecutionRuntime(deps: { await owner.assertOwnedProxyRecoverySettled(record.execution); await owner.repairTerminal(record.execution); await owner.publishTerminal(record.execution); - try { - await owner.publishContainedTerminal(record.execution); - continue; - } catch {} const authority = restoreAdmittedControlExecution(owner.installation, record.execution), lease = record.input.kind === "generation-hardlinks" - ? restoreGenerationHardlinksLeaseForSettlement(authority) + ? undefined : restoreArtifactTargetLeaseForSettlement(authority); try { - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(owner.installation, record.execution), - ); - if (lease.mode === "owner") { - await releaseArtifactTargetLease(lease); - await owner.publishContainedTerminal(record.execution); - } else if (record.input.kind === "generation-hardlinks") - await owner.publishContainedTerminal(record.execution); + await releaseOwnLease(owner, record.execution, lease); } finally { - closeArtifactTargetLeaseHandle(lease); + if (lease) closeArtifactTargetLeaseHandle(lease); } } catch { /* Original absent proof or released lease grants no new authority. */ @@ -574,13 +571,13 @@ export function createDataExecutionRuntime(deps: { // Final approval is the fixed graph's last effect. Reconcile only // its original address and authenticated, fully settled child tree. cancelControlExecution(observation.authority); - await cancelArtifactTargetLease(observation.targetAuthority); + await cancelArtifactTargetLease(ownedLease(observation)); if ((await cancelAndSettleOwnedDataOpsChildren(observation)) !== "settled") throw error; await owner.repairCommittedCatalogActivation(record.execution); recovered = await owner.restoreCompletion(record.execution); } cancelControlExecution(recovered.authority); - await cancelArtifactTargetLease(recovered.lease); + if (recovered.lease) await cancelArtifactTargetLease(recovered.lease); const context = observation; if ( (await cancelAndSettleOwnedDataOpsChildren(context)) !== "settled" || @@ -614,15 +611,7 @@ export function createDataExecutionRuntime(deps: { ); if (!saved) throw new DataExecutionOwnershipLost("Recovery terminal CAS lost"); await owner.assertOwnedProxyRecoverySettled(record.execution); - await settleArtifactTargetLeaseChild( - recovered.lease, - verifyControlEffectSettlement(owner.installation, record.execution), - ); - if (recovered.lease.mode === "owner") { - await releaseArtifactTargetLease(recovered.lease); - await owner.publishContainedTerminal(record.execution); - } else if (record.input.kind === "generation-hardlinks") - await owner.publishContainedTerminal(record.execution); + await releaseOwnLease(owner, record.execution, recovered.lease); } catch (error) { jobChildLogger({ job: "data-execution-recovery", jobId: record.status.id }).warn( { @@ -652,8 +641,9 @@ export function createDataExecutionRuntime(deps: { } } catch {} } finally { - if (recovered) closeArtifactTargetLeaseHandle(recovered.lease); - if (observation) closeArtifactTargetLeaseHandle(observation.targetAuthority); + if (recovered?.lease) closeArtifactTargetLeaseHandle(recovered.lease); + if (observation?.targetAuthority) + closeArtifactTargetLeaseHandle(observation.targetAuthority); } } }, diff --git a/services/data-manager/src/jobs/execution/source-mutation.ts b/services/data-manager/src/jobs/execution/source-mutation.ts index 4c03c97fe..e09258cb0 100644 --- a/services/data-manager/src/jobs/execution/source-mutation.ts +++ b/services/data-manager/src/jobs/execution/source-mutation.ts @@ -4,6 +4,7 @@ import type { DataExecutionInput } from "@openmapx/core/data-manager-contracts"; import { prepareTransitSourceMutation } from "../../transit-sources.js"; import { resolveTransitOverlayPath } from "../transitous/internal.js"; import type { MotisOperationsPolicy } from "../transitous/operations-profile.js"; +import { ownedLease } from "./owned-lease.js"; import type { DataExecutionContext } from "./runtime.js"; import { dataExecutionWorkerContext, publishDataChange } from "./worker-scope.js"; /** Fixed first phase of the original transit-sync, not an HTTP-side mutation. */ @@ -16,7 +17,7 @@ export async function publishOwnedTransitSourceMutation( if (!input.sourceMutation) return; if (dataExecutionWorkerContext() !== context) throw Error("Owned transit worker required"); context.assertForward(); - assertArtifactTargetLeaseForward(context.targetAuthority); + assertArtifactTargetLeaseForward(ownedLease(context)); const retained = await context.record(); if ( retained.input.kind !== "transit-sync" || @@ -34,7 +35,7 @@ export async function publishOwnedTransitSourceMutation( ); await publishDataChange(() => { context.assertForward(); - assertArtifactTargetLeaseForward(context.targetAuthority); + assertArtifactTargetLeaseForward(ownedLease(context)); mutation.persist(); }); } diff --git a/services/data-manager/src/jobs/execution/store-postgres.test.ts b/services/data-manager/src/jobs/execution/store-postgres.test.ts index 89c51c682..fa1268c01 100644 --- a/services/data-manager/src/jobs/execution/store-postgres.test.ts +++ b/services/data-manager/src/jobs/execution/store-postgres.test.ts @@ -159,7 +159,6 @@ describe.skipIf( reservations: record.reservations.map(dataChildControlReservation), }); const lease = await control.acquireArtifactTargetLease({ - dataRoot: join(root, "data"), serviceIds: services, owner: authority, }); diff --git a/services/data-manager/src/jobs/transitous/catalog-activation-record.ts b/services/data-manager/src/jobs/transitous/catalog-activation-record.ts index 3f476ade2..0a0189569 100644 --- a/services/data-manager/src/jobs/transitous/catalog-activation-record.ts +++ b/services/data-manager/src/jobs/transitous/catalog-activation-record.ts @@ -14,6 +14,7 @@ import { } from "@openmapx/core/catalog-pins"; import { canonicalJson, type ExecutionRef } from "@openmapx/core/ops"; import { runOpsOperation } from "../../ops-client.js"; +import { ownedLease } from "../execution/owned-lease.js"; import type { DataExecutionContext } from "../execution/runtime.js"; import { publishDataChange } from "../execution/worker-scope.js"; export async function retainCatalogActivationRecord( @@ -48,8 +49,8 @@ export async function retainCatalogActivationRecord( ) ) throw Error("Original settled promotion and fixed approval reservation required"); - const physical = inspectArtifactTargetLeaseChildSettlement( - context.targetAuthority, + const physical = await inspectArtifactTargetLeaseChildSettlement( + ownedLease(context), promotion.execution, ); if ( diff --git a/services/data-manager/src/jobs/transitous/catalog-activation.ts b/services/data-manager/src/jobs/transitous/catalog-activation.ts index 5b852d900..bc6ddf6e7 100644 --- a/services/data-manager/src/jobs/transitous/catalog-activation.ts +++ b/services/data-manager/src/jobs/transitous/catalog-activation.ts @@ -1,7 +1,7 @@ /** Activate exactly one reviewed candidate under the existing finite root and child reservations. */ import { createHash } from "node:crypto"; import { realpathSync } from "node:fs"; -import { controlDigest, verifyDataControlTerminalWitness } from "@openmapx/core/build-artifacts"; +import { controlDigest, readDataControlTerminalObservation } from "@openmapx/core/build-artifacts"; import { catalogProposalSchema, catalogRuntimeInputsSchema, @@ -55,7 +55,7 @@ export async function runCatalogActivation(options: { currentValidation.selected.validationDigest !== controlDigest(validation) ) throw Error("Validation was invalidated before slot reuse; validate again"); - const terminal = verifyDataControlTerminalWitness( + const terminal = readDataControlTerminalObservation( context.installation, validation.sourceExecution, ); diff --git a/services/data-manager/src/jobs/transitous/catalog-review.ts b/services/data-manager/src/jobs/transitous/catalog-review.ts index a09a58eb8..236aa3259 100644 --- a/services/data-manager/src/jobs/transitous/catalog-review.ts +++ b/services/data-manager/src/jobs/transitous/catalog-review.ts @@ -5,9 +5,9 @@ import { join } from "node:path"; import { type ControlInstallation, controlDigest, + readDataControlTerminalObservation, signControl, verifyControl, - verifyDataControlTerminalWitness, } from "@openmapx/core/build-artifacts"; import { type CatalogProposal, @@ -338,7 +338,7 @@ export function createCatalogReviewService(options: { ); validation = retained.validation; if (validation.status === "passed") { - const completed = verifyDataControlTerminalWitness( + const completed = readDataControlTerminalObservation( options.installation, retained.sourceExecution, ), diff --git a/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts b/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts index 6dafd7107..2a5195fb5 100644 --- a/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts +++ b/services/data-manager/src/jobs/transitous/catalog-rollback-retention.ts @@ -19,6 +19,7 @@ import { import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; import { safeArtifactPath } from "../../../../../packages/core/src/build-artifacts/manifest"; import { atomicWriteJsonSync } from "../../utils/atomic-write"; +import { ownedLease } from "../execution/owned-lease"; import type { DataExecutionContext } from "../execution/runtime"; import { dataExecutionInputFingerprint } from "../execution/store"; import { publishDataChange } from "../execution/worker-scope"; @@ -55,15 +56,15 @@ export async function retainPreviousHealthyCatalogSlot( const owner = executionAuthorityState(context.authority), record = await context.record(); assertControlExecutionForward(context.authority); - assertArtifactTargetLeaseForward(context.targetAuthority); + assertArtifactTargetLeaseForward(ownedLease(context)); if ( owner.kind !== "catalog.validate" || record.input.kind !== "catalog.validate" || record.input.proposalId !== proposalId || dataExecutionInputFingerprint(record.input) !== owner.execution.operationFingerprint || - context.targetAuthority.mode !== "owner" || + ownedLease(context).mode !== "owner" || !["motis", "motis-staging", "motis-feed-proxy"].every((id) => - context.targetAuthority.serviceIds.includes(id), + ownedLease(context).serviceIds.includes(id), ) || !Number.isSafeInteger(minimumDiskBytes) || minimumDiskBytes < 0 @@ -79,8 +80,8 @@ export async function retainPreviousHealthyCatalogSlot( if ( !stop || stop.state !== "terminal" || - inspectArtifactTargetLeaseChildSettlement(context.targetAuthority, stop.execution)?.outcome !== - "committed" + (await inspectArtifactTargetLeaseChildSettlement(ownedLease(context), stop.execution)) + ?.outcome !== "committed" ) throw Error( "Stop staging under the original validation reservation before preserving rollback data", diff --git a/services/data-manager/src/jobs/transitous/catalog-validation.ts b/services/data-manager/src/jobs/transitous/catalog-validation.ts index 41c30bd98..1f1a9b705 100644 --- a/services/data-manager/src/jobs/transitous/catalog-validation.ts +++ b/services/data-manager/src/jobs/transitous/catalog-validation.ts @@ -24,6 +24,7 @@ import { runOpsOperation } from "../../ops-client.js"; import type { StateStore } from "../../state.js"; import { dataExecutionDiagnostic } from "../execution/diagnostics.js"; import { DataOpsObservationPending, runOwnedDataOpsOperation } from "../execution/ops-children.js"; +import { ownedLease } from "../execution/owned-lease.js"; import type { DataExecutionContext } from "../execution/runtime.js"; import { DataPublicationUnknown, publishDataChange } from "../execution/worker-scope.js"; import { readFeedOverlay } from "../transitous-feeds-overlay.js"; @@ -96,7 +97,7 @@ export async function runCatalogValidation(options: { }; await child(0); await publishDataChange(() => - publishCatalogValidationCurrent(context.authority, context.targetAuthority), + publishCatalogValidationCurrent(context.authority, ownedLease(context)), ); await retainPreviousHealthyCatalogSlot( context, @@ -240,7 +241,7 @@ export async function runCatalogValidation(options: { ); if (validation.status === "passed") await publishDataChange(() => - publishCatalogValidationCurrent(context.authority, context.targetAuthority, input.proposalId), + publishCatalogValidationCurrent(context.authority, ownedLease(context), input.proposalId), ); return validation; } diff --git a/services/data-manager/src/jobs/transitous/isolate-producer.ts b/services/data-manager/src/jobs/transitous/isolate-producer.ts index 8c154327f..62a60b7ed 100644 --- a/services/data-manager/src/jobs/transitous/isolate-producer.ts +++ b/services/data-manager/src/jobs/transitous/isolate-producer.ts @@ -7,6 +7,7 @@ import { } from "@openmapx/core/build-artifacts"; import { isolateArtifactHardlinks } from "../../../../../packages/core/src/build-artifacts/copy-tree"; import { AnchoredDirectory } from "../../../../../packages/core/src/build-artifacts/filesystem"; +import { ownedLease } from "../execution/owned-lease"; import type { DataExecutionContext } from "../execution/runtime"; import { dataExecutionWorkerContext, publishDataChange } from "../execution/worker-scope"; @@ -18,16 +19,16 @@ export async function isolateOwnedTransitProducer( context.assertForward(); const owner = executionAuthorityState(context.authority), record = await context.record(); - assertArtifactTargetLeaseForward(context.targetAuthority); - assertArtifactTargetLeaseExecution(context.targetAuthority, context.execution); + assertArtifactTargetLeaseForward(ownedLease(context)); + assertArtifactTargetLeaseExecution(ownedLease(context), context.execution); if ( owner.installation.installation.role !== "data-manager" || !["transit-sync", "catalog.validate"].includes(owner.kind) || controlDigest(record.execution) !== controlDigest(owner.execution) || controlDigest(record.input) !== owner.execution.operationFingerprint || - context.targetAuthority.mode !== "owner" || + ownedLease(context).mode !== "owner" || !["motis", "motis-staging", "motis-feed-proxy"].every((id) => - context.targetAuthority.serviceIds.includes(id), + ownedLease(context).serviceIds.includes(id), ) ) throw Error("Original complete transit target owner required"); diff --git a/services/data-manager/src/principals.ts b/services/data-manager/src/principals.ts index 9267148a6..f03a9a7c3 100644 --- a/services/data-manager/src/principals.ts +++ b/services/data-manager/src/principals.ts @@ -17,7 +17,6 @@ export const DATA_MANAGER_API_ROUTES: { } = { startDataExecution: true, startGenerationHardlinksExecution: false, - recoverUnadmittedGenerationHardlinks: false, prepareDataConfigurationFence: false, recoverUnadmittedConfigurationPreparation: false, inspectDataConfigurationFence: false, diff --git a/services/data-manager/src/service.ts b/services/data-manager/src/service.ts index f717fce0d..34eec45be 100644 --- a/services/data-manager/src/service.ts +++ b/services/data-manager/src/service.ts @@ -4,6 +4,8 @@ import { setTimeout as waitForDataStatus } from "node:timers/promises"; import { createDataExecutionControlJournal, loadControlStartup, + opsClientLeaseSender, + remoteLeaseBackend, } from "@openmapx/core/build-artifacts"; import { isDataExecutionTerminal } from "@openmapx/core/data-manager-contracts"; import { createFatalProcessHandler } from "@openmapx/core/server"; @@ -46,7 +48,7 @@ import { OfflinePackageGenerator } from "./offline-packages/generator.js"; import { PostgresOfflinePackageAccountingStore } from "./offline-packages/postgres-accounting.js"; import { createOpenMapxPackageSourceFactory } from "./offline-packages/source-catalog.js"; import { OfflinePackageStorage } from "./offline-packages/storage.js"; -import { observeRetainedHostOperation } from "./ops-client.js"; +import { observeRetainedHostOperation, opsClient } from "./ops-client.js"; import { DataManagerReadiness } from "./readiness.js"; import { initializeRequiredSubsystems } from "./startup.js"; import { StateStore } from "./state.js"; @@ -66,6 +68,8 @@ const controlStartup = loadControlStartup({ role: "data-manager", container: true, serviceIds: DATA_MANAGER_CONTROL_SERVICE_IDS, + // Data-manager claims its targets through the agent, which keeps every lease. + leases: remoteLeaseBackend(opsClientLeaseSender(opsClient)), }); if (controlStartup.reason) app.log.warn(controlStartup.reason); From 21ccdff45f4e1e1b400d1aab365ddeab6fe0d0e0 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 16:03:33 +0100 Subject: [PATCH 03/42] fix(control): let lock reclaimers take turns so a new holder is never removed A reclaimer that judged an old holder gone could move aside a file a new writer had just created, leaving two writers inside the lock. Reclaimers now hold a short guard file beside the lock and remove the lock only when it is still the file they judged; a reclaimer that died holding the guard is reclaimed the same way. `openmapx control clear --lock` also removes a left guard. --- .../__tests__/recoverable-lock.test.ts | 39 +++++++++++- .../core/src/services/recoverable-lock.ts | 60 +++++++++++++++++-- 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/packages/core/src/services/__tests__/recoverable-lock.test.ts b/packages/core/src/services/__tests__/recoverable-lock.test.ts index 36a5701da..be1ef22e7 100644 --- a/packages/core/src/services/__tests__/recoverable-lock.test.ts +++ b/packages/core/src/services/__tests__/recoverable-lock.test.ts @@ -1,5 +1,13 @@ import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdtempSync, + readdirSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -162,4 +170,33 @@ describe("recoverable lock", () => { expect(readLockHolder(path)?.purpose).toBe("next"); next(); }); + + it("never removes a new holder's file that replaced the stale one it judged", async () => { + const path = lockPath(); + writeFileSync( + path, + JSON.stringify( + foreign({ pidNamespace: "elsewhere", containerId: "c".repeat(64), pid: 4242 }), + ), + ); + let live: number | undefined; + // While this reclaimer asks Docker about the old holder, another reclaimer + // removes that file and a new writer takes the lock. + const replaced = async () => { + unlinkSync(path); + writeFileSync(path, JSON.stringify(currentLockHolder("new writer"))); + live = statSync(path).ino; + return false; + }; + await expect( + acquireRecoverableLock(path, { + purpose: "late reclaimer", + waitMs: 200, + containerRunning: replaced, + }), + ).rejects.toBeInstanceOf(LockBusyError); + expect(statSync(path).ino).toBe(live); + expect(readLockHolder(path)?.purpose).toBe("new writer"); + expect(existsSync(`${path}.reclaim`)).toBe(false); + }); }); diff --git a/packages/core/src/services/recoverable-lock.ts b/packages/core/src/services/recoverable-lock.ts index dabf77f7d..18ea89cec 100644 --- a/packages/core/src/services/recoverable-lock.ts +++ b/packages/core/src/services/recoverable-lock.ts @@ -188,10 +188,11 @@ export function readLockHolder(path: string): LockHolder | undefined { } /** - * Moves a stale lock aside. Reclaimers race through `rename`; the one that - * finds it moved a different file than the one it judged puts that file back. + * Moves the file at `path` aside if it is still the one judged (`ino`); one + * that changed meanwhile is put back. Used for the reclaim guard, and by the + * operator's clear, which acts on a holder it named. */ -function reclaim(path: string, ino: number): void { +function moveAside(path: string, ino: number): void { const aside = `${path}.stale-${randomBytes(6).toString("hex")}`; try { renameSync(path, aside); @@ -210,6 +211,50 @@ function reclaim(path: string, ino: number): void { fsyncDirectory(dirname(path)); } +/** + * Removes a lock whose holder is gone. Reclaimers take turns through a guard + * file beside the lock, and the one holding it removes the lock only if it is + * still the file it judged: a new holder's file is never removed, because + * nobody else removes or replaces the lock while the guard is held. Resolves + * false when another reclaimer holds the guard; the caller looks again. + */ +async function reclaim( + path: string, + ino: number, + self: LockHolder, + containerRunning?: ContainerRunning, +): Promise { + const guard = `${path}.reclaim`; + let descriptor: number; + try { + descriptor = openSync( + guard, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + // A reclaimer that died while holding the guard leaves it behind. + const other = readHolderFile(guard); + if (other?.holder && (await lockHolderGone(other.holder, self, containerRunning))) + moveAside(guard, other.ino); + return false; + } + try { + writeSync(descriptor, JSON.stringify(self)); + } finally { + closeSync(descriptor); + } + try { + if (readHolderFile(path)?.ino !== ino) return true; + unlinkSync(path); + fsyncDirectory(dirname(path)); + return true; + } finally { + unlinkSync(guard); + } +} + export interface RecoverableLockOptions { purpose: string; /** How long to wait for a live holder, 5 seconds by default. */ @@ -269,7 +314,8 @@ export async function acquireRecoverableLock( // An unreadable holder record is only trusted as stale once it has stayed // unreadable past the wait, so a holder still writing it is not displaced. if (existing.holder || performance.now() >= deadline) { - reclaim(path, existing.ino); + if (await reclaim(path, existing.ino, self, options.containerRunning)) continue; + await delay(5, undefined, { signal: options.signal }); continue; } } @@ -291,10 +337,12 @@ export async function withRecoverableLock( } } -/** The operator exit for a lock whose holder cannot be proved gone. */ +/** The operator exit for a lock whose holder cannot be proved gone, and for its reclaim guard. */ export function clearRecoverableLock(path: string): LockHolder | undefined { + const guard = readHolderFile(`${path}.reclaim`); + if (guard) moveAside(`${path}.reclaim`, guard.ino); const existing = readHolderFile(path); if (!existing) return undefined; - reclaim(path, existing.ino); + moveAside(path, existing.ino); return existing.holder; } From 8a7b69265b13f9b9646e6728bf7078cc3d49c4cd Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 16:03:35 +0100 Subject: [PATCH 04/42] fix(control): settle data-manager leaves, refuse busy before recording, and release known failures The host CLI now writes the agent's lease store directly under its lock, always. The `docker compose exec` lease transport, the agent's host-CLI role and token, and the hidden `control lease` command are gone, so a `compose down` that removes the agent no longer leaves its lease held. Data-manager settles the hardlink child it ran under the agent's or the host CLI's lease through a new `lease.settleChild` agent operation (or the store, where it reads it), after its terminal, at startup repair and in recovery. A known outcome is recorded even when the issuer's own observation was interrupted; the same result recorded twice changes nothing. The agent claims its targets once, before it records the job: a held target answers busy with nothing written. The busy reply and the CLI message name the holding operation and point to `openmapx control status`. A leased stop, restart, remove, stack stop or backup whose Docker command ran to its end and reported a failure records a failed result and releases its lease, in the agent and in the host CLI. A command killed by a signal, timed out, cancelled or cut off from the Docker daemon stays unknown and keeps its lease for `openmapx control resolve`. A non-zero compose exit is a failure, not a success. Data-manager claims its lease before it begins: a busy target or an agent that cannot answer ends the execution as failed with its publication unchanged, so it never fences a configuration maintenance. `openmapx control resolve` of a maintenance settles the data-manager fence with the CLI's own data-manager token and the inputs the agent recorded on the fence child in the lease store; data-manager accepts that token's settle as the operator's resolution. `control clear` of a maintenance warns that its fence stays prepared until `control resolve` settles it, which still works after the clear. The fence row no longer stores the maintenance lease id; migration 0064 only drops the old certificate column. --- ...4_configuration_fence_drop_certificate.sql | 1 + .../0064_configuration_fence_owner_lease.sql | 4 - .../src/db/migrations/meta/0064_snapshot.json | 8 +- apps/api/src/db/migrations/meta/_journal.json | 4 +- .../admin-artifact-transfers.test.ts | 2 +- .../services/__tests__/admin-job-ops.test.ts | 2 - ...configuration-maintenance-postgres.test.ts | 8 - .../coordinator-lifecycle-postgres.test.ts | 1 - .../data-configuration-fence-postgres.test.ts | 5 +- ...orm-configuration-runtime-postgres.test.ts | 1 - .../retained-maintenance-job-postgres.test.ts | 10 +- .../migration-postgres.test.ts | 1 - .../extension-workflow-postgres.test.ts | 1 - .../services/integration-application.test.ts | 1 - .../standalone-integrations/install.test.ts | 1 - .../transit-feed-credentials-postgres.test.ts | 1 - apps/ops-agent/src/__fixtures__/shutdown.ts | 2 +- .../artifact-transfer/import-runtime.test.ts | 6 +- .../src/artifact-transfer/ingress.test.ts | 2 +- .../src/artifact-transfer/motis.test.ts | 2 +- .../src/backup-prune-runtime.test.ts | 4 +- apps/ops-agent/src/bootstrap.test.ts | 1 - apps/ops-agent/src/catalog-pins.test.ts | 2 +- .../src/configuration-handoff.test.ts | 2 +- .../src/configuration-maintenance-store.ts | 7 +- apps/ops-agent/src/control-admission.test.ts | 4 +- .../src/control-terminal-restart.test.ts | 5 +- .../src/core-service-runtime.test.ts | 2 +- .../src/data-transit-resource-source.test.ts | 2 +- apps/ops-agent/src/docker-runtime.ts | 24 ++- apps/ops-agent/src/index.ts | 3 +- apps/ops-agent/src/ingress-rate-limit.test.ts | 1 - apps/ops-agent/src/integration-store.test.ts | 2 - apps/ops-agent/src/lease-operations.test.ts | 137 ++++++++++------ apps/ops-agent/src/lease-operations.ts | 37 +++-- apps/ops-agent/src/production-runtime.test.ts | 5 +- apps/ops-agent/src/server-foundation.test.ts | 101 ++++++++++-- apps/ops-agent/src/server-review-2.test.ts | 38 ++--- apps/ops-agent/src/server.test.ts | 34 ++-- apps/ops-agent/src/server.ts | 152 +++++++++++------- .../test-fixtures/data-owned-http-process.ts | 2 +- .../terminal-retention-process.ts | 2 +- .../src/traffic-target-ownership.test.ts | 2 +- .../src/trusted-config-ownership.test.ts | 1 - packages/cli/__tests__/cli-control.test.ts | 64 +++++++- .../cli/__tests__/cli-host-effect.test.ts | 50 ++++-- packages/cli/src/commands/control.ts | 65 +++++--- packages/cli/src/lib/artifact-ownership.ts | 2 - packages/cli/src/lib/control-resolve.ts | 65 ++------ packages/cli/src/lib/data-execution.ts | 19 +-- packages/cli/src/lib/data-update-command.ts | 2 - packages/cli/src/lib/docker.ts | 15 +- packages/cli/src/lib/host-effect.ts | 64 ++++++-- packages/cli/src/lib/host-leases.ts | 54 ------- packages/cli/src/lib/service-lifecycle.ts | 1 - packages/cli/src/lib/transitous-api-keys.ts | 2 - .../src/build-artifacts/control-authority.ts | 16 ++ .../data-configuration-fence.ts | 52 +++--- .../core/src/build-artifacts/lease-remote.ts | 10 +- .../core/src/build-artifacts/lease-store.ts | 26 ++- .../core/src/build-artifacts/target-lease.ts | 85 ++++++++-- packages/core/src/ops/client.ts | 23 ++- packages/core/src/ops/contract.test.ts | 17 ++ packages/core/src/ops/contract.ts | 39 ++++- packages/core/src/ops/lease-records.ts | 5 +- packages/core/src/services/agent-state.ts | 23 +-- .../data-manager/configuration-maintenance.ts | 10 +- .../contracts/configuration-maintenance.ts | 1 - .../inspectDataConfigurationFenceParams.ts | 2 - packages/core/src/services/index.ts | 2 - packages/db-schema/src/data-manager.ts | 2 - .../transitous/catalog-review.test.ts | 2 +- services/data-manager/openapi.json | 10 -- .../src/jobs/execution/api.test.ts | 2 +- .../data-manager/src/jobs/execution/api.ts | 2 +- .../src/jobs/execution/fixed-cli.test.ts | 2 +- .../src/jobs/execution/full-update.test.ts | 2 +- .../legacy-admission-postgres.test.ts | 1 - .../src/jobs/execution/ops-children.test.ts | 2 +- .../src/jobs/execution/runtime.ts | 34 +++- 80 files changed, 863 insertions(+), 543 deletions(-) create mode 100644 apps/api/src/db/migrations/0064_configuration_fence_drop_certificate.sql delete mode 100644 apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql delete mode 100644 packages/cli/src/lib/host-leases.ts diff --git a/apps/api/src/db/migrations/0064_configuration_fence_drop_certificate.sql b/apps/api/src/db/migrations/0064_configuration_fence_drop_certificate.sql new file mode 100644 index 000000000..285556b2b --- /dev/null +++ b/apps/api/src/db/migrations/0064_configuration_fence_drop_certificate.sql @@ -0,0 +1 @@ +ALTER TABLE "data_manager"."configuration_fence" DROP COLUMN "owner_certificate_digest"; \ No newline at end of file diff --git a/apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql b/apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql deleted file mode 100644 index e693102d8..000000000 --- a/apps/api/src/db/migrations/0064_configuration_fence_owner_lease.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Fences name the agent's lease now; rows bound to the old signed certificates cannot be kept. -DELETE FROM "data_manager"."configuration_fence";--> statement-breakpoint -ALTER TABLE "data_manager"."configuration_fence" ADD COLUMN "owner_lease_id" uuid NOT NULL;--> statement-breakpoint -ALTER TABLE "data_manager"."configuration_fence" DROP COLUMN "owner_certificate_digest"; \ No newline at end of file diff --git a/apps/api/src/db/migrations/meta/0064_snapshot.json b/apps/api/src/db/migrations/meta/0064_snapshot.json index c5df3929f..3bdbe40c5 100644 --- a/apps/api/src/db/migrations/meta/0064_snapshot.json +++ b/apps/api/src/db/migrations/meta/0064_snapshot.json @@ -1,5 +1,5 @@ { - "id": "12f39f65-a4b7-4cf2-9250-1d2565604fb2", + "id": "3877f309-a95f-4098-a80d-f59c87d61ba5", "prevId": "6612793a-c6b1-45e2-ad89-200a4214a323", "version": "7", "dialect": "postgresql", @@ -1659,12 +1659,6 @@ "primaryKey": false, "notNull": true }, - "owner_lease_id": { - "name": "owner_lease_id", - "type": "uuid", - "primaryKey": false, - "notNull": true - }, "input": { "name": "input", "type": "jsonb", diff --git a/apps/api/src/db/migrations/meta/_journal.json b/apps/api/src/db/migrations/meta/_journal.json index 61b07b420..03a189687 100644 --- a/apps/api/src/db/migrations/meta/_journal.json +++ b/apps/api/src/db/migrations/meta/_journal.json @@ -453,8 +453,8 @@ { "idx": 64, "version": "7", - "when": 1794734781139, - "tag": "0064_configuration_fence_owner_lease", + "when": 1794754360788, + "tag": "0064_configuration_fence_drop_certificate", "breakpoints": true } ] diff --git a/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts b/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts index a3bfbca75..46dd32378 100644 --- a/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts +++ b/apps/api/src/routes/__tests__/admin-artifact-transfers.test.ts @@ -55,7 +55,7 @@ it("generated SDK streams through real admin/API and typed/raw agent paths, enfo const audit: unknown[] = []; const agent = buildOpsAgentServer({ audit: (event) => audit.push(event), - tokens: { api: token, "data-manager": "b".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: token, "data-manager": "b".repeat(43) }, runtime, artifactTransfers: store, resourcePolicy: { allowGlobal: () => true }, diff --git a/apps/api/src/services/__tests__/admin-job-ops.test.ts b/apps/api/src/services/__tests__/admin-job-ops.test.ts index 740196c7b..3cc7a6654 100644 --- a/apps/api/src/services/__tests__/admin-job-ops.test.ts +++ b/apps/api/src/services/__tests__/admin-job-ops.test.ts @@ -794,7 +794,6 @@ it.each([false, true])( tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url"), - "host-cli": "h".repeat(43), }, journal, controlInstallation: installation, @@ -860,7 +859,6 @@ it.each([false, true])( tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url"), - "host-cli": "h".repeat(43), }, journal: reloaded, controlInstallation: installation, diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index 108685508..1bacbb289 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -235,7 +235,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, @@ -3752,7 +3751,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -3980,7 +3978,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -4635,7 +4632,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -4947,7 +4943,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -5125,7 +5120,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime, resourceClaimer: host.createResourceClaimer(runtime), @@ -5391,7 +5385,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), @@ -5438,7 +5431,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: restartedRuntime, resourceClaimer: host.createResourceClaimer(restartedRuntime), diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index dedcae862..3e3684ee8 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -433,7 +433,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: { diff --git a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts index 0857c1d42..e9c7c0cab 100644 --- a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts +++ b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts @@ -344,8 +344,8 @@ describe.skipIf( } } // A fence row for the same maintenance means the preparation was admitted. - await f.sql`INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,owner_lease_id,input,state) - VALUES(${f.input.maintenanceId},${randomUUID()},${`opk1_${randomUUID().replaceAll("-", "")}`},${f.input.candidateId},${f.input.planDigest},${randomUUID()},${JSON.stringify(f.input)}::jsonb,'prepared')`; + await f.sql`INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,input,state) + VALUES(${f.input.maintenanceId},${randomUUID()},${`opk1_${randomUUID().replaceAll("-", "")}`},${f.input.candidateId},${f.input.planDigest},${JSON.stringify(f.input)}::jsonb,'prepared')`; try { expect((await request()).status).not.toBe(200); } finally { @@ -986,7 +986,6 @@ async function ownedFenceFixture( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, journal: await openOpsJobJournal(join(root, "state/ops-agent/jobs-v1.json")), }), diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 3d64b4b00..5dd94cdcd 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -765,7 +765,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index 2ae865417..6fff9c2f0 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -140,7 +140,7 @@ dbDescribe("retained finite maintenance requests", () => { journal, controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); agent.addHook("preHandler", async (request, reply) => { const operation = ( @@ -353,7 +353,7 @@ dbDescribe("retained finite maintenance requests", () => { resourceClaimer: host.createResourceClaimer(runtime), controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); agent.addHook("onSend", async (request, reply, payload) => { const op = ( @@ -516,7 +516,7 @@ dbDescribe("retained finite maintenance requests", () => { resourceClaimer: host.claimer, journal, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); const tokenFile = join(host.root, "render-api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); @@ -625,7 +625,7 @@ dbDescribe("retained finite maintenance requests", () => { resourceClaimer: host.claimer, journal, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); let lost = false; agent.addHook("onSend", async (request, reply, payload) => { @@ -820,7 +820,6 @@ dbDescribe("retained finite maintenance requests", () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, @@ -1035,7 +1034,6 @@ dbDescribe("retained finite maintenance requests", () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime, resourceClaimer: host.claimer, diff --git a/apps/api/src/services/configuration-migration/migration-postgres.test.ts b/apps/api/src/services/configuration-migration/migration-postgres.test.ts index f58a81620..bc492fdc3 100644 --- a/apps/api/src/services/configuration-migration/migration-postgres.test.ts +++ b/apps/api/src/services/configuration-migration/migration-postgres.test.ts @@ -108,7 +108,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, diff --git a/apps/api/src/services/extension-workflow-postgres.test.ts b/apps/api/src/services/extension-workflow-postgres.test.ts index a133027be..f68f3b5ee 100644 --- a/apps/api/src/services/extension-workflow-postgres.test.ts +++ b/apps/api/src/services/extension-workflow-postgres.test.ts @@ -86,7 +86,6 @@ beforeAll(async () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, diff --git a/apps/api/src/services/integration-application.test.ts b/apps/api/src/services/integration-application.test.ts index 92e56501a..dd76b3412 100644 --- a/apps/api/src/services/integration-application.test.ts +++ b/apps/api/src/services/integration-application.test.ts @@ -131,7 +131,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, diff --git a/apps/api/src/services/standalone-integrations/install.test.ts b/apps/api/src/services/standalone-integrations/install.test.ts index 9a1477137..77c7b06e5 100644 --- a/apps/api/src/services/standalone-integrations/install.test.ts +++ b/apps/api/src/services/standalone-integrations/install.test.ts @@ -76,7 +76,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( tokens: { api: state.host.token, "data-manager": "b".repeat(43), - "host-cli": "h".repeat(43), }, runtime: state.host.runtime, resourceClaimer: state.host.claimer, diff --git a/apps/api/src/services/transit-feed-credentials-postgres.test.ts b/apps/api/src/services/transit-feed-credentials-postgres.test.ts index 659c45699..7b40c24af 100644 --- a/apps/api/src/services/transit-feed-credentials-postgres.test.ts +++ b/apps/api/src/services/transit-feed-credentials-postgres.test.ts @@ -119,7 +119,6 @@ describe.skipIf(!enabled)( tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, diff --git a/apps/ops-agent/src/__fixtures__/shutdown.ts b/apps/ops-agent/src/__fixtures__/shutdown.ts index 3fedee8e3..0ac064b34 100644 --- a/apps/ops-agent/src/__fixtures__/shutdown.ts +++ b/apps/ops-agent/src/__fixtures__/shutdown.ts @@ -7,7 +7,7 @@ let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); process.on("message", (message) => { if (message === "release") release(); }); const app = await startOpsAgentServer({ - tokens: { api: "shutdown-fixture-token", "data-manager": "shutdown-dm-token", "host-cli": "h".repeat(43) }, + tokens: { api: "shutdown-fixture-token", "data-manager": "shutdown-dm-token" }, resourcePolicy: createRegistryResourcePolicy({ serviceIds: [], integrationIds: [], trustedData: denyAllTrustedOpsData }), dispatch: async () => { process.send?.({ event: "working" }); diff --git a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts index 90dc9a7eb..373e55bc2 100644 --- a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts +++ b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts @@ -71,7 +71,7 @@ async function fixture() { resourceClaimer: host.claimer, controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); let lose = false; app.addHook("onSend", async (request, reply, payload) => { @@ -473,7 +473,7 @@ it("reconciles the original prepared publication after exact foreign-inode repai resourceClaimer: f.createResourceClaimer(runtime), controlInstallation: installation, artifactTransfers: store, - tokens: { api: f.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: f.token, "data-manager": "d".repeat(43) }, }); cleanups.push(() => app.close()); const freshClient = new OpsClient({ @@ -678,7 +678,7 @@ async function reassembleCommittedFixture(f: Awaited> resourceClaimer: f.createResourceClaimer(runtime), controlInstallation: installation, artifactTransfers: store, - tokens: { api: f.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: f.token, "data-manager": "d".repeat(43) }, }); cleanups.push(() => app.close()); const freshClient = new OpsClient({ diff --git a/apps/ops-agent/src/artifact-transfer/ingress.test.ts b/apps/ops-agent/src/artifact-transfer/ingress.test.ts index a24a1f2b9..3ee885c2c 100644 --- a/apps/ops-agent/src/artifact-transfer/ingress.test.ts +++ b/apps/ops-agent/src/artifact-transfer/ingress.test.ts @@ -47,7 +47,7 @@ it.skipIf(process.env.RUN_ARTIFACT_INGRESS_TESTS !== "1")( const runtime = createUnavailableRuntime(); installArtifactTransferRuntime(runtime, store); const agent = buildOpsAgentServer({ - tokens: { api: token, "data-manager": "b".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: token, "data-manager": "b".repeat(43) }, runtime, artifactTransfers: store, resourcePolicy: { allowGlobal: () => true }, diff --git a/apps/ops-agent/src/artifact-transfer/motis.test.ts b/apps/ops-agent/src/artifact-transfer/motis.test.ts index ca1072d98..e7e3f06c0 100644 --- a/apps/ops-agent/src/artifact-transfer/motis.test.ts +++ b/apps/ops-agent/src/artifact-transfer/motis.test.ts @@ -462,7 +462,7 @@ it("imports an actual portable candidate through the production agent with desti resourceClaimer: host.claimer, controlInstallation: installation, artifactTransfers: store, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); const token = join(host.root, "portable-token"); writeFileSync(token, host.token, { mode: 0o600 }); diff --git a/apps/ops-agent/src/backup-prune-runtime.test.ts b/apps/ops-agent/src/backup-prune-runtime.test.ts index cad726473..f486eb0ee 100644 --- a/apps/ops-agent/src/backup-prune-runtime.test.ts +++ b/apps/ops-agent/src/backup-prune-runtime.test.ts @@ -74,7 +74,7 @@ it.each([7, 30, 365])( journal, resourceClaimer: host.claimer, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); const tokenFile = join(host.root, "api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); @@ -178,7 +178,7 @@ it("previews and prunes only the unchanged original eligible set through the pro resourceClaimer: host.claimer, journal, controlInstallation: installation, - tokens: { api: host.token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(43) }, }); const baseUrl = await app.listen({ host: "127.0.0.1", port: 0 }), client = new OpsClient({ baseUrl, tokenFile }); diff --git a/apps/ops-agent/src/bootstrap.test.ts b/apps/ops-agent/src/bootstrap.test.ts index 6807dc9ae..d89885db2 100644 --- a/apps/ops-agent/src/bootstrap.test.ts +++ b/apps/ops-agent/src/bootstrap.test.ts @@ -126,7 +126,6 @@ it("production bootstrap retains health and observation when the release store i tokens: { api: token, "data-manager": Buffer.alloc(32, 2).toString("base64url"), - "host-cli": "h".repeat(43), }, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, journal: { records: () => [retained], replace: async () => undefined }, diff --git a/apps/ops-agent/src/catalog-pins.test.ts b/apps/ops-agent/src/catalog-pins.test.ts index 2bad8fc91..d7acbdb8b 100644 --- a/apps/ops-agent/src/catalog-pins.test.ts +++ b/apps/ops-agent/src/catalog-pins.test.ts @@ -52,7 +52,7 @@ it("serializes the largest legal paired pins through the actual HTTP server/clie }); const runtime = createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: root }); const app = buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dmToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dmToken }, runtime, resourcePolicy: createRegistryResourcePolicy({ serviceIds: ["motis"], diff --git a/apps/ops-agent/src/configuration-handoff.test.ts b/apps/ops-agent/src/configuration-handoff.test.ts index 8ae2d009b..8ed88e1e7 100644 --- a/apps/ops-agent/src/configuration-handoff.test.ts +++ b/apps/ops-agent/src/configuration-handoff.test.ts @@ -391,7 +391,7 @@ it("uses actual server job state when excluding a live log observer from drain", release = Promise.withResolvers(), checked = Promise.withResolvers(); const app = buildOpsAgentServer({ - tokens: { api, "data-manager": dm, "host-cli": "h".repeat(43) }, + tokens: { api, "data-manager": dm }, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, dispatch: async (operation, _runtime, context) => { if (operation.kind === "service.logs.follow") { diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 985c9fd1d..115f291b0 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -334,7 +334,6 @@ function inspectOwnedFence(owner: ConfigurationMaintenanceOwnerRecord, signal?: return inspectDataConfigurationFence(owner.plan.operation.maintenanceId, { transport: configurationDataManagerTransport(), candidateId: owner.plan.operation.candidateId, - ownerLeaseId: owner.leaseId, signal, }); } @@ -3606,6 +3605,12 @@ export function registerConfigurationMaintenanceSource(runtime: OpsRuntime, sour ordinal: reserved.ordinal, lease, childPublicKey: controlRolePublicKey(state.installation.installation, "data-manager"), + // What an operator's `openmapx control resolve` needs to settle the fence. + evidence: { + maintenanceId: plan.dmPrepare.maintenanceId, + candidateId: plan.dmPrepare.candidateId, + planDigest: plan.dmPrepare.planDigest, + }, }); consumeControlChildDispatch(issued.dispatchTicket); assertion = issued.assertion; diff --git a/apps/ops-agent/src/control-admission.test.ts b/apps/ops-agent/src/control-admission.test.ts index 3b2131b9d..52b7ec663 100644 --- a/apps/ops-agent/src/control-admission.test.ts +++ b/apps/ops-agent/src/control-admission.test.ts @@ -28,7 +28,7 @@ it("refuses paired approval as a bearer-only root before reserving a worker", as const app = buildOpsAgentServer({ controlInstallation: installation, journal, - tokens: { api: "a".repeat(32), "data-manager": "b".repeat(32), "host-cli": "h".repeat(43) }, + tokens: { api: "a".repeat(32), "data-manager": "b".repeat(32) }, resourcePolicy: { allowGlobal: () => true }, }); try { @@ -151,7 +151,7 @@ it.each(["normal", "ready-failure", "wrong-target", "canceled", "cancel-journal- const token = "b".repeat(32); const journalFile = coreServices.agentStatePaths(root).journal; const options = { - tokens: { api: "a".repeat(32), "data-manager": token, "host-cli": "h".repeat(43) }, + tokens: { api: "a".repeat(32), "data-manager": token }, controlInstallation: agent, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, dispatch: async (_operation: unknown, _runtime: unknown, context: OpsExecutionContext) => { diff --git a/apps/ops-agent/src/control-terminal-restart.test.ts b/apps/ops-agent/src/control-terminal-restart.test.ts index cc9a6d206..4e6b74dbe 100644 --- a/apps/ops-agent/src/control-terminal-restart.test.ts +++ b/apps/ops-agent/src/control-terminal-restart.test.ts @@ -166,7 +166,7 @@ it.each( }); const journal = await openOpsJobJournal(file); app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": "b".repeat(32), "host-cli": "h".repeat(43) }, + tokens: { api: token, "data-manager": "b".repeat(32) }, controlInstallation: installation, journal, dispatch, @@ -310,7 +310,6 @@ it.each( tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32), - "host-cli": "h".repeat(43), }, controlInstallation: installation, journal, @@ -335,7 +334,6 @@ it.each( tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32), - "host-cli": "h".repeat(43), }, controlInstallation: installation, journal: unproved, @@ -502,7 +500,6 @@ it("repairs a borrowed terminal worker after the actual canonical-digest restart tokens: { api: Buffer.alloc(32, 7).toString("base64url"), "data-manager": "b".repeat(32), - "host-cli": "h".repeat(43), }, controlInstallation: installation, journal, diff --git a/apps/ops-agent/src/core-service-runtime.test.ts b/apps/ops-agent/src/core-service-runtime.test.ts index daae15a5c..cf79c8eef 100644 --- a/apps/ops-agent/src/core-service-runtime.test.ts +++ b/apps/ops-agent/src/core-service-runtime.test.ts @@ -83,7 +83,7 @@ describe("production core restart workflow", () => { }); const journal = await openOpsJobJournal(services.agentStatePaths(host.infraDir).journal); const app = buildOpsAgentServer({ - tokens: { api: host.token, "data-manager": "d".repeat(32), "host-cli": "h".repeat(43) }, + tokens: { api: host.token, "data-manager": "d".repeat(32) }, runtime, resourceClaimer: host.claimer, journal, diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts index ca4e42c3f..a865a97bb 100644 --- a/apps/ops-agent/src/data-transit-resource-source.test.ts +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -178,7 +178,7 @@ async function fixture( ...(admissionPolicy === "late" ? { now: () => new Date(Date.now() + 120000) } : {}), controlInstallation: agent, resourceClaimer: claimer, - tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, + tokens: { api: "a".repeat(43), "data-manager": token }, }); await app.listen({ host: "127.0.0.1", port: 0 }); const address = app.server.address(); diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 306a3121c..7c4bf2727 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -39,6 +39,18 @@ const containedProcessFailures = new WeakSet(); export function isContainedProcessFailure(error: unknown): boolean { return typeof error === "object" && error !== null && containedProcessFailures.has(error); } +const reportedProcessFailures = new WeakSet(); +/** A Docker client cut off from the daemon cannot say how far its command got. */ +const LOST_DAEMON = + /Cannot connect to the Docker daemon|error during connect|connection reset by peer/i; +/** + * A command that ran to its own end and reported a failure with an exit code: + * not killed, timed out or cancelled, nothing left running, and the Docker + * daemon still answered it. Its outcome is known even though it failed. + */ +export function isReportedProcessFailure(error: unknown): boolean { + return typeof error === "object" && error !== null && reportedProcessFailures.has(error); +} import { openAnchoredDirectory } from "./anchored-directory"; import type { PersistedOpsJob } from "./journal"; @@ -183,7 +195,17 @@ export async function runContainedStreamingProcess( exit.signal !== null ) { const error = new OpsExecutionFailure(timedOut ? "timeout" : "runtime", { diagnostics }); - if (!exit.residualDescendants && !exit.containmentFailed) containedProcessFailures.add(error); + if (!exit.residualDescendants && !exit.containmentFailed) { + containedProcessFailures.add(error); + if ( + !timedOut && + !options.signal.aborted && + !exit.spawnFailed && + exit.signal === null && + !diagnostics.tail.some((line) => LOST_DAEMON.test(line.message)) + ) + reportedProcessFailures.add(error); + } throw error; } return diagnostics; diff --git a/apps/ops-agent/src/index.ts b/apps/ops-agent/src/index.ts index b0af492e7..4e585677e 100644 --- a/apps/ops-agent/src/index.ts +++ b/apps/ops-agent/src/index.ts @@ -162,7 +162,6 @@ async function main(): Promise { container: existsSync("/run/secrets/openmapx-control-verifiers"), leases: localLeaseBackend(leaseStore), }); - const hostCliToken = services.ensureHostCliToken(paths.infraDir); if (controlStartup.reason) console.error(controlStartup.reason); if (controlStartup.installation) { try { @@ -285,7 +284,7 @@ async function main(): Promise { const initialPolicyAuthority = await policyAuthority(); await startOpsAgentServer( { - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": hostCliToken }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, leaseStore, resourceClaimer: createProductionRegistryResourceClaimer({ services: initialPolicyAuthority.services, diff --git a/apps/ops-agent/src/ingress-rate-limit.test.ts b/apps/ops-agent/src/ingress-rate-limit.test.ts index 3002e0b78..1eb6274e2 100644 --- a/apps/ops-agent/src/ingress-rate-limit.test.ts +++ b/apps/ops-agent/src/ingress-rate-limit.test.ts @@ -91,7 +91,6 @@ it("shares the actual server ingress budget across operations and artifact route return "a".repeat(43); }, "data-manager": "b".repeat(43), - "host-cli": "h".repeat(43), }, artifactTransfers: store, }); diff --git a/apps/ops-agent/src/integration-store.test.ts b/apps/ops-agent/src/integration-store.test.ts index 740bb86ad..cc0cb88c3 100644 --- a/apps/ops-agent/src/integration-store.test.ts +++ b/apps/ops-agent/src/integration-store.test.ts @@ -297,7 +297,6 @@ describe("integration store", () => { tokens: { api: apiToken, "data-manager": Buffer.alloc(32, 32).toString("base64url"), - "host-cli": "h".repeat(43), }, resourcePolicy: allowAllResources, journal, @@ -389,7 +388,6 @@ describe("integration store", () => { tokens: { api: apiToken, "data-manager": Buffer.alloc(32, 32).toString("base64url"), - "host-cli": "h".repeat(43), }, resourcePolicy: allowAllResources, journal, diff --git a/apps/ops-agent/src/lease-operations.test.ts b/apps/ops-agent/src/lease-operations.test.ts index eb973bf40..177c34322 100644 --- a/apps/ops-agent/src/lease-operations.test.ts +++ b/apps/ops-agent/src/lease-operations.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { type ControlInstallation, closeControlInstallation, + leaseRules, leaseStoreDirectory, openControlInstallation, openLeaseStore, @@ -31,22 +32,23 @@ afterEach(() => { rmSync(root, { recursive: true, force: true }); }); const store = () => openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }); -function owner(executor: "data-manager" | "host-cli"): ExecutionRef { +function owner(executor: "data-manager" | "ops-agent" | "host-cli"): ExecutionRef { return { executor, - principal: executor, - executionId: randomUUID(), + principal: executor === "ops-agent" ? "api" : executor, + executionId: + executor === "ops-agent" ? `job1_${randomUUID().replaceAll("-", "")}` : randomUUID(), operationKey: `opk1_${randomUUID().replaceAll("-", "")}`, operationFingerprint: "a".repeat(64), fingerprint: "b".repeat(64), deadlineAt: Date.now() + 60_000, }; } -const acquire = (role: "data-manager" | "host-cli", execution: ExecutionRef, targets: string[]) => - runLeaseOperation(installation, store(), role, { +const acquire = (execution: ExecutionRef, targets: string[]) => + runLeaseOperation(installation, store(), "data-manager", { kind: "lease.acquire", owner: execution, - ownerKind: role === "host-cli" ? "service.restart" : "transit-sync", + ownerKind: "transit-sync", targets, }); @@ -57,35 +59,44 @@ function leaseOf(result: unknown): string { return value.lease.leaseId; } -describe("lease operations for data-manager and the host CLI", () => { - it("acquires for the caller's own execution on registered targets, community ones included", async () => { - const result = await acquire("host-cli", owner("host-cli"), ["community-live"]); - expect(result).toMatchObject({ outcome: "acquired", lease: { targets: ["community-live"] } }); - await expect(acquire("host-cli", owner("data-manager"), ["osrm"])).rejects.toBeInstanceOf( - LeaseRefused, - ); - await expect( - acquire("data-manager", owner("data-manager"), ["unknown"]), - ).rejects.toBeInstanceOf(LeaseRefused); +describe("lease operations for data-manager", () => { + it("acquires for data-manager's own execution on registered targets only", async () => { + expect(await acquire(owner("data-manager"), ["osrm"])).toMatchObject({ + outcome: "acquired", + lease: { targets: ["osrm"] }, + }); + await expect(acquire(owner("ops-agent"), ["redis"])).rejects.toBeInstanceOf(LeaseRefused); + await expect(acquire(owner("data-manager"), ["unknown"])).rejects.toBeInstanceOf(LeaseRefused); }); it("answers busy with the holder instead of failing", async () => { const holder = owner("data-manager"); - await acquire("data-manager", holder, ["osrm"]); - expect(await acquire("host-cli", owner("host-cli"), ["osrm"])).toMatchObject({ + await acquire(holder, ["osrm"]); + expect(await acquire(owner("data-manager"), ["osrm"])).toMatchObject({ outcome: "busy", target: "osrm", holder: { owner: holder, ownerKind: "transit-sync", state: "held" }, }); }); - it("shows and changes only the caller's own leases", async () => { - const leaseId = leaseOf(await acquire("data-manager", owner("data-manager"), ["redis"])); + it("shows and changes only data-manager's own leases, and refuses other callers", async () => { + const agent = owner("ops-agent"); + const leaseId = await store().update( + (leases) => + leaseRules.acquire( + leases, + { owner: agent, ownerKind: "service.restart", targets: ["redis"] }, + Date.now(), + ).leaseId, + ); expect( - await runLeaseOperation(installation, store(), "host-cli", { kind: "lease.read", leaseId }), + await runLeaseOperation(installation, store(), "data-manager", { + kind: "lease.read", + leaseId, + }), ).toEqual({ lease: null }); expect( - await runLeaseOperation(installation, store(), "host-cli", { + await runLeaseOperation(installation, store(), "data-manager", { kind: "lease.update", leaseId, action: { action: "release" }, @@ -97,35 +108,65 @@ describe("lease operations for data-manager and the host CLI", () => { ).rejects.toBeInstanceOf(LeaseRefused); }); - it("lets the host CLI delegate only to data-manager, from the lease's own owner", async () => { + it("settles a hardlink child data-manager ran under another's lease, once", async () => { const cli = owner("host-cli"); - const leaseId = leaseOf( - await runLeaseOperation(installation, store(), "host-cli", { - kind: "lease.acquire", - owner: cli, - ownerKind: "data.update", - targets: ["osrm"], - }), - ); - const child = { - key: randomUUID(), - parentKey: null, - parent: cli, - execution: { ...owner("data-manager"), deadlineAt: cli.deadlineAt }, - kind: "download-osm", - phase: "osm", - recipient: "ops-agent" as const, - targets: ["osrm"], - permittedFurtherEdges: [], - depth: 1, + const leaf = owner("data-manager"); + const leaseId = await store().update((leases) => { + const lease = leaseRules.acquire( + leases, + { owner: cli, ownerKind: "data.link", targets: ["osrm"] }, + Date.now(), + ); + leaseRules.delegate( + leases, + { + leaseId: lease.leaseId, + child: { + key: randomUUID(), + parentKey: null, + parent: cli, + execution: leaf, + kind: "data.link", + phase: "hardlinks", + recipient: "data-manager", + targets: ["osrm"], + permittedFurtherEdges: [], + depth: 1, + }, + }, + Date.now(), + ); + return lease.leaseId; + }); + const settlement = { + identity: leaf, + outcome: "committed" as const, + publication: "committed" as const, + terminalReceiptDigest: "c".repeat(64), }; + // Another role's execution is never settled by data-manager. await expect( - runLeaseOperation(installation, store(), "host-cli", { - kind: "lease.update", - leaseId, - action: { action: "delegate", child }, + runLeaseOperation(installation, store(), "data-manager", { + kind: "lease.settleChild", + settlement: { ...settlement, identity: cli }, + }), + ).rejects.toBeInstanceOf(LeaseRefused); + await runLeaseOperation(installation, store(), "data-manager", { + kind: "lease.settleChild", + settlement, + }); + expect(store().get(leaseId)?.children[0]).toMatchObject({ state: "settled", settlement }); + // The issuer's own observation of the same result changes nothing. + await runLeaseOperation(installation, store(), "data-manager", { + kind: "lease.settleChild", + settlement: { ...settlement, terminalReceiptDigest: "d".repeat(64) }, + }); + expect(store().get(leaseId)?.children[0]?.settlement).toEqual(settlement); + await expect( + runLeaseOperation(installation, store(), "data-manager", { + kind: "lease.settleChild", + settlement: { ...settlement, outcome: "contained" }, }), - ).rejects.toThrow(); - expect(store().get(leaseId)?.children).toEqual([]); + ).rejects.toThrow("Original settlement changed"); }); }); diff --git a/apps/ops-agent/src/lease-operations.ts b/apps/ops-agent/src/lease-operations.ts index b4c97765e..07e6b4bfb 100644 --- a/apps/ops-agent/src/lease-operations.ts +++ b/apps/ops-agent/src/lease-operations.ts @@ -17,20 +17,15 @@ import { childPhaseKinds } from "../../../packages/core/src/build-artifacts/cont type LeaseOperation = Extract< OpsOperation, - { kind: "lease.acquire" | "lease.read" | "lease.update" } + { kind: "lease.acquire" | "lease.read" | "lease.update" | "lease.settleChild" } >; -const recipients: Partial> = { - // Data-manager's children are the agent's own operations; the host CLI's are - // data-manager executions. - "data-manager": "ops-agent", - "host-cli": "data-manager", -}; /** - * Leases data-manager and the host CLI hold through the agent. Each caller - * sees and changes only leases its own executions own, and the same rules - * apply as to the agent's leases: one admission path, children recorded - * before they start, release only once everything has a known outcome. + * Leases data-manager holds through the agent. It sees and changes only + * leases its own executions own, and the same rules apply as to the agent's + * leases: one admission path, children recorded before they start, release + * only once everything has a known outcome. It also reports the outcome of a + * child it ran under the agent's or the host CLI's lease. */ export async function runLeaseOperation( installation: ControlInstallation, @@ -38,10 +33,18 @@ export async function runLeaseOperation( role: OpsRole, operation: LeaseOperation, ): Promise> { - if (role !== "data-manager" && role !== "host-cli") throw new LeaseRefused(); + if (role !== "data-manager") throw new LeaseRefused(); const owned = (lease: LeaseRecord | undefined) => lease && lease.owner.executor === role ? lease : undefined; const now = Date.now(); + if (operation.kind === "lease.settleChild") { + if (operation.settlement.identity.executor !== role) throw new LeaseRefused(); + return { + lease: + (await store.update((leases) => leaseRules.settleChild(leases, operation.settlement))) ?? + null, + }; + } if (operation.kind === "lease.acquire") { const registered = new Set(controlRegisteredServiceIds(installation)); if ( @@ -84,7 +87,7 @@ export async function runLeaseOperation( if ( child.parentKey !== null || child.depth !== 1 || - child.recipient !== recipients[role] || + child.recipient !== "ops-agent" || canonicalJson(child.parent) !== canonicalJson(current.owner) || canonicalJson(child.permittedFurtherEdges) !== canonicalJson(edges) || !(childPhaseKinds as Record)[child.phase]?.includes( @@ -96,12 +99,8 @@ export async function runLeaseOperation( return leaseRules.delegate(leases, { leaseId: current.leaseId, child }, now); } case "settle": { - if (action.key !== null) { - const child = current.children.find((value) => value.key === action.key); - // A caller settles only a child it issued whose recipient cannot reach the store. - if (child?.parentKey !== null || child.recipient !== "data-manager") - throw new LeaseRefused(); - } + // Data-manager's children are the agent's own operations, which settle themselves. + if (action.key !== null) throw new LeaseRefused(); return leaseRules.settle(leases, { leaseId: current.leaseId, ...action }); } case "cancel": diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index 4a48eda6c..aacf61165 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -190,7 +190,6 @@ describe("assembled production service authority", () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, @@ -312,7 +311,6 @@ describe("assembled production service authority", () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, @@ -429,7 +427,6 @@ describe("assembled production service authority", () => { tokens: { api: host.token, "data-manager": Buffer.alloc(32, 23).toString("base64url"), - "host-cli": "h".repeat(43), }, runtime: host.runtime, resourceClaimer: host.claimer, @@ -618,7 +615,7 @@ describe("assembled production service authority", () => { const api = Buffer.alloc(32, 31).toString("base64url"), dm = Buffer.alloc(32, 32).toString("base64url"); const app = buildOpsAgentServer({ - tokens: { api, "data-manager": dm, "host-cli": "h".repeat(43) }, + tokens: { api, "data-manager": dm }, runtime: host.runtime, resourceClaimer: host.claimer, }); diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index bb594fb22..092eb7805 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -84,7 +84,7 @@ describe("ops-agent async foundation", () => { return operation.kind === "service.pull" ? pullResult : { reachable: true }; }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 23), @@ -225,7 +225,7 @@ describe("ops-agent async foundation", () => { it("rejects resource and option mismatches for an exact retained terminal key", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 29), @@ -308,7 +308,7 @@ describe("ops-agent async foundation", () => { }, ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, idByte++), @@ -448,7 +448,7 @@ describe("ops-agent async foundation", () => { }, ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, journal, maxConcurrency: 1, @@ -514,7 +514,7 @@ describe("ops-agent async foundation", () => { let calls = 0; const audits: OpsAuditEvent[] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), operationTimeoutMs: 10, @@ -571,7 +571,7 @@ describe("ops-agent async foundation", () => { it("accepts only bounded UTF-8 log events from the typed execution context", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 24), @@ -629,7 +629,7 @@ describe("ops-agent async foundation", () => { it("reports a host update as its safe error code through the async protocol", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), dispatch: async () => { @@ -665,7 +665,7 @@ describe("ops-agent async foundation", () => { it("rejects oversized sync output and retains the maximum build-all receipt", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, 25), @@ -722,7 +722,7 @@ describe("ops-agent async foundation", () => { let nowMs = baseTime; let randomValue = 30; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(nowMs), randomBytes: () => Buffer.alloc(18, randomValue++), @@ -799,7 +799,7 @@ describe("ops-agent async foundation", () => { it("bounds entries that need recovery separately from capacity for new work", async () => { let randomValue = 60; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, now: () => new Date(baseTime), randomBytes: () => Buffer.alloc(18, randomValue++), @@ -867,7 +867,7 @@ describe("one admission path for target effects", () => { }); const audit: OpsAuditEvent[] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAllResources, controlInstallation: installation, leaseStore, @@ -903,8 +903,13 @@ describe("one admission path for target effects", () => { ] as const) { const refused = await submit(operation, name); expect(refused.statusCode).toBe(409); - expect(refused.json().error.class).toBe("busy"); + expect(refused.json().error).toEqual({ + class: "busy", + message: `Target redis is busy: held by ops-agent operation ${holder} (service.stop, held). See \`openmapx control status\`.`, + }); } + // A refusal records nothing: the leases are unchanged and no job exists. + expect(leaseStore.list().map((lease) => lease.owner.operationKey)).toEqual([holder]); expect( audit .filter((event) => event.heldBy?.includes(holder)) @@ -934,4 +939,76 @@ describe("one admission path for target effects", () => { rmSync(root, { recursive: true, force: true }); } }); + + it("releases the targets of a stop that reported a failure, and keeps those of a killed one", async () => { + const control = await import("@openmapx/core/build-artifacts"); + const { runContainedStreamingProcess } = await import("./docker-runtime"); + const { mkdtempSync, mkdirSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const root = mkdtempSync(join(tmpdir(), "omx-leased-failure-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["redis", "osrm"], + }); + const leaseStore = control.openLeaseStore({ + directory: control.leaseStoreDirectory(root), + purpose: "test", + }); + const app = await buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAllResources, + controlInstallation: installation, + leaseStore, + dispatch: async (operation, _runtime, context) => { + // The Docker command exits 3 for redis; for osrm it is killed by a signal. + const script = + "serviceId" in operation && operation.serviceId === "redis" ? "exit 3" : "kill -9 $$"; + await runContainedStreamingProcess("sh", ["-c", script], { + signal: context.signal, + timeout: 10_000, + maxBuffer: 4096, + }); + return { changed: true }; + }, + }); + try { + const submit = (operation: OpsOperation, name: string) => + app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(), + payload: { + ...request(operation, `ops1_${name}Fail`, key(`opk1_${name}Fail`)), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + }, + }); + for (const [serviceId, name] of [ + ["redis", "reportedStop000"], + ["osrm", "killedStop00000"], + ] as const) { + expect((await submit({ kind: "service.stop", serviceId }, name)).statusCode).toBe(202); + } + let settled = leaseStore.list(); + for ( + const endBy = Date.now() + 10_000; + Date.now() < endBy && + (settled.some((lease) => lease.targets.includes("redis")) || + !settled.some((lease) => lease.targets.includes("osrm") && lease.state !== "held")); + settled = leaseStore.list() + ) + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled.map((lease) => [lease.targets, lease.state])).toEqual([ + [["osrm"], "recovery_required"], + ]); + } finally { + await app.close(); + control.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/apps/ops-agent/src/server-review-2.test.ts b/apps/ops-agent/src/server-review-2.test.ts index 45cfac20e..8fd172b88 100644 --- a/apps/ops-agent/src/server-review-2.test.ts +++ b/apps/ops-agent/src/server-review-2.test.ts @@ -86,7 +86,7 @@ describe("reviewed durable admission boundary", () => { revisionId: `cfg1_${"t".repeat(43)}`, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, claimTimeoutMs: 5, resourceClaimer: { claim: async (_operation, fingerprint) => { @@ -144,7 +144,7 @@ describe("reviewed durable admission boundary", () => { revisionId: `cfg1_${"r".repeat(43)}`, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, journal: { records: () => records, replace: async (next) => { @@ -242,7 +242,7 @@ describe("reviewed durable admission boundary", () => { }); const dispatch = vi.fn(async () => pullResult); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal: { records: () => [tombstone], replace: async () => undefined }, maxJobEntries: 1, @@ -294,7 +294,7 @@ describe("reviewed durable admission boundary", () => { }; const dispatched: string[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: { claim: async (operation, fingerprint) => { sourceReference = operation; @@ -351,7 +351,7 @@ describe("reviewed durable admission boundary", () => { }; const dispatch = vi.fn(async () => pullResult); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime), @@ -391,7 +391,7 @@ describe("reviewed durable admission boundary", () => { }, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime), @@ -464,7 +464,7 @@ describe("reviewed durable admission boundary", () => { let releaseFirst!: () => void; const firstBlocked = new Promise((resolve) => (releaseFirst = resolve)); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime), @@ -560,7 +560,7 @@ describe("reviewed durable admission boundary", () => { return { changed: true }; }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), dispatch, @@ -599,7 +599,7 @@ describe("reviewed durable admission boundary", () => { result.lines[255] = "x".repeat(budget - baseBytes); expect(Buffer.byteLength(JSON.stringify(result), "utf8")).toBe(budget); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), dispatch: async () => result, @@ -651,7 +651,7 @@ describe("reviewed durable admission boundary", () => { }), ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal: { records: () => records, replace: async () => undefined }, now: () => new Date(baseTime), @@ -690,7 +690,7 @@ describe("reviewed durable admission boundary", () => { }); const dispatch = vi.fn(async () => pullResult); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime + 1_000), @@ -747,7 +747,7 @@ describe("reviewed durable admission boundary", () => { terminalAt: new Date(baseTime).toISOString(), }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal: { records: () => [existing], replace: async () => undefined }, now: () => new Date(baseTime), @@ -783,7 +783,7 @@ describe("reviewed durable admission boundary", () => { }; const seen: string[] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: claimer, claimTimeoutMs: 5, maxConcurrency: 1, @@ -856,7 +856,7 @@ describe("reviewed durable admission boundary", () => { }, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: claimer, maxConcurrency: 1, now: () => new Date(baseTime), @@ -881,7 +881,7 @@ describe("reviewed durable admission boundary", () => { it("contains a throwing audit sink on success and failure", async () => { let fail = false; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), audit: () => { @@ -923,7 +923,7 @@ describe("reviewed durable admission boundary", () => { const blocked = new Promise((resolve) => (release = resolve)); let calls = 0; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, now: () => new Date(baseTime), operationTimeoutMs: 5, @@ -1004,7 +1004,7 @@ describe("asynchronous operation slots", () => { it("a throw during terminal bookkeeping releases the concurrency slot", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: releasingClaimer(() => { throw new Error("release failed synchronously"); }), @@ -1031,7 +1031,7 @@ describe("asynchronous operation slots", () => { const blocked = new Promise((resolve) => (unblock = resolve)); let calls = 0; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, maxConcurrency: 2, now: () => new Date(baseTime), @@ -1061,7 +1061,7 @@ describe("operation context for configuration helper drain", () => { const counts: number[] = []; const journal: OpsJobJournal = { records: () => [], replace: async () => {} }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: allow, journal, now: () => new Date(baseTime), diff --git a/apps/ops-agent/src/server.test.ts b/apps/ops-agent/src/server.test.ts index f4cc75f62..09ab7b02e 100644 --- a/apps/ops-agent/src/server.test.ts +++ b/apps/ops-agent/src/server.test.ts @@ -77,7 +77,7 @@ function auth(token: string) { describe("ops-agent request boundary", () => { it("reports health without privacy backup collector evidence", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, }); expect((await app.inject({ method: "GET", url: "/health" })).json()).toEqual({ ok: true }); await app.close(); @@ -86,7 +86,7 @@ describe("ops-agent request boundary", () => { it("authenticates and resolves role before operation validation or dispatch", async () => { const dispatch = vi.fn(); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch, @@ -113,7 +113,7 @@ describe("ops-agent request boundary", () => { it("rejects unknown fields, stale/future/duplicate requests, and excessive bodies", async () => { const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), bodyLimit: 512, @@ -186,7 +186,7 @@ describe("ops-agent request boundary", () => { const events: OpsAuditEvent[] = []; const secret = `${apiToken} docker argv http://ops-agent:4300`; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), audit: (event) => events.push(event), @@ -221,11 +221,11 @@ describe("ops-agent request boundary", () => { it("requires distinct role credentials and fails closed on resource authorization", async () => { expect(() => buildActualOpsAgentServer({ - tokens: { api: apiToken, "data-manager": apiToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": apiToken }, }), ).toThrow("distinct"); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: {}, now: () => new Date(nowIso), dispatch: vi.fn(), @@ -297,7 +297,7 @@ describe("ops-agent lifecycle deadlines", () => { }, }); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourceClaimer: host.claimer, controlInstallation: installation, now: () => new Date(nowIso), @@ -359,7 +359,7 @@ describe("ops-agent lifecycle deadlines", () => { }), ); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch, @@ -423,7 +423,7 @@ describe("retained execution deadlines", () => { let calls = 0; const records: import("./journal").PersistedOpsJob[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, journal: { records: () => [], @@ -509,7 +509,7 @@ describe("retained execution deadlines", () => { let signal: AbortSignal | undefined; let finish!: (x: unknown) => void; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, dispatch: async (_o, _r, c) => { signal = c.signal; @@ -550,7 +550,7 @@ describe("retained execution deadlines", () => { it("rejects unsigned wire parent and delegation headers before effects", async () => { const dispatch = vi.fn(); const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch, @@ -606,7 +606,7 @@ describe("execution identity at the agent boundary", () => { let finish!: (x: unknown) => void; let execution: import("@openmapx/core/ops").ExecutionRef | undefined; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, dispatch: async (_o, _r, c) => { execution = c.execution; @@ -660,7 +660,7 @@ describe("execution identity at the agent boundary", () => { let fail!: (x: unknown) => void; const snapshots: import("./journal").PersistedOpsJob[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), journal: { @@ -719,7 +719,7 @@ it("observer abort leaves the admitted backend running until its real completion let backendSignal: AbortSignal | undefined; let finish!: (x: unknown) => void; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch: async (_o, _r, c) => { @@ -839,7 +839,7 @@ it.each([0, 1])( }); const retained: import("./journal").PersistedOpsJob[][] = []; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, runtime, journal: { @@ -925,7 +925,7 @@ describe("ops protocol version", () => { maxDurationSeconds: 300, }; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch: async () => @@ -1015,7 +1015,7 @@ describe("ops protocol version", () => { it("refuses cancellation without the execution fingerprint", async () => { let finish!: (x: unknown) => void; const app = await buildOpsAgentServer({ - tokens: { api: apiToken, "data-manager": dataManagerToken, "host-cli": "h".repeat(43) }, + tokens: { api: apiToken, "data-manager": dataManagerToken }, resourcePolicy: allowAll, now: () => new Date(nowIso), dispatch: async () => diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index bba12d9fc..fea5badc1 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -1,6 +1,8 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "node:crypto"; import { + abandonUnstartedLease, acquireArtifactTargetLease, + admitRootLease, assertArtifactTargetLeaseExecution, assertArtifactTargetLeaseForward, assertControlStorageAvailable, @@ -21,6 +23,7 @@ import { inspectFeedProxyValidationFailure, isFixedSingleDataCommand, type LeaseStore, + localLeaseBackend, markFeedProxyReloadAttempted, planFixedSingleDataCommand, controlHeaders as readControlHeaders, @@ -156,6 +159,7 @@ import { type DawarichSubjectExportRouteOptions, registerDawarichSubjectExportRoute, } from "./dawarich-subject-export"; +import { isReportedProcessFailure } from "./docker-runtime"; import { registerOpsIngressRateLimit } from "./ingress-rate-limit"; import type { OpsJobJournal, PersistedOpsJob } from "./journal"; import { LeaseRefused, runLeaseOperation } from "./lease-operations"; @@ -334,11 +338,9 @@ function resolveRole( const candidate = digest(match?.[1] ?? ""); const apiMatches = timingSafeEqual(candidate, digest(tokens.api)); const dataManagerMatches = timingSafeEqual(candidate, digest(tokens["data-manager"])); - const hostCliMatches = timingSafeEqual(candidate, digest(tokens["host-cli"])); if (!match) return null; if (apiMatches) return "api"; if (dataManagerMatches) return "data-manager"; - if (hostCliMatches) return "host-cli"; return null; } @@ -363,6 +365,8 @@ class RequestGuardError extends Error { constructor( readonly errorClass: OpsErrorClass, readonly statusCode: number, + /** For a busy refusal by a target lease: the holder, as the reply names it. */ + readonly busyHolder?: string, ) { super(errorClass); } @@ -608,6 +612,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif }; const controlLeases = new Map(); + /** Jobs whose Docker command ended with a reported failure, so their outcome is known. */ + const reportedFailures = new Set(); const controlAuthorities = new Map(); const maintenanceControls = new Map< string, @@ -988,6 +994,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif committed && job.operation.kind === "artifactTransfer.import" && (updates.result as { outcome?: string })?.outcome === "not_committed"; + // A command that failed by its own report releases its targets with a + // failed result; a crash, timeout or lost daemon stays unknown. + const reportedFailure = state === "failed" && reportedFailures.delete(job.operationId); if ( !originalGenerationNonCommit && !unstartedConfiguration && @@ -999,12 +1008,12 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ? "contained" : committed ? "committed" - : validationFailure + : validationFailure || reportedFailure ? "contained" : "unknown", publication: artifactNonCommit ? "not_started" - : committed + : committed || reportedFailure ? "committed" : validationFailure ? "not_started" @@ -1013,7 +1022,9 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ? { kind: "ops-result", result: updates.result } : validationFailure ? { kind: "feed-proxy-validation-failure", commitmentDigest: validationFailure } - : { kind: "unresolved" }, + : reportedFailure + ? { kind: "reported-failure" } + : { kind: "unresolved" }, }); if (lease && job.operation.kind === "configurationMaintenance.prepare" && !committed) { try { @@ -1044,7 +1055,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif lease, verifyControlEffectSettlement(options.controlInstallation, authority.execution), ); - if (lease.mode === "owner" && committed) { + if (lease.mode === "owner" && (committed || reportedFailure)) { await releaseArtifactTargetLease(lease); if (job.operation.kind === "artifactTransfer.import") { try { @@ -1656,6 +1667,13 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif const classification = timedOut ? { errorClass: "timeout" as const } : classify(error); + if ( + !timedOut && + LEASED_KINDS.has(job.operation.kind) && + (isReportedProcessFailure(error) || + isReportedProcessFailure((error as { cause?: unknown } | undefined)?.cause)) + ) + reportedFailures.add(job.operationId); let failureResult: OpsFailureResult | undefined; if (error instanceof OpsExecutionFailure) { const safe = opsFailureResultSchema.parse(error.failureResult); @@ -2237,7 +2255,8 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if ( operation.kind === "lease.acquire" || operation.kind === "lease.read" || - operation.kind === "lease.update" + operation.kind === "lease.update" || + operation.kind === "lease.settleChild" ) { if (!options.controlInstallation || !options.leaseStore) throw new RequestGuardError("recovery_required", 409); @@ -2257,8 +2276,6 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif .code(200) .send(opsSuccess(parsed.requestId, { execution: "sync", kind, value })); } - // Only lease operations are open to the host CLI's token. - if (role === "host-cli") throw new RequestGuardError("authorization", 403); const policy = OPS_KIND_POLICIES[operation.kind]; const operationFingerprint = opsOperationFingerprint(operation); if ( @@ -2757,54 +2774,24 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw new RequestGuardError("authorization", 403); } } - // A held target refuses the operation as busy before it is recorded, so - // nothing is left to recover. The holder is named in the audit line. - if ( + // The target lease is the one admission: a held target refuses the + // operation as busy before anything is recorded. Its targets, and a + // maintenance plan, are read first; a failure there is raised once the + // job is recorded, so the attempt stays visible. + const rootAdmission = !controlAdmission && !configurationMaintenanceControl && - LEASED_KINDS.has(operation.kind) && - options.controlInstallation && - options.leaseStore - ) { - const wanted = operation.kind.startsWith("valhalla.") - ? ["valhalla"] - : "serviceId" in operation && - ![ - "service.start", - "service.recreate", - "service.recreateIsolated", - "service.update", - "artifactTransfer.export", - ].includes(operation.kind) - ? [operation.serviceId] - : controlRegisteredServiceIds(options.controlInstallation); - const holder = options.leaseStore - .list() - .find((lease) => lease.targets.some((target) => wanted.includes(target))); - if (holder) { - options.audit?.({ - role, - kind, - resourceId, - result: "denied", - durationMs: Date.now() - startedAt, - errorClass: "busy", - heldBy: `${holder.owner.executor} ${holder.owner.operationKey} (${holder.ownerKind}, ${holder.state})`, - }); - throw new RequestGuardError("busy", 409); - } - } - await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); - jobsById.set(job.operationId, job); - jobsByKey.set(ownerKey, job); - if ( - !controlAdmission && - !configurationMaintenanceControl && - opsKindRequiresControlAuthority(operation.kind) - ) { + opsKindRequiresControlAuthority(operation.kind); + let admissionFailure: { error: unknown } | undefined; + let maintenanceAdmission: + | Awaited> + | undefined; + let targets: string[] = []; + let admittedLease: string | undefined; + if (rootAdmission) { try { if (!options.controlInstallation) throw Error("Control trust unavailable"); - const maintenanceAdmission = + maintenanceAdmission = job.operation.kind === "configurationMaintenance.prepare" ? await prepareConfigurationMaintenanceAdmission(runtime, { installation: options.controlInstallation, @@ -2818,16 +2805,14 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif signal: claimController.signal, }) : undefined; - const maintenancePlan = - maintenanceAdmission && - configurationMaintenanceAdmissionState(maintenanceAdmission).plan; if (literalProductionRuntime) replaceControlRegisteredServiceIds( options.controlInstallation, await productionRegisteredServiceIds(runtime), ); - const targets = - maintenancePlan?.serviceIds ?? + targets = + (maintenanceAdmission && + configurationMaintenanceAdmissionState(maintenanceAdmission).plan.serviceIds) ?? (operation.kind === "artifactTransfer.export" ? [ ...new Set([ @@ -2844,6 +2829,55 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ].includes(operation.kind) ? [operation.serviceId] : controlRegisteredServiceIds(options.controlInstallation)); + } catch (error) { + admissionFailure = { error }; + } + if (!admissionFailure && LEASED_KINDS.has(operation.kind) && options.leaseStore) + try { + admittedLease = ( + await admitRootLease( + localLeaseBackend(options.leaseStore), + { + owner: executionFor(job), + ownerKind: operation.kind, + targets: operation.kind.startsWith("valhalla.") ? ["valhalla"] : targets, + }, + claimController.signal, + ) + ).leaseId; + } catch (error) { + if (!(error instanceof TargetBusyError)) admissionFailure = { error }; + else { + options.audit?.({ + role, + kind, + resourceId, + result: "denied", + durationMs: Date.now() - startedAt, + errorClass: "busy", + heldBy: error.message, + }); + throw new RequestGuardError("busy", 409, error.message); + } + } + } + try { + await journal.replace([...jobsById.values()].map(persisted).concat(persisted(job))); + } catch (error) { + // Nothing was recorded, so nothing ran under the lease. + if (admittedLease && options.leaseStore) + await abandonUnstartedLease(options.leaseStore, admittedLease, executionFor(job)); + throw error; + } + jobsById.set(job.operationId, job); + jobsByKey.set(ownerKey, job); + if (rootAdmission) { + try { + if (admissionFailure) throw admissionFailure.error; + if (!options.controlInstallation) throw Error("Control trust unavailable"); + const maintenancePlan = + maintenanceAdmission && + configurationMaintenanceAdmissionState(maintenanceAdmission).plan; const fixedDataPlan = isFixedSingleDataCommand(job.operation.kind) ? planFixedSingleDataCommand( executionFor(job), @@ -2987,7 +3021,7 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif throw error instanceof RequestGuardError ? error : error instanceof TargetBusyError - ? new RequestGuardError("busy", 409) + ? new RequestGuardError("busy", 409, error.message) : new RequestGuardError("runtime", 500); }); if (admission.type === "retained") { diff --git a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts index d273e1c7f..25ea6b4eb 100644 --- a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts +++ b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts @@ -105,7 +105,7 @@ const app = buildOpsAgentServer({ runtime, journal, controlInstallation: installation, - tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, + tokens: { api: "a".repeat(43), "data-manager": token }, resourceClaimer: createProductionRegistryResourceClaimer({ services: ["motis", "motis-feed-proxy"].map((serviceId) => ({ serviceId, diff --git a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts index 778bb4a04..4d578b52c 100644 --- a/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts +++ b/apps/ops-agent/src/test-fixtures/terminal-retention-process.ts @@ -202,7 +202,7 @@ journal.replace = async (records) => { if (input.boundary === "terminal") stop(terminal); }; const app = buildOpsAgentServer({ - tokens: { api: token, "data-manager": "b".repeat(32), "host-cli": "h".repeat(43) }, + tokens: { api: token, "data-manager": "b".repeat(32) }, controlInstallation: installation, journal, resourcePolicy: { allowBackup: () => true, allowGlobal: () => true, allowService: () => true }, diff --git a/apps/ops-agent/src/traffic-target-ownership.test.ts b/apps/ops-agent/src/traffic-target-ownership.test.ts index a1c7d40dc..6ddefe487 100644 --- a/apps/ops-agent/src/traffic-target-ownership.test.ts +++ b/apps/ops-agent/src/traffic-target-ownership.test.ts @@ -66,7 +66,7 @@ it.each(["succeeded", "canceled"] as const)( runtime, journal, controlInstallation: installation, - tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, + tokens: { api: "a".repeat(43), "data-manager": token }, resourceClaimer: createProductionRegistryResourceClaimer({ services: [{ serviceId: "valhalla", enabled: true, isBuiltIn: true }], integrationIds: [], diff --git a/apps/ops-agent/src/trusted-config-ownership.test.ts b/apps/ops-agent/src/trusted-config-ownership.test.ts index 595925379..b72de0e95 100644 --- a/apps/ops-agent/src/trusted-config-ownership.test.ts +++ b/apps/ops-agent/src/trusted-config-ownership.test.ts @@ -534,7 +534,6 @@ describe("authenticated conditional configuration admission", () => { tokens: { api: token, "data-manager": Buffer.alloc(32, 8).toString("base64url"), - "host-cli": "h".repeat(43), }, resourceClaimer: claimer, controlInstallation: installation, diff --git a/packages/cli/__tests__/cli-control.test.ts b/packages/cli/__tests__/cli-control.test.ts index 8ad19ebde..eae214523 100644 --- a/packages/cli/__tests__/cli-control.test.ts +++ b/packages/cli/__tests__/cli-control.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import { tmpdir } from "node:os"; import { join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; +import { settleDataConfigurationFence } from "@openmapx/core/data-manager-sdk"; import type { ExecutionRef } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { Command } from "commander"; @@ -14,10 +15,15 @@ vi.mock("../src/lib/docker", async (original) => ({ ...(await original()), dockerCompose: vi.fn(), })); +vi.mock("@openmapx/core/data-manager-sdk", async (original) => ({ + ...(await original()), + settleDataConfigurationFence: vi.fn(), +})); let root: string; let infraDir: string; let printed: string[]; +let warned: string[]; beforeEach(() => { root = mkdtempSync(join(tmpdir(), "openmapx-control-")); infraDir = join(root, "infra/docker"); @@ -28,7 +34,11 @@ beforeEach(() => { vi.spyOn(console, "log").mockImplementation((line: string) => { printed.push(String(line)); }); - vi.spyOn(console, "error").mockImplementation(() => {}); + warned = []; + vi.spyOn(console, "error").mockImplementation((line: string) => { + warned.push(String(line)); + }); + vi.mocked(settleDataConfigurationFence).mockReset(); vi.spyOn(process, "exit").mockImplementation((code) => { throw Error(`exit:${code}`); }); @@ -125,4 +135,56 @@ describe("openmapx control", () => { expect(services.readLockHolder(path)).toBeUndefined(); expect(auditLog()).toMatchObject([{ action: "clear-lock", lock: "leases.lock", holder }]); }); + + it("warns that clearing a maintenance leaves its fence, which resolve then settles", async () => { + const agent: ExecutionRef = { + ...owner(), + executor: "ops-agent", + principal: "api", + executionId: `job1_${randomUUID().replaceAll("-", "")}`, + }; + const evidence = { + maintenanceId: randomUUID(), + candidateId: randomUUID(), + planDigest: "c".repeat(64), + }; + await store().update((leases) => { + const lease = control.leaseRules.acquire( + leases, + { owner: agent, ownerKind: "configurationMaintenance.prepare", targets: ["alpha"] }, + Date.now(), + ); + control.leaseRules.delegate( + leases, + { + leaseId: lease.leaseId, + child: { + key: randomUUID(), + parentKey: null, + parent: agent, + execution: { ...owner(), executor: "data-manager", principal: "data-manager" }, + kind: "configuration-maintenance.prepare", + phase: "configuration-fence", + recipient: "data-manager", + targets: ["alpha"], + permittedFurtherEdges: [], + depth: 1, + evidence, + }, + }, + Date.now(), + ); + }); + await run("clear", "--op", agent.operationKey); + expect(store().list()).toEqual([]); + expect(warned.join("\n")).toContain( + `fence of this maintenance stays prepared and keeps refusing data jobs until \`openmapx control resolve --op ${agent.operationKey}`, + ); + await run("resolve", "--op", agent.operationKey, "--as", "failed"); + expect(vi.mocked(settleDataConfigurationFence)).toHaveBeenCalledWith( + { ...evidence, outcome: "contained" }, + expect.not.objectContaining({ controlAssertion: expect.anything() }), + ); + expect(auditLog().map((entry) => entry.action)).toEqual(["clear", "resolve"]); + }); }); diff --git a/packages/cli/__tests__/cli-host-effect.test.ts b/packages/cli/__tests__/cli-host-effect.test.ts index 347661ba9..80f1c8ca6 100644 --- a/packages/cli/__tests__/cli-host-effect.test.ts +++ b/packages/cli/__tests__/cli-host-effect.test.ts @@ -4,12 +4,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { dockerCompose } from "../src/lib/docker"; +import { dockerDaemonReachable } from "../src/lib/docker"; import { leasedHostEffect, withHostTargets } from "../src/lib/host-effect"; vi.mock("../src/lib/docker", async (original) => ({ ...(await original()), - dockerCompose: vi.fn(), + dockerDaemonReachable: vi.fn(), })); let root: string; @@ -25,14 +25,7 @@ beforeEach(() => { vi.spyOn(console, "error").mockImplementation((line: string) => { errors.push(String(line)); }); - // The agent's container is not running, so the CLI writes the store itself. - vi.mocked(dockerCompose) - .mockReset() - .mockResolvedValue({ - exitCode: 0, - stdout: "", - stderr: "", - } as Awaited>); + vi.mocked(dockerDaemonReachable).mockReset().mockResolvedValue(true); }); afterEach(() => { vi.restoreAllMocks(); @@ -97,4 +90,41 @@ describe("host effects under a lease", () => { ).rejects.toThrow("restore failed"); expect(held()).toEqual([]); }); + + it("releases the targets of a command that exited with a failure", async () => { + expect(await leasedHostEffect("compose.down", ["alpha"], async () => 1)).toBe(1); + expect(held()).toEqual([]); + }); + + it("keeps the targets of a command killed by a signal, for control resolve", async () => { + expect(await leasedHostEffect("services.stop", ["alpha"], async () => 137)).toBe(137); + const [lease] = control + .openLeaseStore({ directory: control.leaseStoreDirectory(infraDir), purpose: "test" }) + .list(); + expect(lease).toMatchObject({ + targets: ["alpha"], + state: "recovery_required", + settlement: { outcome: "unknown", publication: "unknown" }, + }); + expect(errors.join("\n")).toContain(`control resolve --op ${lease?.owner.operationKey}`); + }); + + it("keeps the targets of a failure while the Docker daemon cannot be reached", async () => { + vi.mocked(dockerDaemonReachable).mockResolvedValue(false); + await expect( + withHostTargets({ kind: "backup.create", targets: ["postgis"] }, async () => { + throw Error("lost the daemon"); + }), + ).rejects.toThrow("lost the daemon"); + expect(held()).toEqual(["postgis"]); + }); + + it("leases a service the agent has never seen, such as a community service just installed", async () => { + expect( + await withHostTargets({ kind: "services.restart", targets: ["community-late"] }, async () => + held(), + ), + ).toEqual(["community-late"]); + expect(held()).toEqual([]); + }); }); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index 7cf787f1e..1110982c0 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -2,15 +2,13 @@ import { createHash } from "node:crypto"; import { closeSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs"; import { basename, join } from "node:path"; import { - type LeaseOperation, type LeaseRecord, leaseStoreDirectory, openLeaseStore, } from "@openmapx/core/build-artifacts"; -import { canonicalJson, OpsClient, opsOperationSchema } from "@openmapx/core/ops"; +import { canonicalJson } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import type { Command } from "commander"; -import { runsInAgent } from "../lib/agent-integrations"; import { completeCliCommand } from "../lib/control-context"; import { settleMaintenanceFence } from "../lib/control-resolve"; import { dockerCompose } from "../lib/docker"; @@ -85,6 +83,38 @@ function findLease(operationKey: string): LeaseRecord { return lease; } +/** + * A maintenance root an operator cleared, from the audit file: its fence can + * still be settled with `control resolve`. + */ +function clearedMaintenance(operationKey: string): LeaseRecord | undefined { + let lines: string[]; + try { + lines = readFileSync( + join(leaseStoreDirectory(repoPaths().infraDir), "control-audit.jsonl"), + "utf8", + ).split("\n"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + for (const line of lines.reverse()) { + if (!line) continue; + const entry = JSON.parse(line) as { action?: string; lease?: LeaseRecord | null }; + if ( + entry.action === "clear" && + entry.lease?.owner.operationKey === operationKey && + entry.lease.ownerKind === "configurationMaintenance.prepare" + ) + return entry.lease; + } + return undefined; +} + +function hasFence(lease: LeaseRecord): boolean { + return lease.children.some((child) => child.phase === "configuration-fence"); +} + function readJournal(): { operationKey: string; kind: string; @@ -170,6 +200,10 @@ export function registerControlCommands(program: Command): void { return index < 0 ? undefined : leases.splice(index, 1)[0]; }); audit({ action: "clear", operationKey: options.op, lease: removed ?? null }); + if (lease.ownerKind === "configurationMaintenance.prepare" && hasFence(lease)) + log.warn( + `The data-manager's configuration fence of this maintenance stays prepared and keeps refusing data jobs until \`openmapx control resolve --op ${lease.owner.operationKey} --as committed|failed\` settles it.`, + ); completeCliCommand(0); }); @@ -181,7 +215,12 @@ export function registerControlCommands(program: Command): void { .action(async (options: { op: string; as: string }) => { if (options.as !== "committed" && options.as !== "failed") throw new Error("--as must be committed or failed"); - const lease = findLease(options.op); + const cleared = store() + .list() + .some((value) => value.owner.operationKey === options.op) + ? undefined + : clearedMaintenance(options.op); + const lease = cleared ?? findLease(options.op); if (lease.owner.operationKey !== options.op) throw new Error("Resolve the operation that owns the lease"); for (const line of describe(lease)) log.info(line); @@ -189,7 +228,7 @@ export function registerControlCommands(program: Command): void { // A maintenance root also holds the data-manager fence; the same idempotent // settle the agent uses releases it. if (lease.ownerKind === "configurationMaintenance.prepare") - await settleMaintenanceFence(repoPaths().infraDir, lease, options.as); + await settleMaintenanceFence(lease, options.as); const outcome: "committed" | "contained" = options.as === "committed" ? "committed" : "contained"; const evidence = (identity: LeaseRecord["owner"]) => ({ @@ -221,20 +260,4 @@ export function registerControlCommands(program: Command): void { audit({ action: "resolve", operationKey: options.op, as: options.as, lease }); completeCliCommand(0); }); - - // The host CLI's transport to the agent: run as the fixed CLI in the agent's - // container, with the host-CLI token the agent created. - control.command("lease ", { hidden: true }).action(async (request: string) => { - if (!runsInAgent()) throw new Error("Runs only in the operations agent's container"); - const operation = opsOperationSchema.parse(JSON.parse(request)) as LeaseOperation; - if (!operation.kind.startsWith("lease.")) throw new Error("Lease operations only"); - const client = new OpsClient({ - baseUrl: `http://127.0.0.1:${process.env.PORT?.trim() || "4300"}`, - tokenFile: services.hostCliTokenPath(repoPaths().infraDir), - }); - const submitted = await client.execute(operation); - if (submitted.execution !== "sync") throw new Error("Lease operations answer at once"); - process.stdout.write(`${JSON.stringify(submitted.value)}\n`); - completeCliCommand(0); - }); } diff --git a/packages/cli/src/lib/artifact-ownership.ts b/packages/cli/src/lib/artifact-ownership.ts index a39329491..64d4c6a9f 100644 --- a/packages/cli/src/lib/artifact-ownership.ts +++ b/packages/cli/src/lib/artifact-ownership.ts @@ -13,7 +13,6 @@ import { import { services } from "@openmapx/core/server"; import { cliControlContext, pruneHostControlRecords } from "./control-context"; import { cliAcquisitionOptions, cliProcessOptions } from "./execution-deadline"; -import { hostCliLeases } from "./host-leases"; import { repoPaths } from "./paths"; type ArtifactOperation = Extract< @@ -36,7 +35,6 @@ export async function openCliArtifactOwner(operation: ArtifactOperation, rootDir infraDir: paths.infraDir, role: "host-cli", serviceIds: targets, - leases: hostCliLeases(paths.infraDir), }); let lease: control.TargetLeaseAuthority | undefined; try { diff --git a/packages/cli/src/lib/control-resolve.ts b/packages/cli/src/lib/control-resolve.ts index e680be014..fe2c4c791 100644 --- a/packages/cli/src/lib/control-resolve.ts +++ b/packages/cli/src/lib/control-resolve.ts @@ -1,5 +1,3 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; import type { LeaseRecord } from "@openmapx/core/build-artifacts"; import { createDataManagerTransport, @@ -7,73 +5,40 @@ import { settleDataConfigurationFence, } from "@openmapx/core/data-manager-sdk"; import { z } from "zod"; +import { cliDataManagerToken } from "./data-manager-token"; import { dataManagerUrl } from "./data-manager-url"; -/** One of the agent's own records, as it keeps them in its private state. */ -function agentRecord(infraDir: string, namespace: string, name: string): unknown { - const raw = JSON.parse( - readFileSync(join(infraDir, "control-private", "ops-agent", namespace, name), "utf8"), - ) as { payload?: unknown }; - return raw.payload ?? raw; -} +/** What the agent records on a maintenance's fence child for this settle. */ +const fenceEvidenceSchema = z.object({ + maintenanceId: z.uuid(), + candidateId: z.uuid(), + planDigest: z.string(), +}); /** - * Settles the data-manager fence of a maintenance the operator resolves, with - * the agent's own data-manager token and the original preparation delegation, - * as the agent itself would. A fence that was never prepared needs nothing. + * Settles the data-manager fence of a maintenance the operator resolves. The + * settle inputs come from the fence child in the agent's lease store; the + * request goes the way the CLI's other data-manager calls go, with the CLI's + * own token, which data-manager accepts as the operator's resolution. A fence + * that was never prepared needs nothing. */ export async function settleMaintenanceFence( - infraDir: string, lease: LeaseRecord, as: "committed" | "failed", ): Promise { const fence = lease.children.find((child) => child.phase === "configuration-fence"); if (!fence) return; - const admission = z - .object({ operation: z.object({ maintenanceId: z.uuid() }).passthrough() }) - .passthrough() - .parse(agentRecord(infraDir, "executions", `${lease.owner.operationKey}.json`)); - const owner = z - .object({ - plan: z - .object({ - operation: z.object({ maintenanceId: z.uuid(), candidateId: z.uuid() }).passthrough(), - planDigest: z.string(), - }) - .passthrough(), - }) - .passthrough() - .parse( - agentRecord( - infraDir, - "configuration-maintenance", - `${admission.operation.maintenanceId}.json`, - ), - ); - const attempt = z - .object({ assertion: z.string() }) - .passthrough() - .parse(agentRecord(infraDir, "attempts", `${fence.key}.json`)); - const token = readFileSync( - join(infraDir, "secrets", "data-manager-ops-agent-token"), - "utf8", - ).trim(); + const evidence = fenceEvidenceSchema.parse(fence.evidence); try { await settleDataConfigurationFence( - { - maintenanceId: owner.plan.operation.maintenanceId, - candidateId: owner.plan.operation.candidateId, - planDigest: owner.plan.planDigest, - outcome: as === "committed" ? "applied" : "contained", - }, + { ...evidence, outcome: as === "committed" ? "applied" : "contained" }, { transport: createDataManagerTransport({ baseUrl: dataManagerUrl(), - authToken: token, + authToken: cliDataManagerToken(), requestTimeoutMs: 15000, maxJsonBytes: 65536, }), - controlAssertion: attempt.assertion, }, ); } catch (error) { diff --git a/packages/cli/src/lib/data-execution.ts b/packages/cli/src/lib/data-execution.ts index f8988137f..fb9be4720 100644 --- a/packages/cli/src/lib/data-execution.ts +++ b/packages/cli/src/lib/data-execution.ts @@ -1,5 +1,4 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { setTimeout as delay } from "node:timers/promises"; import { bindControlChildPhase, cliDataReservation, @@ -46,7 +45,6 @@ import { withHostDataControlContext, } from "./control-context"; import { cliAcquisitionOptions } from "./execution-deadline"; -import { hostCliLeases } from "./host-leases"; import { repoPaths } from "./paths"; export async function executeDurableDataStep( input: DataExecutionInput, @@ -168,19 +166,9 @@ export async function executeDurableDataStep( }); await checkpoints; if (checkpointError) throw checkpointError; - // The terminal GET precedes final physical target release by a small window. - // Wait for the actual DM-signed shared containment witness, never infer it. - for (;;) { - options.signal?.throwIfAborted(); - try { - verifyCliDataTerminal(context.authority, terminal); - break; - } catch { - if (Date.now() >= options.observerDeadlineAt) - throw new DataExecutionObservationPending(identity, "observer_deadline"); - await delay(100, undefined, { signal: options.signal }); - } - } + // Data-manager releases its own lease just after its terminal; a next step + // that claims the same targets waits for that in its own acquisition. + verifyCliDataTerminal(context.authority, terminal); await options.checkpointAttempt({ ...attempt, state: "terminal" }); return terminal; } @@ -259,7 +247,6 @@ export async function runCliDataStep( infraDir: repoPaths().infraDir, role: "host-cli", serviceIds: ["data-manager"], - leases: hostCliLeases(repoPaths().infraDir), }); await pruneHostControlRecords(installation); const normalized = normalizeExecutionAdmission({ diff --git a/packages/cli/src/lib/data-update-command.ts b/packages/cli/src/lib/data-update-command.ts index 625a3d2c4..014f2a519 100644 --- a/packages/cli/src/lib/data-update-command.ts +++ b/packages/cli/src/lib/data-update-command.ts @@ -17,7 +17,6 @@ import { runOwnedDataStep } from "./data-execution"; import { DataUpdateFailure, runDataUpdate } from "./data-update"; import { cliAcquisitionOptions } from "./execution-deadline"; import { readGeneratedHardlinkPlan } from "./hardlinks"; -import { hostCliLeases } from "./host-leases"; import { log } from "./output"; import { repoPaths } from "./paths"; import { buildServices, ServiceBuildExecutionError } from "./service-builds"; @@ -45,7 +44,6 @@ export async function runCliDataUpdate(options: { infraDir: paths.infraDir, role: "host-cli", serviceIds: targets, - leases: hostCliLeases(paths.infraDir), }); await pruneHostControlRecords(installation); const operation = { diff --git a/packages/cli/src/lib/docker.ts b/packages/cli/src/lib/docker.ts index 6dd0801e6..d2e3dffbc 100644 --- a/packages/cli/src/lib/docker.ts +++ b/packages/cli/src/lib/docker.ts @@ -1,3 +1,4 @@ +import { constants } from "node:os"; import { services as coreServices } from "@openmapx/core/server"; import { execa } from "execa"; import { cliProcessOptions } from "./execution-deadline"; @@ -46,7 +47,19 @@ export async function dockerComposeStream(args: string[]): Promise { reject: false, }); const result = await sub; - return result.exitCode ?? 0; + // A command killed by a signal reports 128 plus the signal's number, as a shell does. + if (result.signal) return 128 + (constants.signals[result.signal] ?? 0); + return result.exitCode ?? 1; +} + +/** Whether the Docker daemon answers, to tell a failed command from a lost connection. */ +export async function dockerDaemonReachable(): Promise { + const result = await execa("docker", ["info", "--format", "{{.ID}}"], { + ...cliProcessOptions(), + reject: false, + timeout: 15_000, + }); + return result.exitCode === 0; } /** diff --git a/packages/cli/src/lib/host-effect.ts b/packages/cli/src/lib/host-effect.ts index ec1c03314..1b4bc2ab9 100644 --- a/packages/cli/src/lib/host-effect.ts +++ b/packages/cli/src/lib/host-effect.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes, randomUUID } from "node:crypto"; -import { TargetBusyError } from "@openmapx/core/build-artifacts"; +import { agentStoreLeases, TargetBusyError } from "@openmapx/core/build-artifacts"; import { canonicalJson, createControlRootOperationKey, @@ -8,28 +8,50 @@ import { } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { runsInAgent } from "./agent-integrations"; -import { hostCliLeases } from "./host-leases"; +import { dockerDaemonReachable } from "./docker"; import { log } from "./output"; import { repoPaths } from "./paths"; export class HostTargetsBusyError extends Error {} +type Outcome = "committed" | "contained" | "unknown"; + +/** A shell's exit code of 128 or more means the command was killed by a signal. */ +function exitOutcome(code: number): Outcome { + return code === 0 ? "committed" : code >= 128 ? "unknown" : "contained"; +} + +/** A process killed, timed out or cut off from the daemon did not report how far it got. */ +function thrownOutcome(error: unknown): Outcome { + const value = error as { signal?: unknown; timedOut?: unknown; isTerminated?: unknown }; + return value?.signal || value?.timedOut === true || value?.isTerminated === true + ? "unknown" + : "contained"; +} + /** * Runs one host Docker effect that changes containers (stop, restart, compose - * up or down, a backup that stops services) under a lease on its targets, the - * same admission path the agent uses. A held target refuses it before the - * effect, naming the holder. A finished effect releases the lease whether it - * succeeded or failed; a CLI killed during the effect leaves it held for - * `openmapx control`. + * up or down, a backup that stops services) under a lease on its targets in + * the agent's store, which the CLI writes under the store's lock. A held + * target refuses it before the effect, naming the holder. An effect that + * ended with a reported failure releases the lease with a failed result; one + * killed by a signal, timed out, or cut off from the Docker daemon keeps it + * held for `openmapx control resolve`, as does a CLI killed during the effect. */ export async function withHostTargets( - input: { kind: string; targets: readonly string[] | "all"; budgetMs?: number }, + input: { + kind: string; + targets: readonly string[] | "all"; + budgetMs?: number; + /** How the effect's result ended, for effects that report an exit code. */ + outcomeOf?: (result: T) => Outcome; + }, run: () => Promise, ): Promise { // A fixed command the agent started runs under the lease the agent holds. if (runsInAgent() && process.env.OPENMAPX_CONTROL_EXECUTION) return run(); const targets = input.targets === "all" ? await registeredServiceIds() : input.targets; - const leases = hostCliLeases(repoPaths().infraDir); + const leases = agentStoreLeases(repoPaths().infraDir, "host-cli"); const deadlineAt = Date.now() + (input.budgetMs ?? 2 * 3600_000); const body = { kind: input.kind, targets: [...new Set(targets)].sort() }; const operationFingerprint = createHash("sha256").update(canonicalJson(body)).digest("hex"); @@ -49,28 +71,36 @@ export async function withHostTargets( if (error instanceof TargetBusyError) throw new HostTargetsBusyError(error.message); throw error; } - const finish = async (outcome: "committed" | "contained", receipt: unknown) => { + const finish = async (outcome: Outcome) => { + // A command that failed because the daemon went away may have run in part. + if (outcome === "contained" && !(await dockerDaemonReachable())) outcome = "unknown"; + const known = outcome !== "unknown"; await leases.settle({ leaseId: lease.leaseId, key: null, settlement: { identity: owner, outcome, - publication: "committed", - terminalReceiptDigest: createHash("sha256").update(canonicalJson(receipt)).digest("hex"), + publication: known ? "committed" : "unknown", + terminalReceiptDigest: createHash("sha256") + .update(canonicalJson({ kind: input.kind, outcome })) + .digest("hex"), }, }); - await leases.release({ leaseId: lease.leaseId }); + if (known) await leases.release({ leaseId: lease.leaseId }); + else + log.err( + `The outcome of ${input.kind} is unknown; its targets stay held. Check the containers, then run \`openmapx control resolve --op ${owner.operationKey} --as committed|failed\`.`, + ); }; let result: T; try { result = await run(); } catch (error) { - // The effect ran to its end and failed: its outcome is known. - await finish("contained", { kind: input.kind, failed: true }); + await finish(thrownOutcome(error)); throw error; } - await finish("committed", { kind: input.kind, failed: false }); + await finish(input.outcomeOf ? input.outcomeOf(result) : "committed"); return result; } @@ -91,7 +121,7 @@ export async function leasedHostEffect( run: () => Promise, ): Promise { try { - return await withHostTargets({ kind, targets }, run); + return await withHostTargets({ kind, targets, outcomeOf: exitOutcome }, run); } catch (error) { if (!(error instanceof HostTargetsBusyError)) throw error; log.err(error.message); diff --git a/packages/cli/src/lib/host-leases.ts b/packages/cli/src/lib/host-leases.ts deleted file mode 100644 index 3c57feac2..000000000 --- a/packages/cli/src/lib/host-leases.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { - agentStoreLeases, - type LeaseBackend, - type LeaseOperation, - remoteLeaseBackend, -} from "@openmapx/core/build-artifacts"; -import { agentContainerRunning, runsInAgent } from "./agent-integrations"; -import { dockerCompose } from "./docker"; - -/** Runs one lease operation through the fixed CLI in the agent's container. */ -async function throughAgentContainer(operation: LeaseOperation): Promise { - const result = await dockerCompose([ - "exec", - "-T", - "ops-agent", - "node", - "dist/cli.mjs", - "control", - "lease", - JSON.stringify(operation), - ]); - if (result.exitCode !== 0) - throw new Error( - result.stderr.trim() || "The operations agent could not answer the lease operation", - ); - return JSON.parse(result.stdout); -} - -/** - * The host CLI's leases. While the agent runs they go through it, as the - * fixed CLI in its container with the host-CLI token, so the agent stays the - * only writer of its store. While it is stopped the CLI writes the store under - * its recoverable lock. Inside the agent's container the store is used directly. - */ -export function hostCliLeases(infraDir: string): LeaseBackend { - let chosen: Promise | undefined; - const backend = () => { - chosen ??= (async () => - runsInAgent() || !(await agentContainerRunning()) - ? agentStoreLeases(infraDir, "host-cli") - : remoteLeaseBackend(throughAgentContainer))(); - return chosen; - }; - return { - acquire: async (input) => (await backend()).acquire(input), - read: async (leaseId) => (await backend()).read(leaseId), - list: async () => (await backend()).list(), - delegate: async (input) => (await backend()).delegate(input), - adopt: async (input) => (await backend()).adopt(input), - settle: async (input) => (await backend()).settle(input), - cancel: async (input) => (await backend()).cancel(input), - release: async (input) => (await backend()).release(input), - }; -} diff --git a/packages/cli/src/lib/service-lifecycle.ts b/packages/cli/src/lib/service-lifecycle.ts index 54810b408..7f019ceb9 100644 --- a/packages/cli/src/lib/service-lifecycle.ts +++ b/packages/cli/src/lib/service-lifecycle.ts @@ -176,7 +176,6 @@ export async function runCliServiceLifecycle(options: { infraDir: paths.infraDir, role: "host-cli", serviceIds: targets, - leases: (await import("./host-leases")).hostCliLeases(paths.infraDir), }); if (!inherited) await pruneHostControlRecords(installation); const admittedAt = Date.now(), diff --git a/packages/cli/src/lib/transitous-api-keys.ts b/packages/cli/src/lib/transitous-api-keys.ts index 60f37df11..2bb880e96 100644 --- a/packages/cli/src/lib/transitous-api-keys.ts +++ b/packages/cli/src/lib/transitous-api-keys.ts @@ -32,7 +32,6 @@ import { import { execa } from "execa"; import { cliControlContext, pruneHostControlRecords } from "./control-context"; import { cliProcessOptions } from "./execution-deadline"; -import { hostCliLeases } from "./host-leases"; import { repoPaths } from "./paths"; import { ensurePlatformPrivateDirectory, @@ -332,7 +331,6 @@ export async function generateTransitousApiKeys( infraDir: paths.infraDir, role: "host-cli", serviceIds: ["motis"], - leases: hostCliLeases(paths.infraDir), }); let lease: control.ArtifactTargetLease | undefined; let authority: control.VerifiedExecutionAuthority | undefined; diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index 46fd8bab0..f1ef39524 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1446,6 +1446,17 @@ const effectObservationSchema = z.discriminatedUnion("kind", [ }), z.strictObject({ kind: z.literal("not-started") }), z.strictObject({ kind: z.literal("unresolved") }), + z.strictObject({ kind: z.literal("reported-failure") }), +]); +/** Leased Docker effects whose command reports its failure: it ran to its end, so its outcome is known. */ +const REPORTED_FAILURE_KINDS = new Set([ + "service.stop", + "service.restart", + "service.remove", + "stack.stop", + "serviceRepository.remove", + "backup.create", + "backup.restore", ]); /** No fence exists for this maintenance owner: its fence child was never * issued, or the data-manager settled it without effect (a refusal, or the @@ -1894,6 +1905,11 @@ export function retainControlEffectReceipt( outcome = "committed"; publication = "committed"; } + } else if (receipt.kind === "reported-failure") { + if (!REPORTED_FAILURE_KINDS.has(s.kind) || s.installation.installation.role !== "ops-agent") + throw Error("Only a leased agent Docker effect reports its own failure"); + outcome = "contained"; + publication = "committed"; } else if (receipt.kind === "feed-proxy-validation-failure") { if ( s.kind !== "feedProxy.validateAndReload" || diff --git a/packages/core/src/build-artifacts/data-configuration-fence.ts b/packages/core/src/build-artifacts/data-configuration-fence.ts index 41a477c7f..f50a470e0 100644 --- a/packages/core/src/build-artifacts/data-configuration-fence.ts +++ b/packages/core/src/build-artifacts/data-configuration-fence.ts @@ -52,7 +52,6 @@ const fenceRowSchema = z.object({ operation_key: z.string(), candidate_id: z.uuid(), plan_digest: z.string(), - owner_lease_id: z.uuid(), input: dataConfigurationFencePrepareSchema, state: z.enum(["prepared", "settled"]), outcome: dataConfigurationFenceOutcomeSchema.nullable(), @@ -275,10 +274,9 @@ export function createDataConfigurationFenceStore( if (!fence) return; const receipt = receiptOf(fence); // During maintenance only the maintenance owner's own hardlink leaf runs: - // the agent issued it under the lease the fence was prepared for. + // the agent issued it under the maintenance owner the fence was prepared for. if ( principal !== "ops-agent" || - delegation.leaseId !== fence.owner_lease_id || canonicalJson(delegation.owner) !== canonicalJson(receipt.ownerPrepare) || request.input.revisionId !== receipt.revisionId || request.input.planDigest !== receipt.hardlinkPlanDigest || @@ -442,14 +440,13 @@ export function createDataConfigurationFenceStore( ); if (active.length || transfers.length) throw new DataFenceBusy(); const inserted = await tx.unsafe( - "INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,owner_lease_id,input,state) VALUES($1::uuid,$2::uuid,$3,$4::uuid,$5,$6::uuid,$7::text::jsonb,'prepared') RETURNING *", + "INSERT INTO data_manager.configuration_fence(maintenance_id,fence_id,operation_key,candidate_id,plan_digest,input,state) VALUES($1::uuid,$2::uuid,$3,$4::uuid,$5,$6::text::jsonb,'prepared') RETURNING *", [ input.maintenanceId, execution.executionId, execution.operationKey, input.candidateId, input.planDigest, - verified.delegation.leaseId, canonicalJson(input), ], ); @@ -460,42 +457,42 @@ export function createDataConfigurationFenceStore( /** The fence row of `maintenanceId`, for the agent only. */ async inspect( maintenanceId: string, - expected: { candidateId: string; ownerLeaseId: string }, + expected: { candidateId: string }, transportPrincipal: ControlTransportPrincipal, ) { if (transportPrincipal !== "ops-agent") throw forbidden("Agent transport required"); const saved = await row(sql, "maintenance_id", maintenanceId); if (!saved) return null; - if ( - saved.candidate_id !== expected.candidateId || - saved.owner_lease_id !== expected.ownerLeaseId - ) + if (saved.candidate_id !== expected.candidateId) throw Error("Configuration fence owner changed"); return receiptOf(saved); }, /** Settles the prepared row once. The header is the agent's original * prepare delegation; a repeated settle with the same outcome returns the - * settled receipt, another outcome is a conflict. */ + * settled receipt, another outcome is a conflict. The host CLI's token + * settles without a delegation: that is the operator resolving the + * maintenance with `openmapx control resolve`. */ async settle( value: DataConfigurationFenceSettle, rawHeaders: readonly string[], transportPrincipal: ControlTransportPrincipal, ) { const input = dataConfigurationFenceSettleSchema.parse(value); - let execution: z.infer["input"]["ownerPrepare"]; - try { - if (transportPrincipal !== "ops-agent") throw Error("Agent transport required"); - const delegation = inspectRemoteDelegation(installation, rawHeaders, transportPrincipal); - if ( - delegation.parent.executor !== "ops-agent" || - delegation.kind !== "configuration-maintenance.prepare" || - delegation.phase !== "configuration-fence" - ) - throw Error("Configuration fence delegation identity conflict"); - execution = delegation.execution; - } catch { - throw forbidden("Agent configuration fence delegation rejected"); - } + let execution: z.infer["input"]["ownerPrepare"] | undefined; + if (transportPrincipal !== "host-cli") + try { + if (transportPrincipal !== "ops-agent") throw Error("Agent transport required"); + const delegation = inspectRemoteDelegation(installation, rawHeaders, transportPrincipal); + if ( + delegation.parent.executor !== "ops-agent" || + delegation.kind !== "configuration-maintenance.prepare" || + delegation.phase !== "configuration-fence" + ) + throw Error("Configuration fence delegation identity conflict"); + execution = delegation.execution; + } catch { + throw forbidden("Agent configuration fence delegation rejected"); + } // The conditional UPDATE and its read-back are the guard: a racing data // admission sees the row either prepared (busy) or settled (open). return sql.transaction(async (tx) => { @@ -505,8 +502,9 @@ export function createDataConfigurationFenceStore( statusCode: 404, }); if ( - saved.fence_id !== execution.executionId || - saved.operation_key !== execution.operationKey + execution && + (saved.fence_id !== execution.executionId || + saved.operation_key !== execution.operationKey) ) throw forbidden("Configuration fence delegation identity conflict"); if (saved.candidate_id !== input.candidateId || saved.plan_digest !== input.planDigest) diff --git a/packages/core/src/build-artifacts/lease-remote.ts b/packages/core/src/build-artifacts/lease-remote.ts index 49b2d8afb..5bcbb4fb6 100644 --- a/packages/core/src/build-artifacts/lease-remote.ts +++ b/packages/core/src/build-artifacts/lease-remote.ts @@ -5,7 +5,7 @@ import type { LeaseBackend } from "./target-lease"; export type LeaseOperation = Extract< OpsOperation, - { kind: "lease.acquire" | "lease.read" | "lease.update" } + { kind: "lease.acquire" | "lease.read" | "lease.update" | "lease.settleChild" } >; /** Sends one lease operation to the agent and returns its typed result. */ export type LeaseSender = (operation: LeaseOperation) => Promise; @@ -19,9 +19,9 @@ export function opsClientLeaseSender(client: OpsClient | (() => OpsClient)): Lea } /** - * Leases through the agent, for data-manager and the host CLI. Every call is a - * typed agent operation under the caller's own token; the agent applies the - * same rules as to its own leases and refuses another principal's. + * Leases through the agent, for data-manager. Every call is a typed agent + * operation under data-manager's own token; the agent applies the same rules + * as to its own leases and refuses another principal's. */ export function remoteLeaseBackend(send: LeaseSender): LeaseBackend { async function run( @@ -70,5 +70,7 @@ export function remoteLeaseBackend(send: LeaseSender): LeaseBackend { release: async (input) => { await update(input.leaseId, { action: "release" }); }, + settleChild: async (settlement) => + (await run({ kind: "lease.settleChild", settlement })).lease ?? undefined, }; } diff --git a/packages/core/src/build-artifacts/lease-store.ts b/packages/core/src/build-artifacts/lease-store.ts index 58cccf33b..3b972d5fc 100644 --- a/packages/core/src/build-artifacts/lease-store.ts +++ b/packages/core/src/build-artifacts/lease-store.ts @@ -3,6 +3,7 @@ import { lstatSync, mkdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; +import { opsBusyHolderMessage } from "../ops/contract"; import type { ExecutionRef } from "../ops/execution-ref"; import { type LeaseChild, @@ -46,7 +47,13 @@ export class TargetBusyError extends Error { readonly holder: Pick, ) { super( - `Target lease busy: ${target} is held by ${holder.owner.executor} operation ${holder.owner.operationKey} (${holder.ownerKind}, ${holder.state}). See \`openmapx control status\`.`, + opsBusyHolderMessage({ + target, + executor: holder.owner.executor, + operationKey: holder.owner.operationKey, + ownerKind: holder.ownerKind, + state: holder.state, + }), ); } } @@ -164,6 +171,23 @@ export function admitLease( return record; } +/** + * Drops a lease whose owner never recorded its attempt, so nothing ran under + * it. A lease that already has children is left for its owner. + */ +export async function abandonUnstartedLease( + store: LeaseStore, + leaseId: string, + owner: ExecutionRef, +): Promise { + await store.update((leases) => { + const index = leases.findIndex((lease) => lease.leaseId === leaseId); + const lease = leases[index]; + if (lease && sameExecution(lease.owner, owner) && lease.children.length === 0) + leases.splice(index, 1); + }); +} + /** Whether the owner, or the child `key`, and every ancestor may still begin effects. */ export function leaseNodeForward(lease: LeaseRecord, key: string | null, now: number): boolean { if (lease.state !== "held" || now >= lease.owner.deadlineAt) return false; diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index b4e6721f0..c6dc95490 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -135,6 +135,8 @@ export interface LeaseBackend { }): Promise; cancel(input: { leaseId: string; key: string | null }): Promise; release(input: { leaseId: string }): Promise; + /** Settles the data-manager child named by the settlement's identity, if it is still held. */ + settleChild(settlement: LeaseSettlement): Promise; } const MAX_DESCENDANTS = 64; @@ -243,6 +245,28 @@ export const leaseRules = { node.state = knownSettlement(input.settlement) ? "settled" : "recovery_required"; return lease; }, + /** + * Data-manager reports the outcome of a child it ran under another role's + * lease. The child is found by its identity; an outcome already recorded with + * the same result is kept, and a released lease leaves nothing to settle. + */ + settleChild(leases: LeaseRecord[], settlement: LeaseSettlement) { + for (const lease of leases) { + const child = lease.children.find( + (value) => + value.recipient === "data-manager" && sameExecution(value.execution, settlement.identity), + ); + if (!child) continue; + if ( + child.state === "settled" && + child.settlement?.outcome === settlement.outcome && + child.settlement.publication === settlement.publication + ) + return lease; + return leaseRules.settle(leases, { leaseId: lease.leaseId, key: child.key, settlement }); + } + return undefined; + }, cancel(leases: LeaseRecord[], input: Parameters[0]) { const lease = leases.find((value) => value.leaseId === input.leaseId); if (!lease) return undefined; @@ -275,6 +299,8 @@ export function localLeaseBackend(store: LeaseStore, now: () => number = Date.no settle: (input) => store.update((leases) => leaseRules.settle(leases, input)), cancel: (input) => store.update((leases) => leaseRules.cancel(leases, input)), release: (input) => store.update((leases) => leaseRules.release(leases, input)), + settleChild: (settlement) => + store.update((leases) => leaseRules.settleChild(leases, settlement)), }; } @@ -423,6 +449,28 @@ export function verifyFiniteControlEdge( throw Error("Unauthorized finite control edge"); } +/** + * Acquires the root lease of `owner`, or returns the one it already holds. A + * holder that is finishing gets five seconds to release, within the deadline; + * after that a held target refuses with the holder named. + */ +export async function admitRootLease( + backend: Pick, + input: Parameters[0], + signal?: AbortSignal, +): Promise { + const until = Math.min(Date.now() + 5000, input.owner.deadlineAt); + for (;;) { + try { + return await backend.acquire(input); + } catch (error) { + if (!(error instanceof TargetBusyError) || Date.now() >= until) throw error; + await new Promise((resolve) => setTimeout(resolve, 50)); + signal?.throwIfAborted(); + } + } +} + export async function acquireArtifactTargetLease(options: { serviceIds: readonly string[]; owner: VerifiedExecutionAuthority; @@ -447,23 +495,11 @@ export async function acquireArtifactTargetLease(options: { } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } - // A holder that is finishing gets five seconds to release, within the deadline. - const until = Math.min(Date.now() + 5000, authority.execution.deadlineAt); - let record: LeaseRecord; - for (;;) { - try { - record = await authority.installation.leases.acquire({ - owner: authority.execution, - ownerKind: authority.kind, - targets: serviceIds, - }); - break; - } catch (error) { - if (!(error instanceof TargetBusyError) || Date.now() >= until) throw error; - await new Promise((resolve) => setTimeout(resolve, 50)); - options.signal?.throwIfAborted(); - } - } + const record = await admitRootLease( + authority.installation.leases, + { owner: authority.execution, ownerKind: authority.kind, targets: serviceIds }, + options.signal, + ); // The lease is recorded before this owner's private reference to it, so an // owner that dies in between leaves a held lease that is still visible. writePrivateRecord( @@ -1211,6 +1247,21 @@ export async function settleDataManagerLeaseChild( terminalReceiptDigest: observed.terminalDigest, }); } +/** + * Data-manager reports the known outcome of a hardlink child it ran under the + * agent's or the host CLI's lease, from its own effect settlement. It does so + * after its terminal, at startup repair and in recovery, so a known outcome is + * recorded even when the issuer's own observation was interrupted. + */ +export async function settleOwnLeaseChild( + installation: import("./control-authority").ControlInstallation, + execution: ExecutionRef, +): Promise { + const settlement = childSettlementState( + verifyControlEffectSettlement(installation, execution), + ).evidence; + await installationState(installation).leases.settleChild(settlement); +} /** * Settles the child `execution` this lease node issued, from the answer its * recipient gave to this role over an authenticated route. A recorded diff --git a/packages/core/src/ops/client.ts b/packages/core/src/ops/client.ts index a1a63e820..2e00ebdf9 100644 --- a/packages/core/src/ops/client.ts +++ b/packages/core/src/ops/client.ts @@ -4,6 +4,7 @@ import { open } from "node:fs/promises"; import type { ArtifactStageClaim, TransferStage } from "./artifact-transfer"; import { type ArtifactStageStreamInput, withArtifactStream } from "./artifact-transfer-client"; import { + isOpsBusyHolderMessage, OPS_KIND_POLICIES, OPS_MAX_HTTP_RESPONSE_BYTES, OPS_MAX_REQUEST_TTL_MS, @@ -194,14 +195,28 @@ function parseResponse(raw: string, requestId: string, status: number): OpsRespo Array.isArray(error) || !exactKeys(error as Record, ["class", "message"]) || !ERROR_CLASSES.has((error as { class?: OpsErrorClass }).class as OpsErrorClass) || - (error as { message?: unknown }).message !== - OPS_PUBLIC_ERROR_MESSAGES[(error as { class: OpsErrorClass }).class] + ((error as { message?: unknown }).message !== + OPS_PUBLIC_ERROR_MESSAGES[(error as { class: OpsErrorClass }).class] && + !( + (error as { class: OpsErrorClass }).class === "busy" && + isOpsBusyHolderMessage((error as { message?: unknown }).message) + )) ) { throw new OpsClientError("runtime", status); } return envelope as unknown as OpsResponseEnvelope; } +/** The client error for a refused request; a busy refusal names the holding operation. */ +function opsEnvelopeError( + error: { class: OpsErrorClass; message: string }, + status: number, +): OpsClientError { + const failure = new OpsClientError(error.class, status); + if (isOpsBusyHolderMessage(error.message)) failure.message = error.message; + return failure; +} + export interface OpsClientOptions { baseUrl: string; tokenFile: string; @@ -353,7 +368,7 @@ export class OpsClient { requestId, response.status, ); - if (!envelope.ok) throw new OpsClientError(envelope.error.class, response.status); + if (!envelope.ok) throw opsEnvelopeError(envelope.error, response.status); if (!response.ok) throw new OpsClientError("runtime", response.status); return envelope.result; } catch (error) { @@ -567,7 +582,7 @@ export class OpsClient { requestId, response.status, ); - if (!envelope.ok) throw new OpsClientError(envelope.error.class, response.status); + if (!envelope.ok) throw opsEnvelopeError(envelope.error, response.status); if (!response.ok) throw new OpsClientError("runtime", response.status); const result = envelope.result; if (!result || typeof result !== "object" || Array.isArray(result)) { diff --git a/packages/core/src/ops/contract.test.ts b/packages/core/src/ops/contract.test.ts index dd94a8bbe..5f167db5b 100644 --- a/packages/core/src/ops/contract.test.ts +++ b/packages/core/src/ops/contract.test.ts @@ -514,6 +514,23 @@ const samples: Record<(typeof OPS_OPERATION_KINDS)[number], OpsOperation> = { leaseId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", action: { action: "release" }, }, + "lease.settleChild": { + kind: "lease.settleChild", + settlement: { + identity: { + executor: "data-manager", + principal: "data-manager", + executionId: "91a78b3e-208b-4ba4-9ddf-870fbdf1da22", + operationKey: `opk1_${"a".repeat(32)}`, + operationFingerprint: "a".repeat(64), + fingerprint: "b".repeat(64), + deadlineAt: 2_000_000_000_000, + }, + outcome: "committed", + publication: "committed", + terminalReceiptDigest: "c".repeat(64), + }, + }, }; it("compares projections independent of object ordering while preserving array and value identity", () => { diff --git a/packages/core/src/ops/contract.ts b/packages/core/src/ops/contract.ts index cfde602bb..92c84ef10 100644 --- a/packages/core/src/ops/contract.ts +++ b/packages/core/src/ops/contract.ts @@ -201,11 +201,12 @@ export const OPS_OPERATION_KINDS = [ "lease.acquire", "lease.read", "lease.update", + "lease.settleChild", ] as const; export type OpsOperationKind = (typeof OPS_OPERATION_KINDS)[number]; -export type OpsRole = "api" | "data-manager" | "host-cli"; +export type OpsRole = "api" | "data-manager"; export type OpsExecutionMode = "sync" | "async"; export interface OpsKindPolicy { @@ -347,10 +348,11 @@ export const OPS_KIND_POLICIES = { "gbfsCatalogLock.inspect": sync("data-manager", 30_000), // Read-only, for the admin panel: the pins, never a proposal or a change. "transitousPin.inspect": sync("api", 30_000), - // Data-manager and the host CLI claim targets through the agent; see authorizeOpsOperation. + // Data-manager claims targets, and settles the children it ran, through the agent. "lease.acquire": sync("data-manager", 15_000, 256 * 1024), "lease.read": sync("data-manager", 15_000, 256 * 1024), "lease.update": sync("data-manager", 15_000, 256 * 1024), + "lease.settleChild": sync("data-manager", 15_000, 256 * 1024), } as const satisfies Record; /** Read streams/diagnostics do not acquire control-root authority or a retention dependency. */ @@ -748,8 +750,6 @@ export type OpsRequest = z.infer; export function authorizeOpsOperation(role: OpsRole, kind: OpsOperationKind): boolean { // Decode retained historical records, but never admit a new remote file writer. if (kind === "data.generateApiKeys") return false; - // The host CLI reaches the agent only to claim targets for its own commands. - if (role === "host-cli") return kind.startsWith("lease."); return ( OPS_KIND_POLICIES[kind].role === role || (role === "data-manager" && kind === "catalogPins.inspect") @@ -1654,16 +1654,43 @@ export function opsSuccess( ): OpsSuccessEnvelope { return { version: 1, requestId, ok: true, result }; } +/** + * The message of a busy refusal caused by a target lease: it names the holding + * operation, which is not secret to any caller, and where an operator looks. + */ +export function opsBusyHolderMessage(holder: { + target: string; + executor: string; + operationKey: string; + ownerKind: string; + state: string; +}): string { + return `Target ${holder.target} is busy: held by ${holder.executor} operation ${holder.operationKey} (${holder.ownerKind}, ${holder.state}). See \`openmapx control status\`.`; +} +const BUSY_HOLDER_MESSAGE = + /^Target [a-z0-9][a-z0-9-]{0,63} is busy: held by [a-z-]{1,32} operation opk1_[A-Za-z0-9_-]{16,64} \([A-Za-z0-9_.:-]{1,128}, [a-z_]{1,32}\)\. See `openmapx control status`\.$/; +/** Whether `message` is a holder message a busy error may carry instead of the fixed text. */ +export function isOpsBusyHolderMessage(message: unknown): message is string { + return typeof message === "string" && BUSY_HOLDER_MESSAGE.test(message); +} + export function redactedOpsError( requestId: string, errorClassValue: OpsErrorClass, - _cause?: unknown, + cause?: unknown, ): OpsErrorEnvelope { + const holder = (cause as { busyHolder?: unknown } | undefined)?.busyHolder; return { version: 1, requestId, ok: false, - error: { class: errorClassValue, message: OPS_PUBLIC_ERROR_MESSAGES[errorClassValue] }, + error: { + class: errorClassValue, + message: + errorClassValue === "busy" && isOpsBusyHolderMessage(holder) + ? holder + : OPS_PUBLIC_ERROR_MESSAGES[errorClassValue], + }, }; } diff --git a/packages/core/src/ops/lease-records.ts b/packages/core/src/ops/lease-records.ts index 73e9dcbdf..f9baf7b6a 100644 --- a/packages/core/src/ops/lease-records.ts +++ b/packages/core/src/ops/lease-records.ts @@ -67,7 +67,7 @@ const holderSchema = leaseRecordSchema.pick({ state: true, }); -/** What data-manager and the host CLI ask the agent to do with their leases. */ +/** What data-manager asks the agent to do with its leases, and with the children it runs. */ export const leaseOperationSchemas = [ z.strictObject({ kind: z.literal("lease.acquire"), @@ -90,6 +90,8 @@ export const leaseOperationSchemas = [ z.strictObject({ action: z.literal("release") }), ]), }), + /** Data-manager reports the known outcome of a child it ran under another's lease. */ + z.strictObject({ kind: z.literal("lease.settleChild"), settlement: leaseSettlementSchema }), ] as const; export const leaseResultSchemas = { "lease.acquire": z.discriminatedUnion("outcome", [ @@ -98,4 +100,5 @@ export const leaseResultSchemas = { ]), "lease.read": z.strictObject({ lease: leaseRecordSchema.nullable() }), "lease.update": z.strictObject({ lease: leaseRecordSchema.nullable() }), + "lease.settleChild": z.strictObject({ lease: leaseRecordSchema.nullable() }), } as const; diff --git a/packages/core/src/services/agent-state.ts b/packages/core/src/services/agent-state.ts index 0df6b3f27..dbc365746 100644 --- a/packages/core/src/services/agent-state.ts +++ b/packages/core/src/services/agent-state.ts @@ -1,28 +1,7 @@ -import { randomBytes } from "node:crypto"; -import { chmodSync, lstatSync, mkdirSync, readFileSync, renameSync } from "node:fs"; +import { chmodSync, lstatSync, mkdirSync, renameSync } from "node:fs"; import { dirname, join } from "node:path"; import { atomicWrite } from "../release-files"; -/** - * The token the host CLI presents to the agent. The agent creates it at - * startup in its own state; only the deployment user, through `docker compose - * exec ops-agent`, can read it. - */ -export function hostCliTokenPath(infraDir: string): string { - return join(infraDir, "state", "ops-agent", "host-cli.token"); -} -export function ensureHostCliToken(infraDir: string): string { - const path = hostCliTokenPath(infraDir); - try { - const existing = readFileSync(path, "utf8").trim(); - if (/^[A-Za-z0-9_-]{43}$/.test(existing)) return existing; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - const token = randomBytes(32).toString("base64url"); - writeAgentStateFile(infraDir, "host-cli.token", `${token}\n`); - return token; -} /** Replaces one of the agent's state files atomically, creating its directory (0700) first. */ export function writeAgentStateFile(infraDir: string, name: string, contents: string): void { const directory = join(infraDir, "state", "ops-agent"); diff --git a/packages/core/src/services/data-manager/configuration-maintenance.ts b/packages/core/src/services/data-manager/configuration-maintenance.ts index 33310d963..fa69b4888 100644 --- a/packages/core/src/services/data-manager/configuration-maintenance.ts +++ b/packages/core/src/services/data-manager/configuration-maintenance.ts @@ -45,13 +45,12 @@ export async function inspectDataConfigurationFence( options: { transport: DataManagerTransport; candidateId: string; - ownerLeaseId: string; signal?: AbortSignal; }, ): Promise { const result = await sdk.inspectDataConfigurationFence( maintenanceId, - { candidateId: options.candidateId, ownerLeaseId: options.ownerLeaseId }, + { candidateId: options.candidateId }, { transport: options.transport, signal: options.signal }, ); return dataConfigurationFenceReceiptSchema.parse(unwrapDataManagerResponse(result)); @@ -70,15 +69,18 @@ export async function recoverUnadmittedDataConfigurationFence( ); return configurationPrepareNonAdmissionSchema.parse(unwrapDataManagerResponse(result)); } +/** Without a control assertion the request is the operator's resolution, sent with the host CLI's token. */ export async function settleDataConfigurationFence( input: DataConfigurationFenceSettle, - options: ControlOptions, + options: Omit & { controlAssertion?: string }, ): Promise { const body = boundedBody(dataConfigurationFenceSettleSchema.parse(input)); const result = await sdk.settleDataConfigurationFence(body.maintenanceId, body, { transport: options.transport, signal: options.signal, - headers: { "x-openmapx-control-delegation": boundedHeader(options.controlAssertion) }, + ...(options.controlAssertion === undefined + ? {} + : { headers: { "x-openmapx-control-delegation": boundedHeader(options.controlAssertion) } }), }); return dataConfigurationFenceReceiptSchema.parse(unwrapDataManagerResponse(result)); } diff --git a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts index 53a4f7e9d..ac53a9d63 100644 --- a/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts +++ b/packages/core/src/services/data-manager/contracts/configuration-maintenance.ts @@ -40,7 +40,6 @@ export type ConfigurationPrepareNonAdmission = z.infer< >; export const dataConfigurationFenceInspectQuerySchema = z.strictObject({ candidateId: z.uuid(), - ownerLeaseId: z.uuid(), }); export const dataConfigurationFenceReceiptSchema = z.strictObject({ ...binding, diff --git a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts index 5ad8bf9e7..5e1262e82 100644 --- a/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts +++ b/packages/core/src/services/data-manager/generated/models/inspectDataConfigurationFenceParams.ts @@ -4,6 +4,4 @@ export type InspectDataConfigurationFenceParams = { /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ candidateId: string; - /** @pattern ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ */ - ownerLeaseId: string; }; diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index ef8bc1472..a15058db2 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -2,8 +2,6 @@ export { openAdminDataWorkflow } from "./admin-data-workflow"; export { type AgentStatePaths, agentStatePaths, - ensureHostCliToken, - hostCliTokenPath, prepareAgentStateDirectories, } from "./agent-state"; export { prepareAgentStores } from "./agent-stores"; diff --git a/packages/db-schema/src/data-manager.ts b/packages/db-schema/src/data-manager.ts index ccc5adb71..42f744098 100644 --- a/packages/db-schema/src/data-manager.ts +++ b/packages/db-schema/src/data-manager.ts @@ -86,8 +86,6 @@ export const configurationFence = dataManager.table( operationKey: text("operation_key").notNull().unique(), candidateId: uuid("candidate_id").notNull(), planDigest: text("plan_digest").notNull(), - /** The agent's maintenance lease the fence was prepared for. */ - ownerLeaseId: uuid("owner_lease_id").notNull(), /** The prepare request body, compared on a repeated prepare. */ input: jsonb("input").notNull(), /** "prepared" | "settled" */ diff --git a/services/data-manager/__tests__/transitous/catalog-review.test.ts b/services/data-manager/__tests__/transitous/catalog-review.test.ts index 360b83b08..7fe84adc9 100644 --- a/services/data-manager/__tests__/transitous/catalog-review.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-review.test.ts @@ -184,7 +184,7 @@ it.skipIf(!enabled)( tokenFile = join(root, "dm-token"); writeFileSync(tokenFile, token, { mode: 0o600 }); agent = buildOpsAgentServer({ - tokens: { api: "a".repeat(43), "data-manager": token, "host-cli": "h".repeat(43) }, + tokens: { api: "a".repeat(43), "data-manager": token }, runtime: createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: root }), resourcePolicy: createRegistryResourcePolicy({ serviceIds: ["motis"], diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 9f5eac1cc..915562247 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -5327,16 +5327,6 @@ "name": "candidateId", "required": true }, - { - "schema": { - "type": "string", - "format": "uuid", - "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$" - }, - "in": "query", - "name": "ownerLeaseId", - "required": true - }, { "schema": { "type": "string", diff --git a/services/data-manager/src/jobs/execution/api.test.ts b/services/data-manager/src/jobs/execution/api.test.ts index c87f9e5c8..9ceaeff8c 100644 --- a/services/data-manager/src/jobs/execution/api.test.ts +++ b/services/data-manager/src/jobs/execution/api.test.ts @@ -33,7 +33,7 @@ it("keeps observation and mutation routes closed without a durable receiver", as }, { method: "GET" as const, - url: `/executions/configuration-maintenance/${id}?candidateId=${id}&ownerLeaseId=${id}`, + url: `/executions/configuration-maintenance/${id}?candidateId=${id}`, }, ]) { const result = await app.inject({ ...request, headers }); diff --git a/services/data-manager/src/jobs/execution/api.ts b/services/data-manager/src/jobs/execution/api.ts index 34f466902..88c910a5d 100644 --- a/services/data-manager/src/jobs/execution/api.ts +++ b/services/data-manager/src/jobs/execution/api.ts @@ -63,7 +63,7 @@ export interface DataExecutionReceiver { ): Promise; inspectFence( maintenanceId: string, - expected: { candidateId: string; ownerLeaseId: string }, + expected: { candidateId: string }, transportPrincipal: DataManagerPrincipal, ): Promise; settleFence( diff --git a/services/data-manager/src/jobs/execution/fixed-cli.test.ts b/services/data-manager/src/jobs/execution/fixed-cli.test.ts index 3b0a47597..1a495e2ad 100644 --- a/services/data-manager/src/jobs/execution/fixed-cli.test.ts +++ b/services/data-manager/src/jobs/execution/fixed-cli.test.ts @@ -184,7 +184,7 @@ it.skipIf( runtime: agentRuntime, journal: agentJournal, controlInstallation: agentInstallation, - tokens: { api: token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: token, "data-manager": "d".repeat(43) }, resourcePolicy: { allowGlobal: () => true, allowService: () => true }, }); await agent.listen({ host: "127.0.0.1", port: 0 }); diff --git a/services/data-manager/src/jobs/execution/full-update.test.ts b/services/data-manager/src/jobs/execution/full-update.test.ts index 1d7438d84..fef5fbef4 100644 --- a/services/data-manager/src/jobs/execution/full-update.test.ts +++ b/services/data-manager/src/jobs/execution/full-update.test.ts @@ -249,7 +249,7 @@ it.skipIf( runtime: host.runtime, journal: agentJournal, controlInstallation: agentInstallation, - tokens: { api: token, "data-manager": "d".repeat(43), "host-cli": "h".repeat(43) }, + tokens: { api: token, "data-manager": "d".repeat(43) }, resourceClaimer: createProductionRegistryResourceClaimer({ services: host.loaded.map((s) => ({ serviceId: s.manifest.id, diff --git a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts index d8d2e4900..87efe6b44 100644 --- a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts +++ b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts @@ -1030,7 +1030,6 @@ it.skipIf(!enabled)( tokens: { api: Buffer.alloc(32, 6).toString("base64url"), "data-manager": token, - "host-cli": Buffer.alloc(32, 7).toString("base64url"), }, controlInstallation: agent, runtime: createTransitousLockRuntime(createUnavailableRuntime(), { rootDir: repository }), diff --git a/services/data-manager/src/jobs/execution/ops-children.test.ts b/services/data-manager/src/jobs/execution/ops-children.test.ts index f34b1eec9..1df61f87c 100644 --- a/services/data-manager/src/jobs/execution/ops-children.test.ts +++ b/services/data-manager/src/jobs/execution/ops-children.test.ts @@ -1126,7 +1126,7 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( control.recordOpsControlTerminalObservation(dm, bound.execution, status); expect(control.readOpsControlTerminalObservation(dm, bound.execution).state).toBe("failed"); await expect( - await control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution), + control.inspectArtifactTargetLeaseChildSettlement(lease, bound.execution), ).resolves.toMatchObject({ outcome: "contained", publication: "not_started" }); const repaired = JSON.parse(readFileSync(terminalPath, "utf8")).payload; expect(repaired.terminalDigest).toBe(originalTerminal.terminalDigest); diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index 040959012..05bf46185 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -14,6 +14,8 @@ import { restoreArtifactTargetLeaseForSettlement, retainAdmittedControlExecution, settleArtifactTargetLeaseChild, + settleOwnLeaseChild, + TargetBusyError, type VerifiedExecutionAuthority, type VerifiedPreparedGenerationAdmission, verifyControlEffectSettlement, @@ -26,7 +28,11 @@ import { dataExecutionResultSchema, isDataExecutionTerminal, } from "@openmapx/core/data-manager-contracts"; -import type { ExecutionRef, VerifiedParentExecution } from "@openmapx/core/ops"; +import { + type ExecutionRef, + OpsClientError, + type VerifiedParentExecution, +} from "@openmapx/core/ops"; import { jobChildLogger } from "../../logger.js"; import { cancelAndSettleRetainedTransitousScripts } from "../transitous/script-runner.js"; import { dataExecutionDiagnostic } from "./diagnostics.js"; @@ -90,7 +96,8 @@ export interface DataExecutionRuntime { close(): Promise; } /** Settles and releases the execution's own lease after its terminal record, - * then records the terminal for later steps that build on it. */ + * or, for a hardlink leaf, settles its child in the issuer's lease; then + * records the terminal for later steps that build on it. */ async function releaseOwnLease( journal: { installation: ControlInstallation; @@ -105,7 +112,7 @@ async function releaseOwnLease( verifyControlEffectSettlement(journal.installation, execution), ); await releaseArtifactTargetLease(lease); - } + } else await settleOwnLeaseChild(journal.installation, execution); await journal.recordContainedTerminal(execution); } export function dataExecutionTargetIds(input: DataExecutionInput): string[] { @@ -267,11 +274,20 @@ export function createDataExecutionRuntime(deps: { if (hardlinkPlan === undefined) throw Error("Durable generation plan required"); await owner.retainHardlinksPlan(authority, hardlinkPlan); } - await owner.begin(authority); // A hardlink leaf runs under the lease of the agent or host CLI that issued it. - const lease = leaf - ? undefined - : await acquireArtifactTargetLease({ serviceIds: targets, owner: authority }); + // Other executions claim their targets first: a held target, or an agent + // that cannot answer, ends the execution as failed before any effect, with + // its publication unchanged. + let lease: ArtifactTargetLease | undefined; + let refusal: unknown; + if (!leaf) + try { + lease = await acquireArtifactTargetLease({ serviceIds: targets, owner: authority }); + } catch (error) { + if (!(error instanceof TargetBusyError || error instanceof OpsClientError)) throw error; + refusal = error; + } + await owner.begin(authority); const controller = new AbortController(); let entry: LiveEntry; const context: DataExecutionContext = { @@ -360,6 +376,7 @@ export function createDataExecutionRuntime(deps: { authority, Promise.resolve() .then(() => { + if (refusal) throw refusal; context.assertForward(); return withDataExecutionScope(context, () => deps.dispatch(record.input, context)); }) @@ -437,7 +454,8 @@ export function createDataExecutionRuntime(deps: { }); if (isDataExecutionTerminal(entry.record.status)) { await owner.assertOwnedProxyRecoverySettled(record.execution); - await releaseOwnLease(owner, record.execution, lease); + if (!refusal) await releaseOwnLease(owner, record.execution, lease); + else await owner.recordContainedTerminal(record.execution); } } catch (error) { // An I/O/CAS/containment gap is not a terminal receipt or permission to From 2ee89a8063659d6528b8b8f2807cd262c5ec2453 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 16:22:56 +0100 Subject: [PATCH 05/42] refactor(control): check the edge grammar once in the store and trim the lease interfaces The store's delegate rule now checks which child an issuer may record, from the lease's owner or the parent child; rows no longer carry the permitted further edges or their depth, and the agent no longer repeats the check for data-manager. The configuration-fence settle edge had no issuer and is gone. Roles that read the store use a local lease backend that also reads, lists and adopts without waiting; data-manager's client over the agent's routes has only the calls it makes. The transit candidate on a child's row is just the candidate: the row already names the child, its operation and its targets. --- .../src/data-transit-resource-source.test.ts | 19 +-- apps/ops-agent/src/lease-operations.test.ts | 2 - apps/ops-agent/src/lease-operations.ts | 11 +- apps/ops-agent/src/server-foundation.test.ts | 3 +- packages/cli/__tests__/cli-control.test.ts | 2 - .../core/src/build-artifacts/child-phase.ts | 3 +- .../src/build-artifacts/control-authority.ts | 2 - .../data-execution-candidate.ts | 44 ++----- .../core/src/build-artifacts/lease-remote.ts | 6 - .../src/build-artifacts/lease-store.test.ts | 16 +-- .../core/src/build-artifacts/target-lease.ts | 122 ++++++++++-------- packages/core/src/ops/lease-records.ts | 2 - 12 files changed, 105 insertions(+), 127 deletions(-) diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts index a865a97bb..d0c911b92 100644 --- a/apps/ops-agent/src/data-transit-resource-source.test.ts +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -372,7 +372,6 @@ it.each([ "foreign-phase", "foreign-body", "foreign-target", - "foreign-origin", "inode", "bytes", "directory", @@ -387,13 +386,17 @@ it.each([ let key = f.slot.execution.operationKey, operation = f.operation; if (mode === "missing") await f.published(() => undefined); - if (mode === "forged" || mode === "foreign-target" || mode === "foreign-origin") - await f.published((evidence) => { - if (mode === "forged") return { ...evidence, version: 3 }; - if (mode === "foreign-target") return { ...evidence, serviceIds: ["motis"] }; - const origin = evidence.origin as { notAfter: number }; - return { ...evidence, origin: { ...origin, notAfter: origin.notAfter + 1 } }; - }); + if (mode === "forged") await f.published((evidence) => ({ ...evidence, version: 3 })); + // The child's row names its targets; one naming another service is refused. + if (mode === "foreign-target") + await control + .openLeaseStore({ directory: control.leaseStoreDirectory(f.root), purpose: "test" }) + .update((leases) => { + for (const held of leases) + for (const child of held.children) + if (child.execution.operationKey === f.slot.execution.operationKey) + child.targets = ["motis"]; + }); // A candidate belongs to the child whose grant carries it; another key has none. if (mode === "foreign-root" || mode === "foreign-phase") key = control.controlChildOperationKey( diff --git a/apps/ops-agent/src/lease-operations.test.ts b/apps/ops-agent/src/lease-operations.test.ts index 177c34322..7375516a8 100644 --- a/apps/ops-agent/src/lease-operations.test.ts +++ b/apps/ops-agent/src/lease-operations.test.ts @@ -130,8 +130,6 @@ describe("lease operations for data-manager", () => { phase: "hardlinks", recipient: "data-manager", targets: ["osrm"], - permittedFurtherEdges: [], - depth: 1, }, }, Date.now(), diff --git a/apps/ops-agent/src/lease-operations.ts b/apps/ops-agent/src/lease-operations.ts index 07e6b4bfb..56b192ad1 100644 --- a/apps/ops-agent/src/lease-operations.ts +++ b/apps/ops-agent/src/lease-operations.ts @@ -5,7 +5,6 @@ import { type LeaseStore, leaseRules, TargetBusyError, - verifyFiniteControlEdge, } from "@openmapx/core/build-artifacts"; import { canonicalJson, @@ -77,19 +76,13 @@ export async function runLeaseOperation( const action = operation.action; switch (action.action) { case "delegate": { + // The store's rules check the edge itself; data-manager issues only + // the agent's own operations, directly under its own execution. const child = action.child; - const edges = verifyFiniteControlEdge( - role, - current.ownerKind, - child.phase as never, - child.recipient, - ); if ( child.parentKey !== null || - child.depth !== 1 || child.recipient !== "ops-agent" || canonicalJson(child.parent) !== canonicalJson(current.owner) || - canonicalJson(child.permittedFurtherEdges) !== canonicalJson(edges) || !(childPhaseKinds as Record)[child.phase]?.includes( child.kind, ) || diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 092eb7805..d705080d5 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -970,7 +970,8 @@ describe("one admission path for target effects", () => { await runContainedStreamingProcess("sh", ["-c", script], { signal: context.signal, timeout: 10_000, - maxBuffer: 4096, + onLine: () => {}, + redact: (message) => message, }); return { changed: true }; }, diff --git a/packages/cli/__tests__/cli-control.test.ts b/packages/cli/__tests__/cli-control.test.ts index eae214523..3e4ac5255 100644 --- a/packages/cli/__tests__/cli-control.test.ts +++ b/packages/cli/__tests__/cli-control.test.ts @@ -167,8 +167,6 @@ describe("openmapx control", () => { phase: "configuration-fence", recipient: "data-manager", targets: ["alpha"], - permittedFurtherEdges: [], - depth: 1, evidence, }, }, diff --git a/packages/core/src/build-artifacts/child-phase.ts b/packages/core/src/build-artifacts/child-phase.ts index df26bbc5d..237ce662f 100644 --- a/packages/core/src/build-artifacts/child-phase.ts +++ b/packages/core/src/build-artifacts/child-phase.ts @@ -49,6 +49,7 @@ import { import { delegateArtifactTargetLease, inspectArtifactDelegation, + localLeases, type TargetLeaseAuthority, verifyFiniteControlEdge, } from "./target-lease"; @@ -385,7 +386,7 @@ export function lookupControlChildPhaseAttempt( throw Error("Original phase outbox digest changed"); // A role that reaches the lease store only through the agent relies on its // own outbox; the agent checked the row when it recorded it. - if (channel === "delegation" && s.installation.leases.readSync) { + if (channel === "delegation" && localLeases(s.installation.leases)) { const observed = inspectArtifactDelegation({ assertion: saved.assertion, installation: s.installation.installation, diff --git a/packages/core/src/build-artifacts/control-authority.ts b/packages/core/src/build-artifacts/control-authority.ts index f1ef39524..6d3b3d5e7 100644 --- a/packages/core/src/build-artifacts/control-authority.ts +++ b/packages/core/src/build-artifacts/control-authority.ts @@ -1101,8 +1101,6 @@ const delegatedRowSchema = z.strictObject({ phase: z.string().max(64), recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), targets: targetSchema, - permittedFurtherEdges: z.array(z.string().max(64)).max(1), - depth: z.number().int().min(1).max(2), }); /** * A child row of the agent's lease store, sent to the child's recipient. A diff --git a/packages/core/src/build-artifacts/data-execution-candidate.ts b/packages/core/src/build-artifacts/data-execution-candidate.ts index 50d8d1235..13b2132a5 100644 --- a/packages/core/src/build-artifacts/data-execution-candidate.ts +++ b/packages/core/src/build-artifacts/data-execution-candidate.ts @@ -14,8 +14,7 @@ import { z } from "zod"; import { readCatalogProposalFile, readCatalogValidationFile } from "../catalog-pins/record-files"; import { canonicalJson } from "../ops/canonical-json"; import { pairedPinApprovalInputSchema } from "../ops/catalog-pin-approval"; -import { type OpsJobStatus, type OpsOperation, parseOpsJobStatus } from "../ops/contract"; -import { controlTerminalReceiptDigest } from "../ops/control-terminal-receipt"; +import type { OpsOperation } from "../ops/contract"; import { retainedParentExecutionSchema } from "../ops/execution-identity"; import { type ExecutionRef, executionRefSchema } from "../ops/execution-ref"; import { opsOperationFingerprint } from "../ops/fingerprint"; @@ -41,10 +40,11 @@ import { controlOriginSchema, sameControlOrigin, signControl, - targetSchema, verifyControl, } from "./control-records"; import { AnchoredDirectory, identity, sameInode } from "./filesystem"; +// biome-ignore lint/suspicious/noImportCycles: The candidate reads the agent's store through the same lease module as its issuer. +import { localLeases } from "./target-lease"; const schema = z.strictObject({ version: z.literal(1), @@ -448,15 +448,8 @@ export function assertOwnedDataTransitCandidate( return controlDigest(saved); } -const publishedSchema = z.strictObject({ - version: z.literal(2), - installationId: z.uuid(), - origin: controlOriginSchema, - candidate: schema, - childOperationKey: z.string(), - operationFingerprint: z.string().regex(/^[a-f0-9]{64}$/), - serviceIds: targetSchema, -}); +/** The candidate on the child's row; the row itself names the child, its body and targets. */ +const publishedSchema = z.strictObject({ candidate: schema }); /** * The candidate a transit phase acts on, as the agent receives it with the * child's grant. Null when the phase has no retained candidate. @@ -484,18 +477,7 @@ export function publishedDataTransitCandidate( s.plan.find((r) => r.phase === candidate.phase && r.ordinal === candidate.ordinal); if (!slot || candidate.phase !== input.phase) throw Error("Dated original candidate slot required"); - return publishedSchema.parse({ - version: 2, - installationId: s.installation.installation.installationId, - origin: s.origin, - candidate, - childOperationKey: slot.execution.operationKey, - operationFingerprint: - "operationFingerprint" in slot - ? slot.operationFingerprint - : slot.execution.operationFingerprint, - serviceIds: [...slot.serviceIds], - }); + return publishedSchema.parse({ candidate }); } /** * Fixed registry resource input: the candidate data-manager recorded with the @@ -513,20 +495,19 @@ export function inspectDataTransitCandidate( ) throw Error("Fixed transit candidate required"); const local = installationState(installation); - if (!local.leases.listSync) throw Error("This role cannot read delegations"); - const grant = local.leases + const leases = localLeases(local.leases); + if (!leases) throw Error("This role cannot read delegations"); + const grant = leases .listSync() .flatMap((lease) => lease.children) .find( (child) => child.execution.operationKey === operationKey && child.recipient === "ops-agent", ); if (!grant?.evidence) throw Error("Transit candidate missing from the child grant"); - const proof = publishedSchema.parse(grant.evidence), - candidate = proof.candidate; + const candidate = publishedSchema.parse(grant.evidence).candidate, + proof = { candidate, serviceIds: grant.targets }; if ( - proof.installationId !== installation.installationId || - proof.childOperationKey !== operationKey || - proof.operationFingerprint !== controlDigest(operation) || + grant.execution.operationFingerprint !== controlDigest(operation) || operationKey !== controlChildOperationKey( candidate.execution.operationKey, @@ -626,7 +607,6 @@ export function verifyDataTransitCandidateForExecution( checked.digest !== expectedDigest || canonicalJson(admission.parent) !== canonicalJson(retainedControlParent(checked.proof.candidate.execution)) || - !sameControlOrigin(checked.proof.origin, s.origin) || canonicalJson(s.serviceIds) !== canonicalJson(checked.proof.serviceIds) ) throw Error("Candidate differs from original borrowed admission"); diff --git a/packages/core/src/build-artifacts/lease-remote.ts b/packages/core/src/build-artifacts/lease-remote.ts index 5bcbb4fb6..96e25e19e 100644 --- a/packages/core/src/build-artifacts/lease-remote.ts +++ b/packages/core/src/build-artifacts/lease-remote.ts @@ -45,17 +45,11 @@ export function remoteLeaseBackend(send: LeaseSender): LeaseBackend { return result.lease; }, read: async (leaseId) => (await run({ kind: "lease.read", leaseId })).lease ?? undefined, - list: async () => { - throw Error("Only the agent lists every lease"); - }, async delegate(input) { const lease = await update(input.leaseId, { action: "delegate", child: input.child }); if (!lease) throw Error("Target lease is not held"); return lease; }, - adopt: async () => { - throw Error("Children of another role are adopted in the agent"); - }, async settle(input) { const lease = await update(input.leaseId, { action: "settle", diff --git a/packages/core/src/build-artifacts/lease-store.test.ts b/packages/core/src/build-artifacts/lease-store.test.ts index 8a0a0a5fa..a2e27deba 100644 --- a/packages/core/src/build-artifacts/lease-store.test.ts +++ b/packages/core/src/build-artifacts/lease-store.test.ts @@ -97,12 +97,10 @@ describe("the agent's lease store", () => { parentKey: null, parent: owner, execution: child, - kind: "download-osm", - phase: "osm", + kind: "generation-hardlinks", + phase: "hardlinks", recipient: "data-manager", targets: ["osrm"], - permittedFurtherEdges: [], - depth: 1, }, }); // A widened target subset and a replayed child key are refused. @@ -114,12 +112,10 @@ describe("the agent's lease store", () => { parentKey: null, parent: owner, execution: execution("data-manager"), - kind: "download-osm", - phase: "osm", + kind: "generation-hardlinks", + phase: "hardlinks", recipient: "data-manager", targets: ["motis"], - permittedFurtherEdges: [], - depth: 1, }, }), ).rejects.toThrow(/Widened/); @@ -129,7 +125,7 @@ describe("the agent's lease store", () => { await leases.settle({ leaseId: lease.leaseId, key, settlement: settled(child) }); await leases.settle({ leaseId: lease.leaseId, key: null, settlement: settled(owner) }); await leases.release({ leaseId: lease.leaseId }); - expect(await leases.list()).toEqual([]); + expect(leases.listSync()).toEqual([]); }); it("marks an unknown outcome as needing recovery and keeps the lease", async () => { @@ -164,6 +160,6 @@ describe("the agent's lease store", () => { ), ); expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); - expect((await leases.list()).map((lease) => lease.targets)).toEqual([["shared"]]); + expect(leases.listSync().map((lease) => lease.targets)).toEqual([["shared"]]); }); }); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index c6dc95490..e46f35396 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -108,9 +108,9 @@ export type BorrowedArtifactTargetLease = { export type TargetLeaseAuthority = ArtifactTargetLease | BorrowedArtifactTargetLease; /** - * Where a role keeps its leases. The agent, its fixed CLI children and its - * configuration helper use the store directly; data-manager and the host CLI - * ask the agent, which applies the same rules on their behalf. + * The lease calls every role makes. The agent, its fixed CLI children, its + * configuration helper and the host CLI use the store directly; data-manager + * asks the agent, which applies the same rules on its behalf. */ export interface LeaseBackend { acquire(input: { @@ -119,15 +119,10 @@ export interface LeaseBackend { targets: readonly string[]; }): Promise; read(leaseId: string): Promise; - /** Only the store itself can answer without waiting. */ - readSync?(leaseId: string): LeaseRecord | undefined; - listSync?(): LeaseRecord[]; - list(): Promise; delegate(input: { leaseId: string; child: Omit; }): Promise; - adopt(input: { leaseId: string; key: string; execution: ExecutionRef }): Promise; settle(input: { leaseId: string; key: string | null; @@ -138,6 +133,21 @@ export interface LeaseBackend { /** Settles the data-manager child named by the settlement's identity, if it is still held. */ settleChild(settlement: LeaseSettlement): Promise; } +/** A role that reads the store itself also reads it without waiting and adopts children. */ +export interface LocalLeaseBackend extends LeaseBackend { + readSync(leaseId: string): LeaseRecord | undefined; + listSync(): LeaseRecord[]; + adopt(input: { leaseId: string; key: string; execution: ExecutionRef }): Promise; +} +/** The store-reading backend of a local role, or undefined for data-manager. */ +export function localLeases(backend: LeaseBackend): LocalLeaseBackend | undefined { + return "readSync" in backend ? (backend as LocalLeaseBackend) : undefined; +} +function requireLocalLeases(backend: LeaseBackend): LocalLeaseBackend { + const local = localLeases(backend); + if (!local) throw Error("This role cannot read the lease store"); + return local; +} const MAX_DESCENDANTS = 64; function nodeOf(lease: LeaseRecord, key: string | null) { @@ -165,7 +175,26 @@ function settledKnown(node: { state: string; settlement?: LeaseSettlement }): bo return node.state === "settled" && knownSettlement(node.settlement); } -/** The lease rules, applied under the store lock by every writer. */ +/** Who issues under `parentKey`, and the kind it runs: that child, or the lease's owner. */ +function issuerOf(lease: LeaseRecord, parentKey: string | null) { + if (parentKey === null) + return { role: lease.owner.executor, kind: lease.ownerKind, parent: undefined }; + const parent = lease.children.find((entry) => entry.key === parentKey); + if (!parent) throw Error("Missing retained child node"); + return { role: parent.recipient, kind: parent.kind, parent }; +} +/** The phase a child may issue under itself, if any; a grandchild issues none. */ +function furtherEdges(lease: LeaseRecord, child: LeaseChild): string[] { + if (child.parentKey !== null) return []; + return verifyFiniteControlEdge( + lease.owner.executor, + lease.ownerKind, + child.phase as ControlChildPhase, + child.recipient, + ); +} + +/** The lease rules, applied under the store lock by every writer; the edge grammar is checked here. */ export const leaseRules = { acquire(leases: LeaseRecord[], input: Parameters[0], now: number) { const existing = leases.find((lease) => sameExecution(lease.owner, input.owner)); @@ -201,20 +230,21 @@ export const leaseRules = { ) ) throw Error("Attempted child cannot be replayed"); - const parent = - input.child.parentKey === null ? undefined : nodeOf(lease, input.child.parentKey); - const scope = parent && "targets" in parent ? parent.targets : lease.targets; + const issuer = issuerOf(lease, input.child.parentKey); + const scope = issuer.parent?.targets ?? lease.targets; if (input.child.targets.some((id) => !scope.includes(id))) throw Error("Widened target subset"); - if ( - parent && - "permittedFurtherEdges" in parent && - (!parent.permittedFurtherEdges.includes(input.child.phase) || parent.depth >= 2) - ) + if (issuer.parent && !furtherEdges(lease, issuer.parent).includes(input.child.phase)) throw Error("Unauthorized descendant edge"); + verifyFiniteControlEdge( + issuer.role, + issuer.kind, + input.child.phase as ControlChildPhase, + input.child.recipient, + ); lease.children.push({ ...input.child, adopted: false, state: "held" }); return lease; }, - adopt(leases: LeaseRecord[], input: Parameters[0], now: number) { + adopt(leases: LeaseRecord[], input: Parameters[0], now: number) { const lease = leases.find((value) => value.leaseId === input.leaseId); const child = lease?.children.find((value) => value.key === input.key); if (!lease || !child || !sameExecution(child.execution, input.execution)) @@ -286,14 +316,16 @@ export const leaseRules = { }; /** The agent's own lease backend over its store. */ -export function localLeaseBackend(store: LeaseStore, now: () => number = Date.now): LeaseBackend { +export function localLeaseBackend( + store: LeaseStore, + now: () => number = Date.now, +): LocalLeaseBackend { const get = (leaseId: string) => store.get(leaseId); return { acquire: (input) => store.update((leases) => leaseRules.acquire(leases, input, now())), read: async (leaseId) => get(leaseId), readSync: get, listSync: () => store.list(), - list: async () => store.list(), delegate: (input) => store.update((leases) => leaseRules.delegate(leases, input, now())), adopt: (input) => store.update((leases) => leaseRules.adopt(leases, input, now())), settle: (input) => store.update((leases) => leaseRules.settle(leases, input)), @@ -346,8 +378,9 @@ function handleFor(state: State): TargetLeaseAuthority { } /** The current record of a lease the role can read without waiting, else the last one seen. */ function current(s: State): LeaseRecord | undefined { - if (s.backend.readSync) { - const record = s.backend.readSync(s.leaseId); + const local = localLeases(s.backend); + if (local) { + const record = local.readSync(s.leaseId); s.seen = record; return record; } @@ -426,11 +459,7 @@ export function verifyFiniteControlEdge( ) return []; if (role === "ops-agent" && kind === "configurationMaintenance.prepare") { - if ( - recipient === "data-manager" && - ["configuration-fence", "configuration-fence-settle"].includes(phase) - ) - return []; + if (recipient === "data-manager" && phase === "configuration-fence") return []; if ( recipient === "ops-agent" && [ @@ -549,7 +578,7 @@ export async function delegateArtifactTargetLease( if (!plan && parent) { const kinds = childPhaseKinds[child.phase]; if ( - parent.permittedFurtherEdges.includes(child.phase) && + furtherEdges(record, parent).includes(child.phase) && kinds.length === 1 && child.execution.operationKey === controlChildOperationKey(s.execution.operationKey, child.phase, 0) @@ -559,14 +588,6 @@ export async function delegateArtifactTargetLease( if (!plan) throw Error("Child is absent from immutable admitted plan"); if (child.serviceIds.some((id) => !lease.serviceIds.includes(id))) throw Error("Widened target subset"); - if (parent && (!parent.permittedFurtherEdges.includes(child.phase) || parent.depth >= 2)) - throw Error("Unauthorized descendant edge"); - const edges = verifyFiniteControlEdge( - parent?.recipient ?? owner.installation.installation.role, - owner.kind, - child.phase, - child.recipient, - ); if ( owner.kind === "configurationMaintenance.prepare" && [ @@ -590,8 +611,6 @@ export async function delegateArtifactTargetLease( phase: child.phase, recipient: child.recipient, targets: [...child.serviceIds], - permittedFurtherEdges: edges, - depth: (parent?.depth ?? 0) + 1, }; // What the child acts on stays in the store row, out of the header; a // recipient that needs it reads the store. @@ -643,8 +662,7 @@ export function inspectArtifactDelegation(options: { }) { const parsed = parseControlAssertion(options.assertion); const local = installationState(options.installation); - if (!local.leases.readSync) throw Error("This role cannot read delegations"); - const lease = local.leases.readSync(parsed.leaseId), + const lease = requireLocalLeases(local.leases).readSync(parsed.leaseId), grant = lease?.children.find((child) => child.key === parsed.delegationId); if (!lease || !grant) throw Error("Unanchored delegation"); const { @@ -699,7 +717,7 @@ export async function adoptArtifactTargetLease(options: { assertion: options.assertion, installation: peer.installation.installation, }); - const lease = peer.installation.leases.readSync?.(assertion.leaseId); + const lease = localLeases(peer.installation.leases)?.readSync(assertion.leaseId); if (!lease) throw Error("Unanchored delegation"); if ( !sameExecution(inspected.execution, options.execution) || @@ -749,7 +767,7 @@ export async function adoptArtifactTargetLease(options: { ) throw Error("Durable duplicate admission conflict"); } else writePrivateRecord(peer.installation, "adoptions", name, pending, true); - const adopted = await peer.installation.leases.adopt({ + const adopted = await requireLocalLeases(peer.installation.leases).adopt({ leaseId: assertion.leaseId, key: inspected.delegationId, execution: inspected.execution, @@ -938,8 +956,8 @@ export function verifyLocalArtifactTargetRootRelease( childSettlementState(verifyControlEffectSettlement(a.installation.installation, a.execution)); const recorded = executionLease(authority).releasedAt; if (recorded) return Object.freeze({ releasedAt: recorded }); - if (!a.installation.leases.readSync) throw Error("This role cannot read the lease store"); - if (a.installation.leases.readSync(leaseId)) throw Error("Original root release is not proved"); + if (requireLocalLeases(a.installation.leases).readSync(leaseId)) + throw Error("Original root release is not proved"); return Object.freeze({ releasedAt: rootReleasedAt(authority) }); } @@ -1045,9 +1063,9 @@ function assertImagePullTargetFree( const target = admitted.operation.serviceId; if (!state.installation.registeredTargets.has(target)) throw Error("Registered image target required"); - const list = state.installation.leases.listSync; - if (!list) throw Error("Image pulls run in the agent"); - for (const lease of list()) + const local = localLeases(state.installation.leases); + if (!local) throw Error("Image pulls run in the agent"); + for (const lease of local.listSync()) if (lease.targets.includes(target)) throw Error("Coordinator target is held"); } @@ -1206,7 +1224,7 @@ export function inspectLocalLeaseChildSettlement( execution: ExecutionRef, ): ChildSettlement | null { const s = stateOf(lease); - if (!s.backend.readSync) throw Error("This role cannot read the lease store"); + requireLocalLeases(s.backend); return childSettlementOf(current(s), s.node, execution); } /** @@ -1355,7 +1373,7 @@ export function assertArtifactDelegationRootHeld( ): void { const local = installationState(installation), parsed = parseControlAssertion(assertion), - lease = local.leases.readSync?.(parsed.leaseId); + lease = localLeases(local.leases)?.readSync(parsed.leaseId); if (!lease || lease.state === "settled") throw Error("Original owner lease is no longer held"); } /** Read-only terminal evidence of a borrowed child after the agent restarted. */ @@ -1365,7 +1383,7 @@ export function verifyRetainedArtifactTargetSettlement( execution: ExecutionRef, ): void { const local = installationState(installation); - const lease = local.leases.readSync?.(adoption.leaseId); + const lease = localLeases(local.leases)?.readSync(adoption.leaseId); const child = lease?.children.find((value) => value.key === adoption.delegationId); const proof = childSettlementState(verifyControlEffectSettlement(installation, execution)); if ( @@ -1429,7 +1447,7 @@ function maintenanceLease(authority: VerifiedExecutionAuthority): LeaseRecord { owner.installation.fixedChildId ) throw Error("Original maintenance runtime tree required"); - const lease = owner.installation.leases.readSync?.(executionLeaseId(authority)); + const lease = localLeases(owner.installation.leases)?.readSync(executionLeaseId(authority)); if (!lease || !sameExecution(lease.owner, owner.execution)) throw Error("Original maintenance root is not held"); return lease; @@ -1447,7 +1465,7 @@ export function verifyConfigurationMaintenanceRootClosure( // owner records the release; the released tree is no longer kept. const state = executionAuthorityState(authority); if ( - !state.installation.leases.readSync?.(executionLeaseId(authority)) && + !localLeases(state.installation.leases)?.readSync(executionLeaseId(authority)) && executionLease(authority).releasedAt ) return; diff --git a/packages/core/src/ops/lease-records.ts b/packages/core/src/ops/lease-records.ts index f9baf7b6a..87b4078c0 100644 --- a/packages/core/src/ops/lease-records.ts +++ b/packages/core/src/ops/lease-records.ts @@ -29,8 +29,6 @@ export const leaseChildIntentSchema = z.strictObject({ phase: z.string().min(1).max(64), recipient: z.enum(["fixed-cli", "data-manager", "ops-agent", "configuration-helper"]), targets: leaseTargetsSchema, - permittedFurtherEdges: z.array(z.string().max(64)).max(1), - depth: z.number().int().min(1).max(2), /** * What the issuer prepared for the recipient to act on, such as the transit * candidate data-manager staged for the agent. The recipient re-checks it From 29836afbd477d07e07439294b3aabb902a0b6094 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 16:23:43 +0100 Subject: [PATCH 06/42] docs(control): describe the shared lease store lock and the cases control clear must settle The host CLI writes the lease store under the same lock as the agent and its helper. Holders in another PID namespace without a container ID, and holders on macOS, cannot be proved gone, so their locks need `openmapx control clear --lock`, which also removes a left reclaim guard. Clearing a maintenance leaves its data-manager fence prepared until `control resolve` settles it. --- .../docs/developer/build-artifacts-control.md | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index eb25a5b7e..91c2587ee 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -520,10 +520,11 @@ child names a data-manager job or helper container. ### Other target rules -- **Writer lock.** The CLI goes through the agent when it is up. When the agent is - down, a lock file records PID, boot ID and process start time, and is reclaimed - only when that process is provably gone. The lock is never held across a Docker - effect. +- **Lease store and writer lock.** The agent, its helper and the host CLI write the + lease store directly, each under the same lock file. A lock file records PID, + boot ID, PID namespace, container and process start time, and is reclaimed only + when that process is provably gone. Reclaimers take turns through a short guard + file beside the lock. The lock is never held across a Docker effect. - **Desired configuration.** The payload travels in the authenticated operation. The job row moves through three phases: candidate prepared; rows and decision saved in one transaction; generation applied by a keyed operation. Recovery runs @@ -532,6 +533,18 @@ child names a data-manager job or helper container. previous pointer is the manual rollback target. - **Operator exit.** `openmapx control status`, plus `control clear --op ` or `control resolve --as committed|failed`: host-only, printing what is believed - live, and audited. + live, and audited. Clearing a configuration maintenance leaves the data-manager's + configuration fence prepared; `control resolve` of the same operation settles it, + also after a clear. +- **Lock reclaim limits.** Some dead holders cannot be proved gone, and their lock + stays until `openmapx control clear --lock `: + - a holder in another PID namespace without a container ID, such as the host CLI + as the agent or the helper sees it; + - any holder on macOS, which records no process start time, so a reused process + ID counts as alive. + + Check that the named process is gone first. The same command also removes a + reclaim guard that a dead reclaimer left beside the lock. + - **Self-update order.** Pull first, drain, run the helper from the current image, recreate in order with verification after each service, agent last. From d698876ffa0036c2beaf2e7a5bd23be84dd353a4 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 16:53:00 +0100 Subject: [PATCH 07/42] fix(control): run data-manager hardlink leaves on their issuer's targets Since the leases moved into the agent's store, a data-manager hardlink leaf ran with data-manager's own default targets instead of the consumer services of the lease it runs under, so data-manager refused every non-empty plan and the issuer waited until its deadline. The leaf now takes the targets the delegation names. The agent records data-manager's terminal status even when data-manager already settled the leaf itself; later steps of the owner build on it. A configuration maintenance whose root was released no longer compares its final against the released lease tree, which the store does not keep; the comparison ran before the release. --- .../src/configuration-maintenance-store.ts | 34 +++++++++++-------- .../core/src/build-artifacts/target-lease.ts | 29 ++++++++++------ .../src/jobs/execution/receiver.ts | 1 + .../src/jobs/execution/runtime.ts | 15 ++++++-- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index 115f291b0..bb6182291 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -69,6 +69,7 @@ import { cancelArtifactTargetLease, cancelControlExecution, closeArtifactTargetLeaseHandle, + configurationMaintenanceRootReleased, configurationMaintenanceTargetOwner, consumeControlChildDispatch, controlDigest, @@ -2097,21 +2098,26 @@ export function restoreOwnedMaintenanceFinal( const contained = final.ownerFinal.payload.outcome === "contained"; if (contained) assertContainedConfigurationMaintenanceFinalState(owner); else assertAppliedConfigurationMaintenanceFinalState(owner); - const actual = readOriginalMaintenanceFinalChildren( - authority, - observeConfigurationMaintenanceRuntimeTree(authority), - ); + // The children were compared before the root was released; the store + // keeps no released tree to compare again. + const actual = configurationMaintenanceRootReleased(authority) + ? undefined + : readOriginalMaintenanceFinalChildren( + authority, + observeConfigurationMaintenanceRuntimeTree(authority), + ); if ( - (!contained && !actual.complete) || - (contained && - (actual.generationResult?.outcome !== "not_committed" || - actual.generationResult.cleanupFailed || - actual.children.length !== 1)) || - canonicalJson(actual.children) !== canonicalJson(final.evidence.children) || - final.evidence.generation.commitReceiptDigest !== - (contained ? null : actual.generation?.terminalDigest) || - canonicalJson(final.evidence.followUp.receiptDigests) !== - canonicalJson(actual.followUpReceiptDigests) + actual && + ((!contained && !actual.complete) || + (contained && + (actual.generationResult?.outcome !== "not_committed" || + actual.generationResult.cleanupFailed || + actual.children.length !== 1)) || + canonicalJson(actual.children) !== canonicalJson(final.evidence.children) || + final.evidence.generation.commitReceiptDigest !== + (contained ? null : actual.generation?.terminalDigest) || + canonicalJson(final.evidence.followUp.receiptDigests) !== + canonicalJson(actual.followUpReceiptDigests)) ) throw Error("Original applied final child sources changed"); } else assertUnusedConfigurationMaintenanceFinalState(owner); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index e46f35396..1ab199674 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -1243,17 +1243,19 @@ export async function settleDataManagerLeaseChild( ? leaseChildrenOf(record, s.node).find((value) => sameExecution(value.execution, execution)) : undefined; if (!record || !child) throw Error("Exact retained child grant required"); - if (settledKnown(child)) return child.settlement as ChildSettlement; if ( !["ok", "partial", "error", "canceled", "interrupted"].includes(status.status) || status.publication === "unknown" ) - return null; + return settledKnown(child) ? (child.settlement as ChildSettlement) : null; + // Recorded even when data-manager already settled the child itself: later + // steps of this owner build on the status it observed. const observed = recordDataControlTerminalObservation( s.installation.installation, execution, status, ); + if (settledKnown(child)) return child.settlement as ChildSettlement; return settleIssuedLeaseChild(lease, execution, { outcome: status.publication === "not_started" @@ -1458,17 +1460,24 @@ export function observeConfigurationMaintenanceRuntimeTree( ): ConfigurationMaintenanceClosureObservation { return closureFromRecord(maintenanceLease(authority), authority); } -export function verifyConfigurationMaintenanceRootClosure( +/** + * Whether the maintenance root is gone from the store after the release its + * owner recorded. The store releases it only after the closure check passed, + * and the released tree is no longer kept. + */ +export function configurationMaintenanceRootReleased( authority: VerifiedExecutionAuthority, -): void { - // The store releases a maintenance root only after this check passed, and the - // owner records the release; the released tree is no longer kept. +): boolean { const state = executionAuthorityState(authority); - if ( + return ( !localLeases(state.installation.leases)?.readSync(executionLeaseId(authority)) && - executionLease(authority).releasedAt - ) - return; + Boolean(executionLease(authority).releasedAt) + ); +} +export function verifyConfigurationMaintenanceRootClosure( + authority: VerifiedExecutionAuthority, +): void { + if (configurationMaintenanceRootReleased(authority)) return; const observation = observeConfigurationMaintenanceRuntimeTree(authority); try { assertConfigurationMaintenanceRootClosure(authority, observation); diff --git a/services/data-manager/src/jobs/execution/receiver.ts b/services/data-manager/src/jobs/execution/receiver.ts index 178626199..3ea7cbd18 100644 --- a/services/data-manager/src/jobs/execution/receiver.ts +++ b/services/data-manager/src/jobs/execution/receiver.ts @@ -111,6 +111,7 @@ export function createDataExecutionReceiver(deps: { principal: verified.execution.principal, adopt: verified.adopt, hardlinkPlan: request.plan, + serviceIds: verified.serviceIds, }); }, async admitLocal(value, source) { diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index 05bf46185..23ce74920 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -81,6 +81,8 @@ export interface DataExecutionRuntime { id?: string; principal?: ExecutionRef["principal"]; hardlinkPlan?: unknown; + /** For a hardlink leaf: the targets of the issuer's lease it runs under. */ + serviceIds?: readonly string[]; sourceJobId?: string; triggeredBy?: string; idempotencyKey?: string; @@ -253,9 +255,11 @@ export function createDataExecutionRuntime(deps: { adopt?: (execution: ExecutionRef) => Promise, parent?: VerifiedParentExecution, hardlinkPlan?: unknown, + leafTargets?: readonly string[], ) { const owner = requireJournal(), - targets = dataExecutionTargetIds(record.input); + leaf = record.input.kind === "generation-hardlinks", + targets = leaf && leafTargets ? [...leafTargets] : dataExecutionTargetIds(record.input); await adopt?.(record.execution); const authority = retainAdmittedControlExecution(owner.installation, { execution: record.execution, @@ -269,7 +273,6 @@ export function createDataExecutionRuntime(deps: { serviceIds: targets, reservations: record.reservations.map(dataChildControlReservation), }); - const leaf = record.input.kind === "generation-hardlinks"; if (leaf) { if (hardlinkPlan === undefined) throw Error("Durable generation plan required"); await owner.retainHardlinksPlan(authority, hardlinkPlan); @@ -534,7 +537,13 @@ export function createDataExecutionRuntime(deps: { const { created, record } = await store.admit(input, options); if (created) try { - await start(record, options.adopt, options.parent, options.hardlinkPlan); + await start( + record, + options.adopt, + options.parent, + options.hardlinkPlan, + options.serviceIds, + ); } catch (cause) { await store.checkpoint(record.status.id, record.version, { publication: "unknown" }); throw new DataExecutionRuntimeUnavailable("Data admission requires ownership recovery", { From 03859a4f3b87cd94050483b017c977b1597bd041 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 18:45:48 +0100 Subject: [PATCH 08/42] fix(data-manager): name a busy configuration fence refusal in its reply Fastify's default error reply carried the refusal reason in `message`, while the data-manager client reads `error`, so the agent could not tell a busy fence refusal, which wrote nothing, from another conflict. Its fence child then stayed unsettled and the maintenance root was never released. The prepare route now answers a busy refusal with that name. --- services/data-manager/src/jobs/execution/api.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/services/data-manager/src/jobs/execution/api.ts b/services/data-manager/src/jobs/execution/api.ts index 88c910a5d..8fb2c88e3 100644 --- a/services/data-manager/src/jobs/execution/api.ts +++ b/services/data-manager/src/jobs/execution/api.ts @@ -106,7 +106,14 @@ export function registerDataExecutionApi( }); registerDataManagerRoute(app, "prepareDataConfigurationFence", async (request, reply) => { if (!receiver) return unavailable(reply); - return receiver.prepareFence(request.body, request.raw.rawHeaders, principal(request)); + try { + return await receiver.prepareFence(request.body, request.raw.rawHeaders, principal(request)); + } catch (error) { + // The agent tells a busy refusal, which wrote nothing, from other conflicts by this name. + if ((error as Error).message === "configuration_maintenance_busy") + return reply.code(409).send({ error: "configuration_maintenance_busy" }); + throw error; + } }); registerDataManagerRoute( app, From 7224216fe87924139d618cb170ea3469b6a80a4a Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 20:06:15 +0100 Subject: [PATCH 09/42] fix(data-manager): record a contained terminal before releasing its lease Recording the terminal checks how the execution's proxy children settled, which the lease shows only while it is held; the store keeps no released lease. The terminal is now recorded after the settlement and before the release, so a catalog validation no longer leaves its terminal unrecorded. The catalog workflow test no longer reads a released lease's children, which the store does not keep. --- .../catalog-workflow-postgres.test.ts | 20 ++++++---------- .../src/jobs/execution/runtime.ts | 24 +++++++++++-------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts index acc3ef789..72220ac6d 100644 --- a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts @@ -797,20 +797,14 @@ it installation, proof.sourceExecution, ); + // The validation released its lease only after every child settled; the + // store keeps no released lease, so the settled child is no longer listed. expect( - ( - await control.inspectRetainedOwnedArtifactTargetChildSettlement( - originalOwner, - child.execution, - ) - )?.outcome, - ).toBe("committed"); - await expect( - await control.inspectRetainedOwnedArtifactTargetChildSettlement(originalOwner, { - ...child.execution, - executionId: randomUUID(), - }), - ).rejects.toThrow(/original child/); + await control.inspectRetainedOwnedArtifactTargetChildSettlement( + originalOwner, + child.execution, + ), + ).toBeNull(); expect(control.heldTargets(dirname(data)).includes("motis")).toBe(false); const validationBytes = readFileSync( join(data, "motis/proposals", proposalId, `validation-${validation.id}.json`), diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index 23ce74920..ce2587879 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -97,9 +97,10 @@ export interface DataExecutionRuntime { reconcile(): Promise; close(): Promise; } -/** Settles and releases the execution's own lease after its terminal record, - * or, for a hardlink leaf, settles its child in the issuer's lease; then - * records the terminal for later steps that build on it. */ +/** Settles the execution's own lease after its terminal record, or, for a + * hardlink leaf, its child in the issuer's lease; records the terminal for + * later steps that build on it, while the lease still shows how its children + * settled; then releases the lease. */ async function releaseOwnLease( journal: { installation: ControlInstallation; @@ -108,14 +109,17 @@ async function releaseOwnLease( execution: ExecutionRef, lease: ArtifactTargetLease | undefined, ) { - if (lease) { - await settleArtifactTargetLeaseChild( - lease, - verifyControlEffectSettlement(journal.installation, execution), - ); - await releaseArtifactTargetLease(lease); - } else await settleOwnLeaseChild(journal.installation, execution); + if (!lease) { + await settleOwnLeaseChild(journal.installation, execution); + await journal.recordContainedTerminal(execution); + return; + } + await settleArtifactTargetLeaseChild( + lease, + verifyControlEffectSettlement(journal.installation, execution), + ); await journal.recordContainedTerminal(execution); + await releaseArtifactTargetLease(lease); } export function dataExecutionTargetIds(input: DataExecutionInput): string[] { return ["transit-sync", "catalog.validate", "catalog.activate"].includes(input.kind) From c7fc9ba34233d131de4b08e16cd1af9f1ef11128 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 20:12:40 +0100 Subject: [PATCH 10/42] refactor(ops-agent): keep the contained-process check private to the Docker runtime Its only other user, the in-process lifecycle exclusion, is gone. --- apps/ops-agent/src/docker-runtime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 7c4bf2727..251576e7b 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -36,7 +36,7 @@ import { decideServiceAction } from "./service-action-policy"; const containedProcessFailures = new WeakSet(); /** Only actual contained-process adapters mint this observation; it is not request authority. */ -export function isContainedProcessFailure(error: unknown): boolean { +function isContainedProcessFailure(error: unknown): boolean { return typeof error === "object" && error !== null && containedProcessFailures.has(error); } const reportedProcessFailures = new WeakSet(); From 17a90048c3335193bd1f1470be529af5592466e3 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 21:11:32 +0100 Subject: [PATCH 11/42] test(crash-points): drop the services restart known failure and restate the killed-start one A host `services restart` during an update now takes a lease and exits busy, so that scenario passes and leaves the registry. An agent killed during a service start no longer leaves its configuration writer held; the restarted agent keeps the target held with an unknown outcome while the API job stays running, which is what that entry now describes. --- .../services/__tests__/crash-points/known-failures.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/api/src/services/__tests__/crash-points/known-failures.json b/apps/api/src/services/__tests__/crash-points/known-failures.json index fecb24d4e..250872d24 100644 --- a/apps/api/src/services/__tests__/crash-points/known-failures.json +++ b/apps/api/src/services/__tests__/crash-points/known-failures.json @@ -12,15 +12,11 @@ "fails": "^Timed out waiting for job [0-9a-f-]+\\nlast seen: (?=.*\"status\":\"running\")(?=.*\"observation\":\\{\"state\":\"unknown\",\"reason\":\"transport\")" }, "never starts the service a second time and keeps the target held": { - "reason": "An agent killed during a service start leaves its configuration writer held, so the restarted agent refuses to start.", - "fails": "^Timed out waiting for agent to answer: fetch failed\\n[\\s\\S]*ops-agent failed to start: Configuration writer is busy" + "reason": "After the agent restarts, the interrupted start keeps its target held with an unknown outcome, but the API job stays running with observation recovery_required instead of ending, so the scenario never sees a final state.", + "fails": "^Timed out waiting for job [0-9a-f-]+\\nlast seen: (?=.*\"status\":\"running\")(?=.*\"observation\":\\{\"state\":\"unknown\",\"reason\":\"recovery_required\")" }, "finishes the recreate in flight, starts no further effect and never succeeds": { "reason": "A cancelled update's helper keeps recreating services, including the agent, and the update succeeds.", "fails": "^expected 'success' not to be 'success'" - }, - "exit busy without a Compose effect and succeed after the update": { - "reason": "services restart is unleased: it runs compose restart while the helper recreates services.", - "fails": "^services restart: [\\s\\S]*expected \\+0 not to be \\+0" } } From 25cc687a268ac9c39e46a5f17129a94dec141725 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 21:56:35 +0100 Subject: [PATCH 12/42] fix(data-manager): refuse the API's token on unknown paths before routing The API's route-restricted token is checked against its allowlist before routing, so a path outside that list is refused with 403 even when no route matches it. Unknown paths still reach the not-found reply for the agent and the host CLI, and unauthenticated requests are still refused with 401. --- services/data-manager/__tests__/auth.test.ts | 13 +++++++++++-- services/data-manager/src/auth.ts | 7 +++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/services/data-manager/__tests__/auth.test.ts b/services/data-manager/__tests__/auth.test.ts index d898869c6..eedc35ebc 100644 --- a/services/data-manager/__tests__/auth.test.ts +++ b/services/data-manager/__tests__/auth.test.ts @@ -392,7 +392,7 @@ describe("per-caller route allowlist", () => { } }); - it("refuses the API's token on a route that does not admit it", async () => { + it("refuses the API's token on a route that does not admit it, and on an unknown path", async () => { const app = Fastify(); registerAuth(app, TOKENS); app.get("/unlisted", async () => ({ ok: true })); @@ -411,7 +411,16 @@ describe("per-caller route allowlist", () => { url: "/missing", headers: { authorization: `Bearer ${TOKEN}` }, }); - expect(missing.statusCode).toBe(404); + expect(missing.statusCode).toBe(403); + for (const token of [TOKENS["ops-agent"], TOKENS["host-cli"]]) { + const unknown = await app.inject({ + url: "/missing", + headers: { authorization: `Bearer ${token}` }, + }); + expect(unknown.statusCode).toBe(404); + } + const anonymous = await app.inject({ url: "/missing" }); + expect(anonymous.statusCode).toBe(401); } finally { await app.close(); } diff --git a/services/data-manager/src/auth.ts b/services/data-manager/src/auth.ts index faa11e5d4..9b9bd8af5 100644 --- a/services/data-manager/src/auth.ts +++ b/services/data-manager/src/auth.ts @@ -108,7 +108,6 @@ function assertDistinct(tokens: DataManagerCallerTokens): void { throw new Error("Each data-manager caller needs its own token"); } -/** The callers a route admits, from its contract; a route without that list admits none. */ /** Whether a route admits the API's token; the agent and host CLI reach every route. */ function admitsApi(req: FastifyRequest): boolean { const schema = req.routeOptions.schema as { "x-openmapx-api-caller"?: unknown } | undefined; @@ -153,8 +152,9 @@ export function registerAuth( .send({ error: "Too many authentication attempts" }); return reply.code(401).send({ error: "Unauthorized" }); } - // An unknown path falls through to the not-found reply. - if (req.routeOptions.url !== undefined && principal === "api" && !admitsApi(req)) + // The API's token is checked against its allowlist before routing, so an + // unknown path is refused for it as well; other callers get the not-found reply. + if (principal === "api" && (req.routeOptions.url === undefined || !admitsApi(req))) return reply.code(403).send({ error: "This caller may not use this route" }); principals.set(req, principal); // Durable receiver admission only. Never silently ignore an internal authority header. @@ -166,7 +166,6 @@ export function registerAuth( ([ "/executions", "/executions/generation-hardlinks", - "/executions/generation-hardlinks/recovery", "/executions/configuration-maintenance/prepare", "/executions/configuration-maintenance/recovery", ].includes(path) || From 9348c2396b915ccbdaeb45317e5960b9c346f7b0 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 22:02:26 +0100 Subject: [PATCH 13/42] refactor(control): hold the backup-store and update receipt locks as recoverable lock files Backup create, restore, delete, prune and a retention change now serialise on one lock file in the backups directory under the recoverable lock's rules: a holder that died is reclaimed when it is provably gone, a live one is waited for up to a minute and then named in a busy error. The CLI's system update receipt uses the same lock and refuses a second writer at once. Neither lock expires by age any more, and inventories skip the lock and its reclaim files. --- apps/ops-agent/src/administrative-runtime.ts | 54 ++++++------------ apps/ops-agent/src/backup-prune-runtime.ts | 28 ++++----- .../cli-system-update-receipt.test.ts | 29 +++++++++- packages/cli/src/commands/control.ts | 2 +- packages/cli/src/lib/system-update-receipt.ts | 18 ++---- .../__tests__/backup-retention-store.test.ts | 30 +++++++++- .../core/src/services/backup-prune-plan.ts | 3 +- .../src/services/backup-retention-store.ts | 57 ++++++++++++------- packages/core/src/services/index.ts | 7 ++- 9 files changed, 133 insertions(+), 95 deletions(-) diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index e8c62b093..0f29f2cc4 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -58,8 +58,7 @@ import { type OpsRuntime, } from "./runtime"; -const { acquireStoreLock, acquireReleaseStoreLock } = coreServices; -type ReleaseStoreLock = coreServices.ReleaseStoreLock; +const { acquireReleaseStoreLock } = coreServices; const MAX_BACKUP_MANIFEST_BYTES = 1024 * 1024; const MAX_BACKUP_SERVICES = 256; @@ -80,7 +79,6 @@ const INSPECT_RELEASE_SERVICE_IDS = [ const MAX_RELEASE_MANIFEST_BYTES = coreServices.RELEASE_STORE_LIMITS.manifestBytes; const MAX_RELEASE_STATE_BYTES = coreServices.RELEASE_STORE_LIMITS.stateBytes; const MAX_RELEASE_SELECTIONS_BYTES = coreServices.RELEASE_STORE_LIMITS.selectionsBytes; -const BACKUP_STORE_LOCK_NAME = ".backup-store.lock"; const BACKUP_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const SERVICE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; const _REGION_ID = /^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$/; @@ -545,17 +543,9 @@ function releaseDigest(contents: string): string { return createHash("sha256").update(contents).digest("hex"); } -function acquireBackupStoreLock(rootDir: string): Promise { - const root = backupRoot(rootDir); - mkdirSync(root, { recursive: true, mode: 0o700 }); - return acquireStoreLock( - root, - BACKUP_STORE_LOCK_NAME, - {}, - Date.now, - false, - coreServices.BACKUP_STORE_LOCK_TTL_MS, - ); +function withBackupStoreLock(rootDir: string, run: () => Promise): Promise { + mkdirSync(backupRoot(rootDir), { recursive: true, mode: 0o700 }); + return coreServices.withBackupStoreLock(infraDirectory(rootDir), run); } export async function pruneBackupRetention( @@ -566,8 +556,7 @@ export async function pruneBackupRetention( if (!Number.isSafeInteger(retentionDays) || retentionDays <= 0 || retentionDays > 36_500) { throw new Error("Backup retention days must be an integer between 1 and 36500"); } - const lock = await acquireBackupStoreLock(rootDir); - try { + await withBackupStoreLock(rootDir, async () => { if ( coreServices.readAppliedPlatformValues(join(rootDir, "infra", "docker")) .backupRetentionDays !== retentionDays @@ -577,9 +566,7 @@ export async function pruneBackupRetention( signal: new AbortController().signal, emitLog: () => {}, }); - } finally { - lock.release(); - } + }); } /** @@ -1110,15 +1097,12 @@ export function createAdministrativeRuntime( }; runtime["backup.list"] = async () => inspectBackupInventory(options.rootDir); runtime["backup.create"] = async (operation, context) => { - const lock = await acquireBackupStoreLock(options.rootDir); - try { - await runFixedCli(["backup", "create", "--name", operation.backupId], { + await withBackupStoreLock(options.rootDir, () => + runFixedCli(["backup", "create", "--name", operation.backupId], { ...context, operationKind: context.claim.operation.kind, - }); - } finally { - lock.release(); - } + }), + ); return { backupId: operation.backupId }; }; runtime["backup.restore"] = async (operation, context) => { @@ -1127,8 +1111,7 @@ export function createAdministrativeRuntime( // exact bytes immediately before dispatch, so a manifest replaced between // the resource claim and the effect cannot supply different service // authority under the same caller-visible backup ID. - const lock = await acquireBackupStoreLock(options.rootDir); - try { + await withBackupStoreLock(options.rootDir, async () => { const digest = inspectBackupManifestDigest(options.rootDir, operation.backupId); if (digest === null) throw new Error("Backup authority rejected"); const manifest = parseManifest(options.rootDir, operation.backupId); @@ -1149,21 +1132,16 @@ export function createAdministrativeRuntime( if (inspectBackupManifestDigest(options.rootDir, operation.backupId) !== digest) { throw new Error("Backup authority rejected"); } - } finally { - lock.release(); - } + }); return { backupId: operation.backupId }; }; runtime["backup.delete"] = async (operation, context) => { - const lock = await acquireBackupStoreLock(options.rootDir); - try { - await runFixedCli(["backup", "delete", operation.backupId], { + await withBackupStoreLock(options.rootDir, () => + runFixedCli(["backup", "delete", operation.backupId], { ...context, operationKind: context.claim.operation.kind, - }); - } finally { - lock.release(); - } + }), + ); return { backupId: operation.backupId }; }; runtime["data.inspect"] = async (operation) => inspectDataInventory(options.rootDir, operation); diff --git a/apps/ops-agent/src/backup-prune-runtime.ts b/apps/ops-agent/src/backup-prune-runtime.ts index f7211e8fe..6bc6515da 100644 --- a/apps/ops-agent/src/backup-prune-runtime.ts +++ b/apps/ops-agent/src/backup-prune-runtime.ts @@ -1,7 +1,6 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import { fsyncSync } from "node:fs"; import { rmdir, unlink } from "node:fs/promises"; -import { join } from "node:path"; import { AnchoredDirectory, beginControlEffect, @@ -84,23 +83,18 @@ export function installBackupPruneRuntime( if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } if (!root) return run(); - const lock = await services.acquireStoreLock( - join(options.infraDir, "backups"), - ".backup-store.lock", - {}, - Date.now, - false, - services.BACKUP_STORE_LOCK_TTL_MS, + const opened = root; + return await services.withBackupStoreLock( + options.infraDir, + async () => { + signal.throwIfAborted(); + opened.check(); + const result = await run(); + opened.check(); + return result; + }, + signal, ); - try { - signal.throwIfAborted(); - root.check(); - const result = await run(); - root.check(); - return result; - } finally { - lock.release(); - } } finally { root?.close(); infra.close(); diff --git a/packages/cli/__tests__/cli-system-update-receipt.test.ts b/packages/cli/__tests__/cli-system-update-receipt.test.ts index 18e0e4480..43370074c 100644 --- a/packages/cli/__tests__/cli-system-update-receipt.test.ts +++ b/packages/cli/__tests__/cli-system-update-receipt.test.ts @@ -1,6 +1,8 @@ -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { services } from "@openmapx/core/server"; import { afterEach, expect, it } from "vitest"; import { SystemUpdateReceiptStore } from "../src/lib/system-update-receipt"; @@ -52,3 +54,28 @@ it("keeps backups enabled when reading an older receipt without a backup choice" writeFileSync(store.path, JSON.stringify(original)); expect(store.read().createBackup).toBe(true); }); +it("refuses a second writer at once while the receipt lock is held", async () => { + const store = fixture(); + const lock = join(store.path, "..", ".update.json.lock"); + await services.withRecoverableLock(lock, { purpose: "other update" }, async () => { + await expect(store.create(identity)).rejects.toThrow(/held by other update/); + }); + await expect(store.create(identity)).resolves.toMatchObject({ version: 1 }); + expect(existsSync(lock)).toBe(false); +}); +it("reclaims the receipt lock of a writer that has exited", async () => { + const store = fixture(); + const lock = join(store.path, "..", ".update.json.lock"); + const exited = spawnSync(process.execPath, ["-e", "process.stdout.write(String(process.pid))"], { + encoding: "utf8", + }); + let self: ReturnType | undefined; + await services.withRecoverableLock(lock, { purpose: "crashed update" }, () => { + self = services.readLockHolder(lock); + }); + writeFileSync(lock, JSON.stringify({ ...self, pid: Number(exited.stdout), startTime: null }), { + mode: 0o600, + }); + await expect(store.create(identity)).resolves.toMatchObject({ version: 1 }); + expect(existsSync(lock)).toBe(false); +}); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index 1110982c0..9ca716b96 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -24,7 +24,7 @@ import { repoPaths } from "../lib/paths"; const LOCKS = { leases: () => join(leaseStoreDirectory(repoPaths().infraDir), "leases.lock"), writer: () => services.configurationWriterLockPath(repoPaths().infraDir), - backups: () => join(repoPaths().infraDir, "backups", ".backup-store.lock"), + backups: () => services.backupStoreLockPath(repoPaths().infraDir), } as const; function store() { diff --git a/packages/cli/src/lib/system-update-receipt.ts b/packages/cli/src/lib/system-update-receipt.ts index cc2416906..9cbfd2a77 100644 --- a/packages/cli/src/lib/system-update-receipt.ts +++ b/packages/cli/src/lib/system-update-receipt.ts @@ -9,7 +9,7 @@ import { openSync, writeFileSync, } from "node:fs"; -import { basename, dirname, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { adminPlatformConfigurationSchema } from "@openmapx/core/api-contracts"; import { atomicWrite, fsyncDirectory } from "@openmapx/core/release-files"; import { services } from "@openmapx/core/server"; @@ -68,20 +68,14 @@ export class SystemUpdateReceiptStore { throw Error("Update receipt unavailable or unsafe; preserve it for inspection"); } } + /** One writer at a time; a second update command is refused at once while one runs. */ private async locked(run: () => T): Promise { const dir = this.parent(); - const lock = await services.acquireStoreLock( - dir, - `.${basename(this.path)}.lock`, - {}, - Date.now, - true, + return services.withRecoverableLock( + join(dir, `.${basename(this.path)}.lock`), + { purpose: "system-update-receipt", waitMs: 0 }, + run, ); - try { - return run(); - } finally { - lock.release(); - } } async create(identity: z.input): Promise { return this.locked(() => { diff --git a/packages/core/src/services/__tests__/backup-retention-store.test.ts b/packages/core/src/services/__tests__/backup-retention-store.test.ts index d5007145b..21cbcce7f 100644 --- a/packages/core/src/services/__tests__/backup-retention-store.test.ts +++ b/packages/core/src/services/__tests__/backup-retention-store.test.ts @@ -10,11 +10,14 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, expect, it } from "vitest"; +import { backupPruneInventory } from "../backup-prune-plan"; import { + backupStoreLockPath, retainBackupExpiryCeilings, withBackupRetentionPolicyChange, + withBackupStoreLock, } from "../backup-retention-store"; -import { acquireStoreLock } from "../release-store-lock"; +import { readLockHolder, withRecoverableLock } from "../recoverable-lock"; let root: string; beforeEach(() => { @@ -34,9 +37,10 @@ beforeEach(() => { afterEach(() => rmSync(root, { recursive: true, force: true })); it("keeps the existing backup lock through policy publication and preserves shorter ceilings", async () => { await withBackupRetentionPolicyChange(root, 30, 60, async () => { + expect(readLockHolder(backupStoreLockPath(root))?.purpose).toBe("backup-store"); await expect( - acquireStoreLock(join(root, "backups"), ".backup-store.lock", {}, Date.now, true), - ).rejects.toThrow(/busy/); + withRecoverableLock(backupStoreLockPath(root), { purpose: "other", waitMs: 0 }, () => {}), + ).rejects.toThrow(/held by backup-store/); const manifest = JSON.parse( readFileSync(join(root, "backups", "original", "manifest.json"), "utf8"), ); @@ -78,3 +82,23 @@ it.each(["symlink", "hardlink"])( expect(readFileSync(other)).toEqual(before); }, ); +it("serialises backup effects on one lock file that inventories skip", async () => { + const order: string[] = []; + let releaseFirst!: () => void; + const first = withBackupStoreLock(root, async () => { + order.push("first"); + await new Promise((resolve) => { + releaseFirst = resolve; + }); + order.push("first done"); + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(() => backupPruneInventory(root, 30, new Date())).not.toThrow(); + const second = withBackupStoreLock(root, async () => { + order.push("second"); + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + releaseFirst(); + await Promise.all([first, second]); + expect(order).toEqual(["first", "first done", "second"]); +}); diff --git a/packages/core/src/services/backup-prune-plan.ts b/packages/core/src/services/backup-prune-plan.ts index cc682be57..48488375a 100644 --- a/packages/core/src/services/backup-prune-plan.ts +++ b/packages/core/src/services/backup-prune-plan.ts @@ -4,6 +4,7 @@ import { lstatSync } from "node:fs"; import { AnchoredDirectory, type FileIdentity, identity } from "../build-artifacts/filesystem"; import { canonicalJson } from "../ops/canonical-json"; import { backupRetentionExpiry } from "./backup-retention"; +import { isBackupStoreLockEntry } from "./backup-retention-store"; export interface BackupPruneCandidate { backupId: string; @@ -31,7 +32,7 @@ export function backupPruneInventory(infraDir: string, retentionDays: number, at if (root) { records.push({ root: identity(root.stat) }); for (const name of root.listBounded(129).sort()) { - if (name === ".backup-store.lock") continue; + if (isBackupStoreLockEntry(name)) continue; if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(name)) throw Error("Backup inventory is unsafe"); const directory = root.directory(name); diff --git a/packages/core/src/services/backup-retention-store.ts b/packages/core/src/services/backup-retention-store.ts index 4f865564d..539685a7d 100644 --- a/packages/core/src/services/backup-retention-store.ts +++ b/packages/core/src/services/backup-retention-store.ts @@ -1,12 +1,38 @@ -/** Node-only migration of portable expiry ceilings under the existing backup lock. */ +/** Node-only migration of portable expiry ceilings under the backup-store lock. */ import { join } from "node:path"; import { AnchoredDirectory, sameInode } from "../build-artifacts/filesystem"; import { backupRetentionExpiry } from "./backup-retention"; -import { acquireStoreLock } from "./release-store-lock"; +import { withRecoverableLock } from "./recoverable-lock"; const namePattern = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; -/** Existing backup effects allow30 minutes; retain exclusion through cleanup. */ -export const BACKUP_STORE_LOCK_TTL_MS = 35 * 60000; +const BACKUP_STORE_LOCK_FILE = ".backup-store.lock"; + +/** The lock every backup create, restore, delete, prune and retention change holds. */ +export function backupStoreLockPath(infraDir: string): string { + return join(infraDir, "backups", BACKUP_STORE_LOCK_FILE); +} + +/** Whether a name in the backups directory is the store lock or one of its reclaim files. */ +export function isBackupStoreLockEntry(name: string): boolean { + return name.startsWith(BACKUP_STORE_LOCK_FILE); +} + +/** + * Runs `run` holding the backup-store lock. It is held across a whole backup + * effect, so a second one waits up to a minute and is then refused as busy. A + * holder that died is reclaimed under the recoverable lock's rules. + */ +export function withBackupStoreLock( + infraDir: string, + run: () => Promise | T, + signal?: AbortSignal, +): Promise { + return withRecoverableLock( + backupStoreLockPath(infraDir), + { purpose: "backup-store", waitMs: 60_000, ...(signal ? { signal } : {}) }, + run, + ); +} export function retainBackupExpiryCeilings(infraDir: string, retentionDays: number): void { backupRetentionExpiry(new Date(0).toISOString(), retentionDays); const infra = new AnchoredDirectory(infraDir); @@ -19,7 +45,7 @@ export function retainBackupExpiryCeilings(infraDir: string, retentionDays: numb throw error; } for (const name of root.listBounded(129)) { - if (name === ".backup-store.lock") continue; + if (isBackupStoreLockEntry(name)) continue; if (!namePattern.test(name)) throw Error("Backup retention inventory is unsafe"); const directory = root.directory(name); try { @@ -94,24 +120,15 @@ export async function withBackupRetentionPolicyChange( throw error; } // Use the actual store path while retaining its opened identity. Both remain - // checked before and after the existing lock's critical section. - const lock = await acquireStoreLock( - join(infraDir, "backups"), - ".backup-store.lock", - {}, - Date.now, - false, - BACKUP_STORE_LOCK_TTL_MS, - ); - try { - root.check(); + // checked before and after the lock's critical section. + const opened = root; + return await withBackupStoreLock(infraDir, async () => { + opened.check(); retainBackupExpiryCeilings(infraDir, Math.min(oldDays, newDays)); const value = await run(); - root.check(); + opened.check(); return value; - } finally { - lock.release(); - } + }); } finally { root?.close(); infra.close(); diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index a15058db2..73b67d31d 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -7,7 +7,11 @@ export { export { prepareAgentStores } from "./agent-stores"; export { type BackupPruneCandidate, backupPruneInventory } from "./backup-prune-plan"; export { backupRetentionExpiry } from "./backup-retention"; -export { BACKUP_STORE_LOCK_TTL_MS, retainBackupExpiryCeilings } from "./backup-retention-store"; +export { + backupStoreLockPath, + retainBackupExpiryCeilings, + withBackupStoreLock, +} from "./backup-retention-store"; export { type BuiltInManifestSource, type BuiltInServiceManifestSource, @@ -325,7 +329,6 @@ export { } from "./release-state"; export { acquireReleaseStoreLock, - acquireStoreLock, assertReleaseStoreLock, RELEASE_STORE_LOCK_NAME, type ReleaseStoreLock, From d218184bfc7dd47eb02276e9503e1bdc3063fe57 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 22:57:11 +0100 Subject: [PATCH 14/42] feat(control): pin release images in the configuration generation and add control rollback A configuration generation now renders the release image pins into its own compose file and keeps the release manifest beside it, so the generation names the release it runs. A render that names no release keeps the applied generation's pins; a release update's maintenance commits the generation with the release it froze, and the self-update helper only verifies the images and recreates the services. Clearing the selected pin, from the admin route or compose release --clear, renders the applied configuration again without pins. The release overlay, its plumbing through the stack Compose arguments, the release store's pointers and selection receipts and the release-store lock are gone. The agent keeps a cache of the manifests it checked, named by digest; the newest is the release an update can apply. The generation also records its domain, so the agent can render it again, and the configuration directory records the last pointer move, which names the previous generation. openmapx control rollback renders the previous generation's configuration and release again and recreates the services in the self-update order. It refuses when the database's newest applied migration is newer than the target release's migration head, which CI now writes into the release manifest from the API's migration journal, and it says that rolling back also reverts the configuration made since. Without --yes it only prints the plan. The producer and build metadata of a manifest are display metadata, and two operator messages that still named the removed services recover-update now point to openmapx control. --- .github/workflows/docker.yml | 14 + .../coordinator-lifecycle-postgres.test.ts | 43 +- .../data-configuration-fence-postgres.test.ts | 7 +- ...orm-configuration-runtime-postgres.test.ts | 84 +-- .../retained-maintenance-job-postgres.test.ts | 74 +-- .../scripts/verify-traffic-maintenance.ts | 2 +- .../src/__fixtures__/configuration-runtime.ts | 1 - .../owned-compose-path-runtime.ts | 5 +- .../src/__fixtures__/owned-service-host.ts | 1 - .../src/administrative-runtime.test.ts | 336 +++--------- apps/ops-agent/src/administrative-runtime.ts | 182 ++----- .../src/artifact-transfer/import-runtime.ts | 58 +- apps/ops-agent/src/bootstrap.test.ts | 15 +- apps/ops-agent/src/bootstrap.ts | 4 +- .../src/configuration-handoff-application.ts | 250 +++------ .../src/configuration-handoff-store.ts | 14 +- .../src/configuration-handoff.test.ts | 8 +- apps/ops-agent/src/configuration-handoff.ts | 11 - ...tion-maintenance-docker-inspection.test.ts | 30 -- ...iguration-maintenance-docker-inspection.ts | 498 ++++++++---------- .../src/configuration-maintenance-store.ts | 76 ++- .../src/configuration-release-preflight.ts | 94 ++-- .../src/configuration-release.test.ts | 109 ++-- apps/ops-agent/src/configuration-release.ts | 2 +- .../src/data-transit-resource-source.test.ts | 1 - apps/ops-agent/src/docker-runtime.test.ts | 44 +- apps/ops-agent/src/docker-runtime.ts | 73 ++- apps/ops-agent/src/image-stage-runtime.ts | 194 ++++--- apps/ops-agent/src/owned-services.test.ts | 5 +- apps/ops-agent/src/owned-services.ts | 13 +- apps/ops-agent/src/production-runtime.test.ts | 31 +- apps/ops-agent/src/production-runtime.ts | 55 +- apps/ops-agent/src/server.ts | 2 +- apps/ops-agent/src/service-pull-runtime.ts | 359 ++++++------- .../test-fixtures/data-owned-http-process.ts | 1 - .../src/traffic-target-ownership.test.ts | 1 - .../src/transit-credentials-runtime.test.ts | 1 - apps/ops-agent/src/trusted-config-runtime.ts | 23 +- .../cli/__tests__/cli-build-artifacts.test.ts | 20 +- .../__tests__/cli-control-rollback.test.ts | 254 +++++++++ .../cli/__tests__/cli-release-check.test.ts | 97 ++-- .../__tests__/cli-release-coordinator.test.ts | 224 ++++---- packages/cli/__tests__/cli-release.test.ts | 419 +++------------ .../__tests__/cli-service-lifecycle.test.ts | 59 +-- .../cli/__tests__/cli-services-update.test.ts | 62 +-- .../cli/__tests__/cli-system-update.test.ts | 13 +- packages/cli/src/commands/compose.ts | 104 ++-- packages/cli/src/commands/control.ts | 68 ++- packages/cli/src/lib/build-artifacts.ts | 5 +- packages/cli/src/lib/motis-data.ts | 18 +- packages/cli/src/lib/release-service-files.ts | 25 +- packages/cli/src/lib/release.ts | 308 +++-------- packages/cli/src/lib/rollback.ts | 88 ++++ packages/cli/src/lib/service-file-report.ts | 4 +- packages/cli/src/lib/service-lifecycle.ts | 46 +- packages/cli/src/lib/system-update-receipt.ts | 2 +- packages/cli/src/lib/system-update.ts | 16 +- .../configuration-handoff-records.test.ts | 11 +- .../configuration-handoff-records.ts | 38 +- ...configuration-maintenance-final-records.ts | 14 +- .../configuration-source-witness.test.ts | 59 --- .../configuration-source-witness.ts | 83 +-- .../build-artifacts/data-update-sources.ts | 10 +- .../fixed-configuration-maintenance.test.ts | 2 - .../src/build-artifacts/target-lease.test.ts | 5 +- packages/core/src/repo-paths.ts | 12 +- .../__fixtures__/compose-path-host.ts | 16 +- .../__tests__/checked-release.test.ts | 192 ------- .../configuration-generation.test.ts | 113 +++- .../__tests__/deployment-user.test.ts | 1 - .../services/__tests__/infra-state.test.ts | 26 - .../services/__tests__/release-cache.test.ts | 117 ++++ .../__tests__/release-manifest.test.ts | 121 +---- .../release-store-lock-acquisition.test.ts | 62 --- .../__tests__/release-store-lock.test.ts | 112 ---- .../services/checked-configuration-release.ts | 79 --- packages/core/src/services/checked-release.ts | 189 ------- .../core/src/services/compose-renderer.ts | 17 +- .../src/services/configuration-generation.ts | 158 +++++- packages/core/src/services/deployment.ts | 30 -- packages/core/src/services/index.ts | 60 +-- packages/core/src/services/release-cache.ts | 244 +++++++++ .../src/services/release-image-selection.ts | 112 +--- .../core/src/services/release-manifest.ts | 206 ++------ .../src/services/release-selection-clear.ts | 111 ---- packages/core/src/services/release-state.ts | 265 ---------- .../core/src/services/release-store-lock.ts | 231 -------- .../core/src/services/self-update-order.ts | 20 + scripts/docker-release-contract.test.ts | 13 +- scripts/product-release.test.ts | 14 +- 90 files changed, 2618 insertions(+), 4678 deletions(-) create mode 100644 packages/cli/__tests__/cli-control-rollback.test.ts create mode 100644 packages/cli/src/lib/rollback.ts delete mode 100644 packages/core/src/services/__tests__/checked-release.test.ts create mode 100644 packages/core/src/services/__tests__/release-cache.test.ts delete mode 100644 packages/core/src/services/__tests__/release-store-lock-acquisition.test.ts delete mode 100644 packages/core/src/services/__tests__/release-store-lock.test.ts delete mode 100644 packages/core/src/services/checked-configuration-release.ts delete mode 100644 packages/core/src/services/checked-release.ts create mode 100644 packages/core/src/services/release-cache.ts delete mode 100644 packages/core/src/services/release-selection-clear.ts delete mode 100644 packages/core/src/services/release-state.ts delete mode 100644 packages/core/src/services/release-store-lock.ts create mode 100644 packages/core/src/services/self-update-order.ts diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3d0b2b292..8bb39f360 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -305,6 +305,15 @@ jobs: with: pattern: docker-digest-*-${{ github.run_id }} path: digests + # Rollback compares the database with the release's newest migration. + - name: Check out the release's migration journal + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.gate.outputs.sha }} + path: source + persist-credentials: false + sparse-checkout: apps/api/src/db/migrations/meta/_journal.json + sparse-checkout-cone-mode: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 @@ -395,6 +404,11 @@ jobs: !["push", "schedule"].includes(process.env.GITHUB_EVENT_NAME)) throw new Error("Invalid trusted producer identity"); manifest.producer = {repository: process.env.GITHUB_REPOSITORY, workflowPath: ".github/workflows/ci.yml", sourceRevision: process.env.RELEASE_SHA, runId, runAttempt, event: process.env.GITHUB_EVENT_NAME}; manifest.buildMetadata = plan.buildMetadata; + const journal = JSON.parse(readFileSync("source/apps/api/src/db/migrations/meta/_journal.json", "utf8")); + const head = journal.entries?.at(-1); + if (!head || !/^\d{4}_[a-z0-9_]{1,120}$/.test(head.tag) || !Number.isSafeInteger(head.when) || head.when < 1) + throw new Error("The migration journal has no valid newest entry"); + manifest.databaseMigrationHead = {tag: head.tag, when: head.when}; writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`); NODE diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index 3e3684ee8..c7878bf48 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -229,7 +229,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }, } : {}), - ...(test.task.name.includes("3072-byte baseline") ? { nestedRootSegments: 9 } : {}), + ...(test.task.name.includes("3072-byte baseline") ? { nestedRootSegments: 20 } : {}), ...(references ? { imageReferences: { motis: references.sibling, "data-manager": references.target } } : {}), @@ -266,35 +266,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( infraDir: host.infraDir, serviceIds: host.loaded.map((s) => s.manifest.id), }); - if (test.task.name.includes("3072-byte baseline")) { - const images = Object.fromEntries( - services.RELEASE_IMAGE_NAMES.map((name) => [ - name, - `example/${name}@sha256:${"b".repeat(64)}`, - ]), - ); - writeFileSync( - host.stack.composeReleasePath, - JSON.stringify({ - "x-openmapx-release": { release: "task012-baseline-budget", images }, - services: { - "app-api": { image: images.api }, - "app-web": { image: images.web }, - "ops-agent": { image: images["ops-agent"] }, - "data-manager": { - image: images["data-manager"], - environment: { [services.TRANSITOUS_TOOLS_IMAGE_ENV]: images["transitous-tools"] }, - }, - "transitous-runner": { image: images["transitous-runner"] }, - }, - }), - { mode: 0o600 }, - ); - host.row.Labels["com.docker.compose.project.config_files"] = [ - host.firstCompose, - host.stack.composeReleasePath, - ].join(","); - } const old = { ...host.row, Id: "c".repeat(64), @@ -1189,7 +1160,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const { productionReleaseEffects } = await import( "../../../../ops-agent/src/production-runtime" ), - directory = services.releaseStatePaths(host.infraDir).store, + directory = services.releaseCacheDirectory(host.infraDir), manifest = services.parseReleaseManifest( JSON.stringify({ schemaVersion: 1, @@ -1203,11 +1174,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ...releaseProvenance(), }), ); - writeFileSync( - join(directory, "interlock-release.json"), - services.canonicalReleaseManifest(manifest), - { mode: 0o600 }, - ); + services.cacheReleaseManifest(host.infraDir, manifest); const effects = productionReleaseEffects(host.runtime), context = { signal: new AbortController().signal, emitLog() {} }, before = services.currentConfigurationGeneration(host.infraDir), @@ -1216,7 +1183,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( () => null, (error) => error, ); - expect(existsSync(host.stack.composeReleasePath)).toBe(false); + expect(services.readAppliedRelease(host.infraDir)).toBeNull(); expect(services.currentConfigurationGeneration(host.infraDir)).toBe(before); expect(readdirSync(directory).sort()).toEqual(originalFiles); expect(readFileSync(pullPath)).toEqual(originalPull); @@ -2667,7 +2634,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( const owned = await host.owned.inspectOwnedServices(new AbortController().signal); const dmOwned = owned.find((value) => value.serviceId === "data-manager"); expect(dmOwned?.state).toBe("running"); - expect(dmOwned?.provenance.composeConfigFiles).toHaveLength(2); + expect(dmOwned?.provenance.composeConfigFiles).toHaveLength(1); const observed = new Promise((resolve) => { observePullReply = resolve; }); diff --git a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts index e9c7c0cab..7a1bd44e0 100644 --- a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts +++ b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts @@ -1031,12 +1031,7 @@ async function ownedFenceFixture( releaseId: manifest.release, manifestDigest: createHash("sha256").update(raw).digest("hex"), }; - mkdirSync(services.releaseStatePaths(host.infraDir).store, { recursive: true, mode: 0o700 }); - writeFileSync( - join(services.releaseStatePaths(host.infraDir).store, `${manifest.release}.json`), - raw, - { mode: 0o600 }, - ); + services.cacheReleaseManifest(host.infraDir, manifest); } const jobId = randomUUID(), actorId = options.guardedRecovery ? randomUUID() : jobId, diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 5dd94cdcd..4b08743d6 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -601,15 +601,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( releaseId: manifest.release, manifestDigest: createHash("sha256").update(raw).digest("hex"), }; - mkdirSync(services.releaseStatePaths(host.infraDir).store, { - recursive: true, - mode: 0o700, - }); - writeFileSync( - join(services.releaseStatePaths(host.infraDir).store, `${manifest.release}.json`), - raw, - { mode: 0o600 }, - ); + services.cacheReleaseManifest(host.infraDir, manifest); releaseReferences = Object.values(manifest.images) as string[]; // The images reach the host only through the pre-maintenance pull. host.setUnpulledImages(releaseReferences); @@ -653,12 +645,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( version === "launch-native-system-release-killed" ) { await services.withConfigurationGenerationWriter(host.infraDir, async () => { - const lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(host.infraDir).store, - {}, - { failIfBusy: true }, - ); - lock.release(); writersWereReleased = true; }); helperEffectEntered.resolve(); @@ -698,9 +684,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( Labels: { ...old.Labels, "com.docker.compose.project.working_dir": join(compose, ".."), - "com.docker.compose.project.config_files": checkedRelease - ? `${compose},${host.stack.composeReleasePath}` - : compose, + "com.docker.compose.project.config_files": compose, "com.docker.compose.config-hash": hash, }, }, @@ -1084,9 +1068,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( args.includes("--force-recreate"), ), ).toBe(false); - expect( - existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), - ).toBe(false); return; } if (version.endsWith("-api-before-save")) { @@ -1269,9 +1250,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(host.calls.some((args) => args[0] === "compose" && args.includes("up"))).toBe( false, ); - expect( - existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), - ).toBe(false); return; } if (version === "launch-native-system-release-agent-restart") { @@ -1336,18 +1314,12 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .filter((args) => args[0] === "compose" && args.includes("up")) .map((args) => args.at(-1)), ).toEqual(["data-manager"]); - expect( - existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), - ).toBe(false); return; } if (version === "launch-native-release-wrong-config") { await jobRunner.reconcile(); expect((await readJob()).status).toBe("running"); expect(dm.settlePosts).toBe(0); - expect( - existsSync(join(services.releaseStatePaths(host.infraDir).store, "current.json")), - ).toBe(false); expect( host.calls .filter((args) => args[0] === "compose" && args.includes("up")) @@ -1428,17 +1400,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ) .every((row) => row.Image === `sha256:${"e".repeat(64)}`), ).toBe(true); - expect( - JSON.parse( - readFileSync( - join(services.releaseStatePaths(host.infraDir).store, "current.json"), - "utf8", - ), - ), - ).toMatchObject({ - releaseId: checkedRelease.releaseId, - digest: checkedRelease.manifestDigest, - }); + expect(services.readAppliedRelease(host.infraDir)?.manifestDigest).toBe( + checkedRelease.manifestDigest, + ); } expect(dm.settlePosts).toBe(1); expect(host.calls.filter((args) => args[0] === "run")).toHaveLength(1); @@ -1692,44 +1656,6 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( throw error; } if (version === "launch-native-recovery") { - const { readOwnedConfigurationMaintenance } = await import( - "../../../../ops-agent/src/configuration-maintenance-store" - ), - { inspectMaintenanceDockerRuntime } = await import( - "../../../../ops-agent/src/configuration-maintenance-docker-inspection" - ), - { installationState } = await import( - "../../../../../packages/core/src/build-artifacts/control-authority" - ), - record = prospective.validatedConfigurationCandidateState(candidate).record, - receipt = await prospective.inspectConfigurationMaintenance({ - maintenanceId: record.maintenanceId!, - candidateId: candidate.candidateId, - }), - authority = restoreAdmittedControlExecution(installation, receipt.prepare), - owner = readOwnedConfigurationMaintenance(authority), - containers = await host.owned.inspectOwnedServices( - new AbortController().signal, - ), - lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(host.infraDir).store, - ); - const callsBeforeInspection = host.calls.length; - try { - await expect( - inspectMaintenanceDockerRuntime({ - stack: repoPaths(host.root), - execFile: host.execFile, - owner, - containers, - privateKey: installationState(installation).privateKey, - signal: new AbortController().signal, - }), - ).rejects.toThrow("Release store is busy"); - expect(host.calls).toHaveLength(callsBeforeInspection); - } finally { - lock.release(); - } execution.version = 2; ctx.checkpointResult = { ...ctx.checkpointResult, diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index 6fff9c2f0..404d0b67e 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -839,8 +839,14 @@ dbDescribe("retained finite maintenance requests", () => { ...releaseProvenance(), }), ); - const overlay = services.releaseStatePaths(host.infraDir).overlay; - services.writeReleaseComposeArtifacts(manifest, host.infraDir); + await services.commitConfigurationGeneration({ + infraDir: host.infraDir, + services: host.loaded, + input: (({ release: _none, ...input }) => input)( + services.readGenerationConfiguration(host.infraDir, host.initial.revisionId), + ), + release: manifest, + }); const tokenFile = join(host.root, "clear-api-token"); writeFileSync(tokenFile, host.token, { mode: 0o600 }); const agentUrl = await agent.listen({ host: "127.0.0.1", port: 0 }); @@ -872,8 +878,15 @@ dbDescribe("retained finite maintenance requests", () => { }, { timeout: 20000, interval: 100 }, ); - expect(existsSync(overlay)).toBe(false); - expect(services.currentConfigurationGeneration(host.infraDir)).toBe(pointer); + // Clearing renders the applied configuration again without the pin. + expect(services.readAppliedRelease(host.infraDir)).toBeNull(); + expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe(pointer); + expect( + services.readPreviousConfigurationGeneration( + host.infraDir, + services.currentConfigurationGeneration(host.infraDir)!, + ), + ).toBe(pointer); expect( journal.records().filter((row) => row.operation.kind === "release.clear"), ).toHaveLength(1); @@ -936,14 +949,33 @@ dbDescribe("retained finite maintenance requests", () => { newId = `sha256:${"e".repeat(64)}`, digest = `sha256:${"c".repeat(64)}`, stageCalls: string[][] = []; - const compose = services.configurationGenerationComposeFile( - host.infraDir, - host.initial.revisionId, - ), + // A pinned installation's generation names the release whose images are staged. + if (pinned) + await services.commitConfigurationGeneration({ + infraDir: host.infraDir, + services: host.loaded, + input: (({ release: _none, ...input }) => input)( + services.readGenerationConfiguration(host.infraDir, host.initial.revisionId), + ), + release: services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: "stage-pin", + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`, + ]), + ), + ...releaseProvenance(), + }), + ), + }); + const applied = services.currentConfigurationGeneration(host.infraDir)!; + const compose = services.configurationGenerationComposeFile(host.infraDir, applied), args = services.stackComposeArgs({ infraDir: host.infraDir, composePath: compose, - composeReleasePath: host.stack.composeReleasePath, }); const rows = await Promise.all( ids.map(async (id, index) => ({ @@ -969,25 +1001,6 @@ dbDescribe("retained finite maintenance requests", () => { })), ); host.setRows(rows); - const overlay = services.releaseStatePaths(host.infraDir).overlay; - if (pinned) - services.writeReleaseComposeArtifacts( - services.parseReleaseManifest( - JSON.stringify({ - schemaVersion: 1, - release: "stage-pin", - images: Object.fromEntries( - services.RELEASE_IMAGE_NAMES.map((name) => [ - name, - `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`, - ]), - ), - ...releaseProvenance(), - }), - ), - host.infraDir, - ); - const beforeOverlay = existsSync(overlay) ? readFileSync(overlay, "utf8") : null; const execFile: typeof host.execFile = async (file, argv, io) => { stageCalls.push([...argv]); if (argv[0] === "info") return { stdout: "linux/amd64", stderr: "" }; @@ -1104,10 +1117,7 @@ dbDescribe("retained finite maintenance requests", () => { ), }), ).toBe(invalidMetadata ? "failed" : "success"); - expect(services.currentConfigurationGeneration(host.infraDir)).toBe( - host.initial.revisionId, - ); - expect(existsSync(overlay) ? readFileSync(overlay, "utf8") : null).toBe(beforeOverlay); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(applied); const pulls = stageCalls.filter((argv) => argv[0] === "image" && argv[1] === "pull"); expect(pulls).toHaveLength(invalidMetadata ? 0 : 3); expect(pulls.every((argv) => /@sha256:[a-f0-9]{64}$/.test(argv[2] ?? ""))).toBe(true); diff --git a/apps/ops-agent/scripts/verify-traffic-maintenance.ts b/apps/ops-agent/scripts/verify-traffic-maintenance.ts index e973af705..cf1d6645e 100644 --- a/apps/ops-agent/scripts/verify-traffic-maintenance.ts +++ b/apps/ops-agent/scripts/verify-traffic-maintenance.ts @@ -61,7 +61,7 @@ assert.deepEqual(container.NetworkSettings.Ports["8002/tcp"], [ ]); const runtime = createDockerRuntime({ - stack: { infraDir: "/unused", composePath: "/unused", composeReleasePath: "/unused" }, + stack: { infraDir: "/unused", composePath: "/unused" }, fileExists: () => false, trafficDataRoot: TRAFFIC_DATA_ROOT, execFile: async (file, args, options) => { diff --git a/apps/ops-agent/src/__fixtures__/configuration-runtime.ts b/apps/ops-agent/src/__fixtures__/configuration-runtime.ts index 060de98f4..0ec3a155a 100644 --- a/apps/ops-agent/src/__fixtures__/configuration-runtime.ts +++ b/apps/ops-agent/src/__fixtures__/configuration-runtime.ts @@ -19,7 +19,6 @@ export function installConfigurationRuntimeFixture( const stack = { infraDir: options.infraDir, composePath: services.currentConfigurationFile(options.infraDir, services.GENERATED_COMPOSE_FILE), - composeReleasePath: services.releaseStatePaths(options.infraDir).overlay, }; const execFile = async (_file: string, args: readonly string[]) => { if (args[0] !== "container" || args[1] !== "ls") diff --git a/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts b/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts index b321018d9..13817028e 100644 --- a/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts +++ b/apps/ops-agent/src/__fixtures__/owned-compose-path-runtime.ts @@ -27,10 +27,7 @@ export async function ownedComposePathRuntime() { "com.docker.compose.project": services.STACK_PROJECT, "com.docker.compose.service": "alpha", "com.docker.compose.project.working_dir": dirname(host.firstCompose), - "com.docker.compose.project.config_files": [ - host.firstCompose, - realpathSync(host.stack.composeReleasePath), - ].join(","), + "com.docker.compose.project.config_files": host.firstCompose, }, }; const calls: string[][] = []; diff --git a/apps/ops-agent/src/__fixtures__/owned-service-host.ts b/apps/ops-agent/src/__fixtures__/owned-service-host.ts index f71a4fdf8..b40d6ca7b 100644 --- a/apps/ops-agent/src/__fixtures__/owned-service-host.ts +++ b/apps/ops-agent/src/__fixtures__/owned-service-host.ts @@ -110,7 +110,6 @@ export async function ownedServiceHost(options: { configurationReleaseMetadata?: const stack = { infraDir, composePath, - composeReleasePath: services.releaseStatePaths(infraDir).overlay, }; const imageId = `sha256:${"b".repeat(64)}`; const row = { diff --git a/apps/ops-agent/src/administrative-runtime.test.ts b/apps/ops-agent/src/administrative-runtime.test.ts index 694bbbb4c..b894d3174 100644 --- a/apps/ops-agent/src/administrative-runtime.test.ts +++ b/apps/ops-agent/src/administrative-runtime.test.ts @@ -29,8 +29,6 @@ import { import { createUnavailableRuntime, dispatchOpsOperation, OpsExecutionFailure } from "./runtime"; const temporaryRoots: string[] = []; -const releaseState = (rootDir: string) => - services.releaseStatePaths(join(rootDir, "infra", "docker")); function temporaryRoot(): string { const root = mkdtempSync(join(tmpdir(), "openmapx-admin-runtime-")); @@ -849,18 +847,21 @@ async function appliedInspectionFixture(rootDir: string) { } describe("administrative release runtime", () => { - it("stores immutable canonical release IDs with exact-content idempotency and a bounded manifest", async () => { + it("caches each resolved manifest by its digest and reports the one checked last", async () => { const rootDir = temporaryRoot(); + const infraDir = join(rootDir, "infra", "docker"); const release = "immutable-release"; - const makeManifest = (digit: string) => ({ + const makeManifest = (digit: string, runAttempt = 1) => ({ schemaVersion: 1 as const, release, images: Object.fromEntries( - ["api", "web", "data-manager", "ops-agent", "transitous-runner", "transitous-tools"].map( - (name) => [name, `ghcr.io/openmapx/${name}@sha256:${digit.repeat(64)}`], - ), + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${digit.repeat(64)}`, + ]), ), ...releaseProvenance(), + producer: { ...releaseProvenance().producer, runAttempt }, }); let manifestText = JSON.stringify(makeManifest("4")); const docker = async (args: readonly string[]) => { @@ -870,9 +871,7 @@ describe("administrative release runtime", () => { expect(args[1]).toBe(`sha256:${"2".repeat(64)}`); return "a".repeat(64); } - if (args[0] === "cp") { - writeFileSync(args.at(-1) as string, manifestText, { mode: 0o600 }); - } + if (args[0] === "cp") writeFileSync(args.at(-1) as string, manifestText, { mode: 0o600 }); return ""; }; const context = { @@ -880,256 +879,63 @@ describe("administrative release runtime", () => { signal: new AbortController().signal, emitLog: vi.fn(), }; - const first = createDefaultReleaseEffects(rootDir, { - runDocker: docker, - }); - await expect(first.resolve(context)).resolves.toBe(release); - await expect(first.resolve(context)).resolves.toBe(release); - manifestText = JSON.stringify(makeManifest("5")); - await expect(first.resolve(context)).rejects.toThrow("Release authority rejected"); - expect(readFileSync(join(releaseState(rootDir).store, `${release}.json`), "utf8")).toContain( - "4".repeat(64), - ); + const effects = createDefaultReleaseEffects(rootDir, { runDocker: docker }); + await expect(effects.resolve(context)).resolves.toBe(release); + await expect(effects.resolve(context)).resolves.toBe(release); + const first = services.parseReleaseManifest(manifestText); + expect(services.cachedReleases(infraDir, "ghcr.io/openmapx")).toHaveLength(1); + expect(inspectReleaseAuthority(rootDir, release)).toBe(true); + expect(inspectReleaseAuthority(rootDir, "other-release")).toBe(false); + // Another producer attempt of the same release id is another manifest. + manifestText = JSON.stringify(makeManifest("5", 2)); + await expect(effects.resolve(context)).resolves.toBe(release); + const second = services.parseReleaseManifest(manifestText); expect( - JSON.parse(readFileSync(join(releaseState(rootDir).store, `${release}.json`), "utf8")), - ).toMatchObject({ - images: { "ops-agent": `ghcr.io/openmapx/ops-agent@sha256:${"4".repeat(64)}` }, + services.cachedReleases(infraDir, "ghcr.io/openmapx").map((entry) => entry.manifestDigest), + ).toEqual([services.releaseManifestDigest(second), services.releaseManifestDigest(first)]); + await expect(effects.inspect()).resolves.toEqual({ + configuredSelector: "main", + availableReleaseId: release, + availableDigest: services.releaseManifestDigest(second), }); - manifestText = services.canonicalReleaseManifest( - services.parseReleaseManifest( - JSON.stringify({ - ...makeManifest("4"), - release: "oversized-release", - producer: { - repository: `${"x".repeat(32 * 1024)}/repository`, - workflowPath: ".github/workflows/ci.yml", - sourceRevision: "c".repeat(40), - runId: 1, - runAttempt: 1, - event: "push", - }, - }), - ), - ); - await expect(first.resolve(context)).rejects.toThrow(); - }); - - it("stores distinct canonical identities for two producer attempts of the same run", async () => { - const rootDir = temporaryRoot(); - let attempt = 1; - const sourceRevision = "a".repeat(40); - const manifest = () => ({ - schemaVersion: 1, - release: `${sourceRevision}-123-${attempt}`, - images: Object.fromEntries( - ["api", "web", "data-manager", "ops-agent", "transitous-runner", "transitous-tools"].map( - (name) => [name, `ghcr.io/openmapx/${name}@sha256:${String(attempt).repeat(64)}`], - ), - ), - ...releaseProvenance(), + manifestText = JSON.stringify({ + ...makeManifest("4"), + release: "oversized-release", producer: { - repository: "OpenMapX/openmapx", - workflowPath: ".github/workflows/ci.yml", - sourceRevision, - runId: 123, - runAttempt: attempt, - event: "push", - }, - }); - const runtime = createDefaultReleaseEffects(rootDir, { - runDocker: async (args) => { - if (args[0] === "pull") return `Digest: sha256:${String(attempt).repeat(64)}`; - if (args[0] === "image") return `sha256:${"e".repeat(64)}`; - if (args[0] === "create") return "a".repeat(64); - if (args[0] === "cp") - writeFileSync(args.at(-1) as string, JSON.stringify(manifest()), { mode: 0o644 }); - return ""; + ...releaseProvenance().producer, + repository: `${"x".repeat(32 * 1024)}/repository`, }, }); - const context = { - execution: testExecution(), - signal: new AbortController().signal, - emitLog: vi.fn(), - }; - expect(await runtime.resolve(context)).toBe(`${sourceRevision}-123-1`); - attempt = 2; - expect(await runtime.resolve(context)).toBe(`${sourceRevision}-123-2`); - for (const n of [1, 2]) - expect( - JSON.parse( - readFileSync( - join(releaseState(rootDir).store, `${sourceRevision}-123-${n}.json`), - "utf8", - ), - ).producer.runAttempt, - ).toBe(n); + await expect(effects.resolve(context)).rejects.toThrow(); }); - it("refuses store entries and an overlay its group or others could write", async () => { + it("refuses a release cache its group or others could reach", async () => { const rootDir = temporaryRoot(); - const { store, overlay } = releaseState(rootDir); - mkdirSync(store, { recursive: true, mode: 0o700 }); - const manifest = JSON.stringify({ - schemaVersion: 1, - release: "release-1", - images: Object.fromEntries( - services.RELEASE_IMAGE_NAMES.map((name) => [ - name, - `ghcr.io/openmapx/${name}@sha256:${"7".repeat(64)}`, - ]), - ), - ...releaseProvenance(), - }); - writeFileSync(join(store, "release-1.json"), manifest, { mode: 0o600 }); - chmodSync(join(store, "release-1.json"), 0o660); + const directory = services.releaseCacheDirectory(join(rootDir, "infra", "docker")); + mkdirSync(directory, { recursive: true }); + chmodSync(directory, 0o750); expect(inspectReleaseAuthority(rootDir, "release-1")).toBe(false); await expect(createDefaultReleaseEffects(rootDir).initialize?.()).rejects.toThrow( - "Release store contains an unsafe entry", + /Unsafe release cache/, ); - - chmodSync(join(store, "release-1.json"), 0o600); - expect(inspectReleaseAuthority(rootDir, "release-1")).toBe(true); - writeFileSync(overlay, "services: {}\n", { mode: 0o600 }); - chmodSync(overlay, 0o620); - const runDocker = vi.fn(async () => ""); - const effects = createDefaultReleaseEffects(rootDir, { runDocker }); - await expect( - effects.inspectSystem({ signal: new AbortController().signal, emitLog: vi.fn() }), - ).rejects.toThrow(/group or others/); - expect(readFileSync(overlay, "utf8")).toBe("services: {}\n"); }); - it("refuses release state directories of another user", () => { + it("refuses a release cache of another user", async () => { const rootDir = temporaryRoot(); - // Tests cannot chown: inject the uid the state must belong to instead. - const uid = (process.geteuid?.() ?? 0) + 1; - expect(() => createDefaultReleaseEffects(rootDir, { uid })).toThrow( - /Unsafe release state directory/, - ); - }); - - it("uses the durable latest pointer instead of lexical release-ID ordering", async () => { - const rootDir = temporaryRoot(); - const directory = releaseState(rootDir).store; - mkdirSync(directory, { recursive: true, mode: 0o700 }); - const makeManifest = (release: string, digit: string) => ({ - schemaVersion: 1 as const, - release, - images: Object.fromEntries( - ["api", "web", "data-manager", "ops-agent", "transitous-runner", "transitous-tools"].map( - (name) => [name, `ghcr.io/openmapx/${name}@sha256:${digit.repeat(64)}`], - ), - ), - ...releaseProvenance(), + mkdirSync(services.releaseCacheDirectory(join(rootDir, "infra", "docker")), { + recursive: true, + mode: 0o700, }); - const latest = makeManifest("a-new", "1"); - writeFileSync(join(directory, "z-old.json"), JSON.stringify(makeManifest("z-old", "2")), { - mode: 0o600, - }); - writeFileSync(join(directory, "a-new.json"), JSON.stringify(latest), { mode: 0o600 }); - const { createHash } = await import("node:crypto"); - writeFileSync( - join(directory, "latest.json"), - JSON.stringify({ - releaseId: "a-new", - digest: createHash("sha256").update(JSON.stringify(latest)).digest("hex"), - }), - { mode: 0o600 }, + // Tests cannot chown: inject the uid the cache must belong to instead. + const uid = (process.geteuid?.() ?? 0) + 1; + await expect(createDefaultReleaseEffects(rootDir, { uid }).initialize?.()).rejects.toThrow( + /Unsafe release cache/, ); - const effects = createDefaultReleaseEffects(rootDir); - await expect(effects.inspect()).resolves.toEqual({ - configuredSelector: "main", - availableReleaseId: "a-new", - }); - }); - - it("enforces release entry and aggregate bounds before registry reads", async () => { - const rootDir = temporaryRoot(); - const directory = releaseState(rootDir).store; - mkdirSync(directory, { recursive: true, mode: 0o700 }); - for (let index = 0; index < 65; index += 1) { - writeFileSync(join(directory, `release-${index}.json`), "{}", { mode: 0o600 }); - } - const effects = createDefaultReleaseEffects(rootDir); - await expect(effects.initialize?.()).rejects.toThrow("Release store limit exceeded"); - }); - - it("holds the entry bound when two resolutions race the same near-full store", async () => { - const rootDir = temporaryRoot(); - const directory = releaseState(rootDir).store; - mkdirSync(directory, { recursive: true, mode: 0o700 }); - const stored = (release: string) => - JSON.stringify({ - schemaVersion: 1, - release, - images: Object.fromEntries( - ["api", "web", "data-manager", "ops-agent", "transitous-runner", "transitous-tools"].map( - (name) => [name, `ghcr.io/openmapx/${name}@sha256:${"3".repeat(64)}`], - ), - ), - ...releaseProvenance(), - }); - for (let index = 0; index < 63; index += 1) { - writeFileSync(join(directory, `release-${index}.json`), stored(`release-${index}`), { - mode: 0o600, - }); - } - - // Both resolutions read the store, then both await Docker. Before the lock - // each published against its own stale pre-await snapshot of 63. - const resolveFor = (release: string) => { - let copied = false; - return createDefaultReleaseEffects(rootDir, { - runDocker: async (args) => { - const joined = args.join(" "); - if (joined.startsWith("pull")) return `Digest: sha256:${"1".repeat(64)}`; - if (joined.startsWith("image inspect")) return `sha256:${"2".repeat(64)}`; - if (joined.startsWith("create")) return "c".repeat(64); - if (joined.startsWith("cp")) { - const destination = args[2] as string; - writeFileSync(destination, stored(release), { mode: 0o644 }); - copied = true; - // Yield so the sibling resolution interleaves here. - await new Promise((resolve) => setTimeout(resolve, 5)); - return ""; - } - if (joined.startsWith("rm")) return ""; - if (!copied) throw new Error("unexpected docker call"); - return ""; - }, - }); - }; - - const results = await Promise.allSettled([ - resolveFor("release-a").resolve({ - execution: testExecution(), - signal: new AbortController().signal, - emitLog: vi.fn(), - }), - resolveFor("release-b").resolve({ - execution: testExecution(), - signal: new AbortController().signal, - emitLog: vi.fn(), - }), - ]); - - const stayed = readdirSync(directory).filter((name) => name.endsWith(".json")).length; - const fulfilled = results.filter((result) => result.status === "fulfilled").length; - // 63 stored + latest.json; exactly one new release may land. - expect(stayed).toBeLessThanOrEqual(64 + 2); - expect(fulfilled).toBeGreaterThanOrEqual(1); - for (const result of results) { - if (result.status === "rejected") { - expect(String(result.reason)).toMatch(/Release store limit exceeded|Release store is busy/); - } - } }); it("inspects old, current, missing, and partial core images from fixed Docker targets", async () => { const rootDir = temporaryRoot(); - const releaseDirectory = releaseState(rootDir).store; - mkdirSync(releaseDirectory, { recursive: true, mode: 0o700 }); await appliedInspectionFixture(rootDir); writeFileSync( join( @@ -1156,9 +962,10 @@ describe("administrative release runtime", () => { ), ...releaseProvenance(), }; - writeFileSync(join(releaseDirectory, `${releaseId}.json`), JSON.stringify(manifest), { - mode: 0o600, - }); + services.cacheReleaseManifest( + join(rootDir, "infra", "docker"), + services.parseReleaseManifest(JSON.stringify(manifest)), + ); const calls: string[][] = []; const effects = createDefaultReleaseEffects(rootDir, { runDocker: async (args) => { @@ -1209,8 +1016,6 @@ describe("administrative release runtime", () => { runDocker: (args: readonly string[]) => Promise, ) { const rootDir = temporaryRoot(); - const releaseDirectory = releaseState(rootDir).store; - mkdirSync(releaseDirectory, { recursive: true, mode: 0o700 }); await appliedInspectionFixture(rootDir); const composePath = join( rootDir, @@ -1223,27 +1028,28 @@ describe("administrative release runtime", () => { if (compose === "file") writeFileSync(composePath, "services:{}\n", { mode: 0o600 }); if (compose === "directory") mkdirSync(composePath, { mode: 0o700 }); const releaseId = "release-123"; - writeFileSync( - join(releaseDirectory, `${releaseId}.json`), - JSON.stringify({ - schemaVersion: 1, - release: releaseId, - images: Object.fromEntries( - [ - "api", - "web", - "data-manager", - "ops-agent", - "transitous-runner", - "transitous-tools", - ].map((name, index) => [ - name, - `ghcr.io/openmapx/${name}@sha256:${String(index + 1).repeat(64)}`, - ]), - ), - ...releaseProvenance(), - }), - { mode: 0o600 }, + services.cacheReleaseManifest( + join(rootDir, "infra", "docker"), + services.parseReleaseManifest( + JSON.stringify({ + schemaVersion: 1, + release: releaseId, + images: Object.fromEntries( + [ + "api", + "web", + "data-manager", + "ops-agent", + "transitous-runner", + "transitous-tools", + ].map((name, index) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${String(index + 1).repeat(64)}`, + ]), + ), + ...releaseProvenance(), + }), + ), ); return createDefaultReleaseEffects(rootDir, { runDocker }); } diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 0f29f2cc4..49bf9101d 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -58,8 +58,6 @@ import { type OpsRuntime, } from "./runtime"; -const { acquireReleaseStoreLock } = coreServices; - const MAX_BACKUP_MANIFEST_BYTES = 1024 * 1024; const MAX_BACKUP_SERVICES = 256; const MAX_BACKUP_VOLUMES = 4_096; @@ -76,9 +74,7 @@ const INSPECT_RELEASE_SERVICE_IDS = [ "ops-agent", "transitous-runner", ] as const; -const MAX_RELEASE_MANIFEST_BYTES = coreServices.RELEASE_STORE_LIMITS.manifestBytes; -const MAX_RELEASE_STATE_BYTES = coreServices.RELEASE_STORE_LIMITS.stateBytes; -const MAX_RELEASE_SELECTIONS_BYTES = coreServices.RELEASE_STORE_LIMITS.selectionsBytes; +const MAX_RELEASE_MANIFEST_BYTES = 32 * 1024; const BACKUP_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const SERVICE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; const _REGION_ID = /^[a-z0-9][a-z0-9_-]*(?:\/[a-z0-9][a-z0-9_-]*)*$/; @@ -510,39 +506,26 @@ function infraDirectory(rootDir: string): string { return join(rootDir, "infra", "docker"); } -/** The agent's release store, `state/ops-agent/releases/store`. */ -function releaseDirectory(rootDir: string): string { - return coreServices.releaseStatePaths(infraDirectory(rootDir)).store; -} - -function releaseManifestPath(rootDir: string, releaseId: string): string { - if (!coreServices.RELEASE_ID_PATTERN.test(releaseId)) { - throw new Error("Release authority rejected"); - } - return join(releaseDirectory(rootDir), `${releaseId}.json`); -} - function stateOwner(uid?: number): number | undefined { return uid ?? process.geteuid?.(); } +/** Whether the agent has checked a release with this id, for a release operation's resource. */ export function inspectReleaseAuthority(rootDir: string, releaseId: string): boolean { try { - const manifest = coreServices.parseReleaseManifest( - strictReadFile(releaseManifestPath(rootDir, releaseId), MAX_RELEASE_MANIFEST_BYTES, { - owner: stateOwner(), - }), + const infraDir = infraDirectory(rootDir); + const selection = coreServices.readAppliedReleaseSelection(infraDir); + return ( + selection.kind !== "local" && + coreServices + .cachedReleases(infraDir, selection.imagePrefix) + .some(({ manifest }) => manifest.release === releaseId) ); - return manifest.release === releaseId; } catch { return false; } } -function releaseDigest(contents: string): string { - return createHash("sha256").update(contents).digest("hex"); -} - function withBackupStoreLock(rootDir: string, run: () => Promise): Promise { mkdirSync(backupRoot(rootDir), { recursive: true, mode: 0o700 }); return coreServices.withBackupStoreLock(infraDirectory(rootDir), run); @@ -587,7 +570,7 @@ function inspectBackupManifestDigest(rootDir: string, backupId: string): string } } -function releaseStoreBindings(rootDir: string, uid: number | undefined) { +function stackBindings(rootDir: string) { const infraDir = infraDirectory(rootDir); const repositoryPaths: coreServices.StackPaths = { infraDir, @@ -595,18 +578,10 @@ function releaseStoreBindings(rootDir: string, uid: number | undefined) { infraDir, coreServices.GENERATED_COMPOSE_FILE, ), - composeReleasePath: coreServices.releaseStatePaths(infraDir).overlay, }; - const overlay = repositoryPaths.composeReleasePath; - // Owner and mode are checked as for the store: an overlay someone else could - // write is refused, not read and not passed to Compose. - const readOverlay = () => coreServices.readReleaseOverlay(infraDir, { uid }); return { repositoryPaths, - stackArgs: () => - coreServices.stackComposeArgs(repositoryPaths, (path) => - path === overlay ? readOverlay() !== null : isRegularComposeFile(path), - ), + stackArgs: () => coreServices.stackComposeArgs(repositoryPaths, isRegularComposeFile), }; } @@ -616,31 +591,7 @@ export function createDefaultReleaseEffects( ): AdministrativeReleaseEffects { const infraDir = infraDirectory(rootDir); const uid = stateOwner(runtimeOptions.uid); - // `releases/`, `store/` and `evidence/`: created owner-only when missing, - // refused when they are anyone else's. - const { store: directory } = coreServices.ensureReleaseStateDirectories(infraDir, { uid }); - const paths = { - state: join(directory, "current.json"), - latest: join(directory, "latest.json"), - selections: join(directory, "selections.json"), - }; - const { repositoryPaths, stackArgs } = releaseStoreBindings(rootDir, uid); - const manifestNames = () => coreServices.listReleaseStoreManifests(directory, { uid }); - const loadManifest = (releaseId: string) => { - manifestNames(); - const raw = strictReadFile( - releaseManifestPath(rootDir, releaseId), - MAX_RELEASE_MANIFEST_BYTES, - { - owner: uid, - }, - ); - const manifest = coreServices.parseReleaseManifest(raw); - if (manifest.release !== releaseId || raw !== coreServices.canonicalReleaseManifest(manifest)) { - throw new Error("Release authority rejected"); - } - return manifest; - }; + const { repositoryPaths, stackArgs } = stackBindings(rootDir); const runDocker = runtimeOptions.runDocker ?? (async (args: readonly string[], context: FixedCliOptions) => { @@ -653,72 +604,35 @@ export function createDefaultReleaseEffects( emitLines(context.emitLog, "stderr", result.stderr); return args[0] === "pull" ? `${result.stdout}\n${result.stderr}` : result.stdout; }); - const readReleasePointer = ( - path: string, - optional: boolean, - ): { releaseId?: string; digest?: string } => { - try { - const stat = lstatSync(path); - if (coreServices.releaseStateFileProblem(stat, MAX_RELEASE_STATE_BYTES, { uid }) !== null) { - throw new Error("Release authority rejected"); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT" && optional) return {}; - throw new Error("Release authority rejected"); - } - const value = coreServices.parseReleasePointer( - strictReadFile(path, MAX_RELEASE_STATE_BYTES, { owner: uid }), - path === paths.latest, - ); - const manifest = loadManifest(value.releaseId); - const digest = releaseDigest(coreServices.canonicalReleaseManifest(manifest)); - if (value.digest !== undefined && digest !== value.digest) { - throw new Error("Release authority rejected"); - } - return { releaseId: value.releaseId, digest }; - }; - const readState = () => readReleasePointer(paths.state, true); - const configurationIdentity = () => - releaseDigest( - JSON.stringify(coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker"))), - ); - const selectionReceipts = (): coreServices.ReleaseSelectionReceipts => { - if (!existsSync(paths.selections)) return {}; - return coreServices.parseReleaseSelectionReceipts( - strictReadFile(paths.selections, MAX_RELEASE_SELECTIONS_BYTES, { owner: uid }), - ); + /** The release last checked for the configured channel, which an update can apply. */ + const available = () => { + const configured = coreServices.readAppliedReleaseSelection(infraDir); + return configured.kind === "local" + ? undefined + : coreServices.cachedReleases(infraDir, configured.imagePrefix, { uid })[0]; }; const inspectRelease = async (): Promise> => { - const configured = coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker")); - const state = readState(); - const latest = readReleasePointer(paths.latest, true); - const candidates = manifestNames(); - const availableReleaseId = - latest.releaseId ?? (candidates.length === 1 ? candidates[0] : undefined); - const current = state.releaseId ? loadManifest(state.releaseId) : undefined; - const available = availableReleaseId ? loadManifest(availableReleaseId) : undefined; - const receipt = availableReleaseId ? selectionReceipts()[availableReleaseId] : undefined; - let selectedReleaseId: string | undefined; - const overlay = coreServices.releaseStatePaths(infraDir).overlay; - try { - const stats = lstatSync(overlay); - if (!stats.isFile() || stats.isSymbolicLink() || stats.nlink !== 1 || stats.size > 64 * 1024) - throw Error("Unsafe release selection"); - selectedReleaseId = - coreServices.parseReleaseComposeSelection(strictReadFile(overlay, 64 * 1024)).release ?? - undefined; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } + const configured = coreServices.readAppliedReleaseSelection(infraDir); + // The applied generation's pins are both the selected and the current release. + const applied = coreServices.readAppliedRelease(infraDir)?.manifest; + const checked = available(); return { configuredSelector: configured.kind === "local" ? "local" : configured.selector, - ...(selectedReleaseId ? { selectedReleaseId } : {}), - ...(state.releaseId ? { currentReleaseId: state.releaseId } : {}), - ...(current?.product ? { currentVersion: current.product.version } : {}), - ...(availableReleaseId ? { availableReleaseId } : {}), - ...(available?.product ? { availableVersion: available.product.version } : {}), - ...(receipt && receipt.configuration === configurationIdentity() - ? { availableDigest: receipt.digest } + ...(applied + ? { + selectedReleaseId: applied.release, + currentReleaseId: applied.release, + ...(applied.product ? { currentVersion: applied.product.version } : {}), + } + : {}), + ...(checked + ? { + availableReleaseId: checked.manifest.release, + ...(checked.manifest.product + ? { availableVersion: checked.manifest.product.version } + : {}), + availableDigest: checked.manifestDigest, + } : {}), }; }; @@ -728,7 +642,6 @@ export function createDefaultReleaseEffects( await inspectRelease(); }, resolve: async (context) => { - manifestNames(); const selection = coreServices.readAppliedReleaseSelection(join(rootDir, "infra", "docker")); if (selection.kind === "local") throw new coreServices.ReleaseChannelDisabledError(); const image = selection.manifestImage; @@ -764,22 +677,7 @@ export function createDefaultReleaseEffects( const raw = strictReadFile(temporaryManifest, MAX_RELEASE_MANIFEST_BYTES); const manifest = coreServices.parseReleaseManifest(raw, selection.imagePrefix); coreServices.validateResolvedSelection(selection, manifest, ociDigest); - // Re-read the store under the lock. A snapshot taken before the Docker - // pull/create/cp awaits is stale, so two concurrent resolutions could - // each observe the same sub-limit count and both publish past it. - const lock = await acquireReleaseStoreLock(directory); - try { - coreServices.cacheCheckedReleaseManifest({ - infraDir, - manifest, - ociDigest, - configurationIdentity: configurationIdentity(), - lock, - uid, - }); - } finally { - lock.release(); - } + coreServices.cacheReleaseManifest(infraDir, manifest, { uid }); return manifest.release; } finally { await runDocker(["rm", "-f", containerId], context).catch(() => undefined); @@ -791,9 +689,7 @@ export function createDefaultReleaseEffects( }, inspectSystem: async (context) => { const release = await inspectRelease(); - const manifest = release.availableReleaseId - ? loadManifest(release.availableReleaseId) - : undefined; + const manifest = available()?.manifest; let dockerReachable = false; try { await runDocker(["info", "--format", "{{.ServerVersion}}"], context); diff --git a/apps/ops-agent/src/artifact-transfer/import-runtime.ts b/apps/ops-agent/src/artifact-transfer/import-runtime.ts index 8893c1fb8..cbd395149 100644 --- a/apps/ops-agent/src/artifact-transfer/import-runtime.ts +++ b/apps/ops-agent/src/artifact-transfer/import-runtime.ts @@ -1,8 +1,7 @@ /** Fixed prepared-artifact consumer. Filesystem paths and Docker acquisition stay agent-local. */ import { existsSync } from "node:fs"; -import { join, relative, sep } from "node:path"; +import { join } from "node:path"; import { - AnchoredDirectory, buildArtifactSpec, controlDigest, executeOwnedGenerationHardlinks, @@ -359,53 +358,16 @@ export function installPreparedImportRuntime( expectedFingerprint: plan.targetFingerprint, materializeMotis, signal: context.signal, + // The release a generation pins is part of its revision, which the + // compatibility fingerprint names. beforePublication: async () => { - const infra = new AnchoredDirectory(source.infraDir); - const releaseParents: AnchoredDirectory[] = []; - let releases = infra; - try { - for (const part of relative( - source.infraDir, - services.releaseStatePaths(source.infraDir).store, - ).split(sep)) { - releases = releases.directory(part, true, true); - releaseParents.push(releases); - } - } catch (error) { - for (const parent of releaseParents.reverse()) parent.close(); - infra.close(); - throw error; - } - let lock: Awaited> | undefined; - try { - lock = await services.acquireReleaseStoreLock( - releases.anchor, - {}, - { failIfBusy: true }, - ); - const current = await snapshot(source, operation, context.signal); - if ( - current.preview.blockers.length || - current.preview.fingerprint !== plan.preview.fingerprint - ) - throw Error("Prepared compatibility changed"); - return () => { - try { - lock?.release(); - } finally { - for (const parent of releaseParents.reverse()) parent.close(); - infra.close(); - } - }; - } catch (error) { - try { - lock?.release(); - } finally { - for (const parent of releaseParents.reverse()) parent.close(); - infra.close(); - } - throw error; - } + const current = await snapshot(source, operation, context.signal); + if ( + current.preview.blockers.length || + current.preview.fingerprint !== plan.preview.fingerprint + ) + throw Error("Prepared compatibility changed"); + return () => {}; }, }).catch((cause) => { const failure = new OpsExecutionFailure("recovery_required", {}); diff --git a/apps/ops-agent/src/bootstrap.test.ts b/apps/ops-agent/src/bootstrap.test.ts index d89885db2..50ebc7d66 100644 --- a/apps/ops-agent/src/bootstrap.test.ts +++ b/apps/ops-agent/src/bootstrap.test.ts @@ -1,5 +1,5 @@ import { fork } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -93,13 +93,11 @@ it.each(["SIGTERM", "SIGINT"] as const)( 60000, ); -it("production bootstrap retains health and observation when the release store is unreadable", async () => { +it("production bootstrap retains health and observation when the release cache is unsafe", async () => { const root = mkdtempSync(join(tmpdir(), "openmapx-bootstrap-")); - const directory = services.releaseStatePaths(join(root, "infra/docker")).store; - mkdirSync(directory, { recursive: true, mode: 0o700 }); - writeFileSync(join(directory, "unexpected entry"), "{}", { mode: 0o600 }); - const overlay = services.releaseStatePaths(join(root, "infra/docker")).overlay; - writeFileSync(overlay, "operator-state\n"); + const directory = services.releaseCacheDirectory(join(root, "infra/docker")); + mkdirSync(directory, { recursive: true }); + chmodSync(directory, 0o755); const dispatch = vi.fn(async () => ({ reachable: true, version: "test" })); const token = Buffer.alloc(32, 1).toString("base64url"); const retained = persistedJob( @@ -193,10 +191,9 @@ it("production bootstrap retains health and observation when the release store i ).statusCode, ).toBe(200); expect(ready).not.toHaveBeenCalled(); - expect(readFileSync(overlay, "utf8")).toBe("operator-state\n"); // The operator is told to repair and restart; a retry cannot succeed. const written = stderr.mock.calls.map(([chunk]) => String(chunk)).join(""); - expect(written).toContain("release store"); + expect(written).toContain("release cache"); expect(written).toContain("restart the agent"); expect(written).not.toMatch(/retry/i); } finally { diff --git a/apps/ops-agent/src/bootstrap.ts b/apps/ops-agent/src/bootstrap.ts index 8857f496f..d76711a18 100644 --- a/apps/ops-agent/src/bootstrap.ts +++ b/apps/ops-agent/src/bootstrap.ts @@ -52,9 +52,9 @@ export async function startOpsAgentServer( await releaseEffects.initialize?.(); ready = true; } catch { - // An unreadable release store keeps the agent to observation until an operator repairs it. + // An unreadable release cache keeps the agent to observation until an operator repairs it. process.stderr.write( - "ops-agent: the release store under state/ops-agent/releases could not be read. Only read operations are admitted; repair the release store, then restart the agent.\n", + "ops-agent: the release cache under state/ops-agent/releases could not be read. Only read operations are admitted; repair the release cache, then restart the agent.\n", ); } const app = buildOpsAgentServer({ diff --git a/apps/ops-agent/src/configuration-handoff-application.ts b/apps/ops-agent/src/configuration-handoff-application.ts index c0c1a0ef7..51bb1d129 100644 --- a/apps/ops-agent/src/configuration-handoff-application.ts +++ b/apps/ops-agent/src/configuration-handoff-application.ts @@ -16,7 +16,6 @@ import { writePrivateRecord, } from "@openmapx/core/build-artifacts"; import { type ConfigurationReleaseDownloads, canonicalJson } from "@openmapx/core/ops"; -import { atomicWrite } from "@openmapx/core/release-files"; import { repoPaths, services } from "@openmapx/core/server"; import { type ConfigurationHandoffWorkerRecord, @@ -27,12 +26,7 @@ import { } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; import { maintenanceDockerContainerSchema } from "../../../packages/core/src/build-artifacts/configuration-maintenance-docker-records"; import { configurationMaintenanceOwnerRecordSchema } from "../../../packages/core/src/build-artifacts/configuration-maintenance-owner-records"; -import { - readConfigurationHandoffReleaseOverlay, - readConfigurationHandoffReleaseState, -} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { installationState } from "../../../packages/core/src/build-artifacts/control-authority"; -import { configurationHandoffServiceOrder } from "./configuration-handoff"; import { assertConfigurationHandoffHostSources, type ConfigurationHandoffChild, @@ -43,7 +37,6 @@ import { awaitAppliedContainerHealth, inspectMaintenanceDockerRuntime, } from "./configuration-maintenance-docker-inspection"; -import { readCheckedConfigurationRelease } from "./configuration-release"; import { inspectConfigurationReleaseImages } from "./configuration-release-images"; import { type DockerRuntimeOptions, runContainedProcess } from "./docker-runtime"; import { createOwnedServiceResolver } from "./owned-services"; @@ -51,6 +44,12 @@ import { createTrustedConfigurationAuthorityLoader } from "./trusted-configurati const running = new WeakSet(); +/** The maintenance committed the generation that pins this release before the helper ran. */ +function assertAppliedRelease(infraDir: string, manifestDigest: string): void { + if (services.readAppliedRelease(infraDir)?.manifestDigest !== manifestDigest) + throw Error("The applied generation does not pin the release this update froze"); +} + /** Observes one pinned release image that is already on the host. Nothing is * downloaded: a missing image, or one that differs from its pin, fails before * any service is replaced. */ @@ -186,10 +185,6 @@ export async function runConfigurationHandoffApplication(input: { phase: "pending" as const, imageId: null, })), - publication: "not_attempted" as const, - overlay: null, - statePublication: "not_attempted" as const, - state: null, }, } : {}), @@ -205,25 +200,15 @@ export async function runConfigurationHandoffApplication(input: { await services.withConfigurationGenerationWriter( stack.infraDir, async () => { - const lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(stack.infraDir).store, - ); - try { - assertArtifactTargetLeaseForward(state.lease); - assertConfigurationHandoffHostSources(child); - ownerFor(child); - readCheckedConfigurationRelease(stack.infraDir, { - releaseId: release.releaseId, - manifestDigest: release.manifestDigest, - }); - if ( - worker!.release!.images.find((value) => value.image === image.image)?.phase !== - "pending" - ) - throw Error("Original release image observation requires a pending image"); - } finally { - lock.release(); - } + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + ownerFor(child); + assertAppliedRelease(stack.infraDir, release.manifestDigest); + if ( + worker!.release!.images.find((value) => value.image === image.image)?.phase !== + "pending" + ) + throw Error("Original release image observation requires a pending image"); }, signal, ); @@ -253,48 +238,19 @@ export async function runConfigurationHandoffApplication(input: { await services.withConfigurationGenerationWriter( stack.infraDir, async () => { - const lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(stack.infraDir).store, - ); - try { - assertArtifactTargetLeaseForward(state.lease); - assertConfigurationHandoffHostSources(child); - ownerFor(child); - const checked = readCheckedConfigurationRelease(stack.infraDir, { - releaseId: release.releaseId, - manifestDigest: release.manifestDigest, - }), - current = worker!; - if ( - current.release!.images.some((image) => image.phase !== "verified") || - canonicalJson(readConfigurationHandoffReleaseOverlay(stack.infraDir)) !== - canonicalJson(intent.previousReleaseOverlay) - ) - throw Error("Original release publication source changed"); - save({ - ...current, - sequence: current.sequence + 1, - phase: "applying", - release: { ...current.release!, publication: "attempted" }, - updatedAt: Date.now(), - }); - services.writeReleaseComposeArtifacts(checked.manifest, stack.infraDir); - const published = readConfigurationHandoffReleaseOverlay(stack.infraDir); - if ( - !published || - Buffer.from(published.bytes, "base64").toString("utf8") !== - services.renderReleaseCompose(checked.manifest) - ) - throw Error("Original release overlay publication incomplete"); - save({ - ...worker!, - sequence: worker!.sequence + 1, - release: { ...worker!.release!, publication: "published", overlay: published }, - updatedAt: Date.now(), - }); - } finally { - lock.release(); - } + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + ownerFor(child); + assertAppliedRelease(stack.infraDir, release.manifestDigest); + const current = worker!; + if (current.release!.images.some((image) => image.phase !== "verified")) + throw Error("Original release images must be verified before recreation"); + save({ + ...current, + sequence: current.sequence + 1, + phase: "applying", + updatedAt: Date.now(), + }); }, signal, ); @@ -338,7 +294,7 @@ export async function runConfigurationHandoffApplication(input: { throw Error("Original release container changed"); return original; }; - for (const id of configurationHandoffServiceOrder(intent.operation.serviceIds)) { + for (const id of services.selfUpdateServiceOrder(intent.operation.serviceIds)) { signal.throwIfAborted(); const owner = ownerFor(child), authority = await loadAuthority(); @@ -357,58 +313,51 @@ export async function runConfigurationHandoffApplication(input: { const args = await services.withConfigurationGenerationWriter( stack.infraDir, async () => { - const release = await services.acquireReleaseStoreLock( - services.releaseStatePaths(stack.infraDir).store, - ); - try { - assertArtifactTargetLeaseForward(state.lease); - assertConfigurationHandoffHostSources(child); - ownerFor(child); - if ( - (await loadAuthority()).revisionId !== owner.plan.sourceAuthorityRevision || - services.currentConfigurationGeneration(stack.infraDir) !== - intent.operation.expectedCurrentRevisionId - ) - throw Error("Original helper generation changed"); - const fresh = await inspectTarget(id, ownerFor(child)); - if (!fresh || canonicalJson(fresh) !== canonicalJson(target)) - throw Error("Original helper target raced"); - if (intent.release) { - const reference = - intent.release.serviceImages[id as keyof typeof intent.release.serviceImages], - pulled = worker!.release!.images.find((value) => value.reference === reference); - if (!pulled || pulled.phase !== "verified") - throw Error("Original verified release image required"); - const [observed] = await inspectConfigurationReleaseImages({ - downloads: intent.release.downloads, - image: pulled.image, - execFile, - signal, - }); - if (observed?.imageId !== pulled.imageId) - throw Error("Original pulled release image changed"); - } - const current = worker!; - save({ - ...current, - sequence: current.sequence + 1, - phase: "applying", - targets: current.targets.map((value) => - value.serviceId === id ? { ...value, phase: "recreate_attempted" } : value, - ), - updatedAt: Date.now(), + assertArtifactTargetLeaseForward(state.lease); + assertConfigurationHandoffHostSources(child); + ownerFor(child); + if ( + (await loadAuthority()).revisionId !== owner.plan.sourceAuthorityRevision || + services.currentConfigurationGeneration(stack.infraDir) !== + intent.operation.expectedCurrentRevisionId + ) + throw Error("Original helper generation changed"); + const fresh = await inspectTarget(id, ownerFor(child)); + if (!fresh || canonicalJson(fresh) !== canonicalJson(target)) + throw Error("Original helper target raced"); + if (intent.release) { + const reference = + intent.release.serviceImages[id as keyof typeof intent.release.serviceImages], + pulled = worker!.release!.images.find((value) => value.reference === reference); + if (pulled?.phase !== "verified") + throw Error("Original verified release image required"); + const [observed] = await inspectConfigurationReleaseImages({ + downloads: intent.release.downloads, + image: pulled.image, + execFile, + signal, }); - return [ - ...services.stackComposeArgs(stack), - "up", - "-d", - "--force-recreate", - "--no-deps", - id, - ]; - } finally { - release.release(); + if (observed?.imageId !== pulled.imageId) + throw Error("Original pulled release image changed"); } + const current = worker!; + save({ + ...current, + sequence: current.sequence + 1, + phase: "applying", + targets: current.targets.map((value) => + value.serviceId === id ? { ...value, phase: "recreate_attempted" } : value, + ), + updatedAt: Date.now(), + }); + return [ + ...services.stackComposeArgs(stack), + "up", + "-d", + "--force-recreate", + "--no-deps", + id, + ]; }, signal, ); @@ -475,59 +424,6 @@ export async function runConfigurationHandoffApplication(input: { privateKey: local.privateKey, signal, }); - if (intent.release) { - await services.withConfigurationGenerationWriter( - stack.infraDir, - async () => { - const lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(stack.infraDir).store, - ); - try { - assertArtifactTargetLeaseForward(state.lease); - assertConfigurationHandoffHostSources(child); - const owner = ownerFor(child); - readCheckedConfigurationRelease(stack.infraDir, { - releaseId: intent.release!.releaseId, - manifestDigest: intent.release!.manifestDigest, - }); - if ( - owner.plan.apiParent.issuer !== "api" || - canonicalJson(readConfigurationHandoffReleaseState(stack.infraDir)) !== - canonicalJson(intent.previousReleaseState) - ) - throw Error("Original release state publication source changed"); - const current = worker!, - bytes = JSON.stringify({ - releaseId: intent.release!.releaseId, - digest: intent.release!.manifestDigest, - updateJobId: owner.plan.apiParent.apiJobId, - }); - save({ - ...current, - sequence: current.sequence + 1, - release: { ...current.release!, statePublication: "attempted" }, - updatedAt: Date.now(), - }); - atomicWrite( - join(services.releaseStatePaths(stack.infraDir).store, "current.json"), - bytes, - ); - const published = readConfigurationHandoffReleaseState(stack.infraDir); - if (!published || Buffer.from(published.bytes, "base64").toString("utf8") !== bytes) - throw Error("Original release state publication incomplete"); - save({ - ...worker!, - sequence: worker!.sequence + 1, - release: { ...worker!.release!, statePublication: "published", state: published }, - updatedAt: Date.now(), - }); - } finally { - lock.release(); - } - }, - signal, - ); - } const completedAt = Date.now(), current = worker!; save({ @@ -548,7 +444,7 @@ export async function runConfigurationHandoffApplication(input: { ); return { phase: "applied" as const, - completedServiceIds: configurationHandoffServiceOrder(intent.operation.serviceIds), + completedServiceIds: services.selfUpdateServiceOrder(intent.operation.serviceIds), }; } catch (error) { if (worker && worker.completedAt === null) { diff --git a/apps/ops-agent/src/configuration-handoff-store.ts b/apps/ops-agent/src/configuration-handoff-store.ts index 48d43d53a..d27558329 100644 --- a/apps/ops-agent/src/configuration-handoff-store.ts +++ b/apps/ops-agent/src/configuration-handoff-store.ts @@ -42,10 +42,6 @@ import { configurationHandoffRecordDomains as domains, configurationHandoffRecordName as recordName, } from "../../../packages/core/src/build-artifacts/configuration-handoff-records"; -import { - readConfigurationHandoffReleaseOverlay, - readConfigurationHandoffReleaseState, -} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { childSettlementState, installationState, @@ -236,15 +232,7 @@ async function prepareConfigurationHandoffLaunch( delegationId: observed.delegationId, assertion: attempt.assertion, helperContainerName: `openmapx-configuration-handoff-${operation.handoffId}`, - ...(owned.owner.plan.release - ? { - release: owned.owner.plan.release, - previousReleaseOverlay: readConfigurationHandoffReleaseOverlay( - owned.source.stack.infraDir, - ), - previousReleaseState: readConfigurationHandoffReleaseState(owned.source.stack.infraDir), - } - : {}), + ...(owned.owner.plan.release ? { release: owned.owner.plan.release } : {}), preparedAt: Date.now(), }); writePrivateRecord( diff --git a/apps/ops-agent/src/configuration-handoff.test.ts b/apps/ops-agent/src/configuration-handoff.test.ts index 8ed88e1e7..54ccc98a1 100644 --- a/apps/ops-agent/src/configuration-handoff.test.ts +++ b/apps/ops-agent/src/configuration-handoff.test.ts @@ -1,10 +1,10 @@ import { readFileSync } from "node:fs"; import { CONFIGURATION_RELEASE_IMAGE_NAMES } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; import { describe, expect, it, vi } from "vitest"; import { type AgentContainerFacts, configurationHandoffRunArgs, - configurationHandoffServiceOrder, inspectAgentContainer, inspectConfigurationHandoffPrerequisite, } from "./configuration-handoff"; @@ -331,7 +331,7 @@ describe("configuration helper container", () => { it("orders dependencies and the agent last without silently expanding frozen targets", () => { expect( - configurationHandoffServiceOrder([ + services.selfUpdateServiceOrder([ "ops-agent", "traefik", "app-api", @@ -341,9 +341,9 @@ it("orders dependencies and the agent last without silently expanding frozen tar ]), ).toEqual(["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"]); expect( - configurationHandoffServiceOrder(["motis", "app-api", "data-manager", "ops-agent"]), + services.selfUpdateServiceOrder(["motis", "app-api", "data-manager", "ops-agent"]), ).toEqual(["data-manager", "motis", "app-api", "ops-agent"]); - expect(() => configurationHandoffServiceOrder(["app-api", "app-api"])).toThrow(); + expect(() => services.selfUpdateServiceOrder(["app-api", "app-api"])).toThrow(); }); it("requires the actual old image's helper version before any launch", async () => { const calls: string[][] = []; diff --git a/apps/ops-agent/src/configuration-handoff.ts b/apps/ops-agent/src/configuration-handoff.ts index 64b878632..5d69c8b1c 100644 --- a/apps/ops-agent/src/configuration-handoff.ts +++ b/apps/ops-agent/src/configuration-handoff.ts @@ -16,8 +16,6 @@ const CONFIGURATION_HANDOFF_ENTRYPOINT = "dist/configuration-handoff-main.mjs"; export const CONFIGURATION_HANDOFF_LABEL = "openmapx.configuration-handoff"; const CONFIGURATION_HANDOFF_VERSION = "1"; const CONFIGURATION_HANDOFF_IMAGE_LABEL = "openmapx.configuration-handoff.version"; -const protectedOrder = ["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"] as const; -const id = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); const MOUNT_PATH = /^\/[^,\n\r]*$/; /** More would be no layout this release mounts. */ const MAX_ROOT_MOUNTS = 16; @@ -158,15 +156,6 @@ export function configurationHandoffEnvironmentNames(facts: AgentContainerFacts) .refine((names) => Buffer.byteLength(JSON.stringify(names)) <= 8192) .parse([...new Set(facts.envNames.filter((name) => !NOT_INHERITED.has(name)))].sort()); } -export function configurationHandoffServiceOrder(serviceIds: readonly string[]): string[] { - const values = z.array(id).min(1).max(64).parse(serviceIds); - if (new Set(values).size !== values.length) throw Error("Duplicate frozen handoff target"); - const protectedIds = new Set(protectedOrder); - return [ - ...values.filter((value) => !protectedIds.has(value)).sort(), - ...protectedOrder.filter((value) => values.includes(value)), - ]; -} /** Runs the running agent's image with its identity and mounts, sibling role keys * and private receipts hidden, no network and confined resources. */ export function configurationHandoffRunArgs( diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.test.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.test.ts index 499056425..8ac56fd58 100644 --- a/apps/ops-agent/src/configuration-maintenance-docker-inspection.test.ts +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.test.ts @@ -5,7 +5,6 @@ import { join } from "node:path"; import { services } from "@openmapx/core/server"; import { afterEach, expect, it } from "vitest"; import { - acquireReleaseStoreLockBriefly, awaitAppliedContainerHealth, inspectMaintenanceDockerRuntime, } from "./configuration-maintenance-docker-inspection"; @@ -23,7 +22,6 @@ function fixture(volumes = 3, secrets = 1, configs = 1) { const checkout = join(root, "checkout"), infra = join(checkout, "infra", "docker"); mkdirSync(infra, { recursive: true, mode: 0o700 }); - services.ensureReleaseStateDirectories(infra); const revision = `cfg1_${"a".repeat(43)}`; const generation = join(infra, services.CONFIGURATION_GENERATIONS, revision); mkdirSync(generation, { recursive: true, mode: 0o700 }); @@ -122,7 +120,6 @@ function fixture(volumes = 3, secrets = 1, configs = 1) { stack: { infraDir: infra, composePath: join(generation, services.GENERATED_COMPOSE_FILE), - composeReleasePath: join(infra, "unused-release.yml"), }, owner, containers: [container], @@ -219,33 +216,6 @@ it.each(["source", "access", "duplicate", "contents"])( }, ); -it("waits briefly for another holder of the release store lock", async () => { - const store = mkdtempSync(join(tmpdir(), "openmapx-release-store-")); - try { - const held = await services.acquireReleaseStoreLock(store); - setTimeout(() => held.release(), 300); - const started = Date.now(); - const lock = await acquireReleaseStoreLockBriefly(store); - lock.release(); - expect(Date.now() - started).toBeGreaterThanOrEqual(250); - } finally { - rmSync(store, { recursive: true, force: true }); - } -}); - -it("still refuses the release store lock while another holder keeps it", async () => { - const store = mkdtempSync(join(tmpdir(), "openmapx-release-store-")); - try { - const held = await services.acquireReleaseStoreLock(store); - const started = Date.now(); - await expect(acquireReleaseStoreLockBriefly(store)).rejects.toThrow("Release store is busy"); - expect(Date.now() - started).toBeGreaterThanOrEqual(1_900); - held.release(); - } finally { - rmSync(store, { recursive: true, force: true }); - } -}); - it("refuses an applied container whose healthcheck has not passed, naming its service", async () => { const { inspect, observed } = fixture(1, 0, 0); Object.assign(observed.State, { Health: { Status: "starting" } }); diff --git a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts index 66052d0ef..a4e7803c9 100644 --- a/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts +++ b/apps/ops-agent/src/configuration-maintenance-docker-inspection.ts @@ -16,30 +16,6 @@ import { verifyConfigurationRuntimeResources } from "./configuration-resource-in import type { DockerRuntimeOptions } from "./docker-runtime"; import type { OwnedServiceSnapshot } from "./owned-services"; -/** How long the inspection waits for another live holder of the release - * store lock. The agent takes it for short reads while it observes the same - * maintenance, so a helper's inspection waits for a gap instead of failing at - * the first collision, then refuses as before. */ -const RELEASE_STORE_BUSY_WAIT_MS = 2_000; - -export async function acquireReleaseStoreLockBriefly(store: string) { - const deadline = Date.now() + RELEASE_STORE_BUSY_WAIT_MS; - for (let delay = 5; ; delay = Math.min(delay * 2, 100)) { - try { - return await services.acquireReleaseStoreLock(store, {}, { failIfBusy: true }); - } catch (error) { - if ( - !(error instanceof Error && error.message === "Release store is busy") || - Date.now() >= deadline - ) - throw error; - } - await new Promise((resolve) => - setTimeout(resolve, Math.min(delay, Math.max(1, deadline - Date.now()))), - ); - } -} - /** How long the helper waits for one recreated service's healthcheck. */ const APPLIED_HEALTH_WAIT_MS = 300_000; @@ -139,263 +115,253 @@ export async function inspectMaintenanceDockerRuntime(options: { createHmac("sha256", options.privateKey.export({ type: "pkcs8", format: "der" })) .update("openmapx-private-maintenance-runtime-v1\0") .update(value) - .digest("hex"), - lock = await acquireReleaseStoreLockBriefly( - services.releaseStatePaths(options.stack.infraDir).store, - ); - try { + .digest("hex"); + // The pins are part of the generation, which is checked before and after. + if ( + !inspectedRevision || + services.currentConfigurationGeneration(options.stack.infraDir) !== inspectedRevision + ) + throw Error("Original inspected generation changed"); + const args = services.stackComposeArgs(options.stack), + expected = compose.parse(JSON.parse(await run([...args, "config", "--format", "json"]))); + for (const id of ids) { + const current = containers.filter((value) => value.serviceId === id), + original = owner.originalContainers.filter((value) => value.serviceId === id), + spec = expected.services[id]; + if (!spec || current.length !== 1 || original.length !== 1 || current[0]!.state !== "running") + throw Error("Original applied container expectation unavailable"); + const frozenImage = + id === "data-manager" && owner.receipt.generation === "committed" + ? owner.plan.coordinatorImage + : undefined, + release = owner.receipt.generation === "committed" ? owner.plan.release : undefined, + releaseReference = release?.serviceImages[id as keyof typeof release.serviceImages], + download = release?.downloads.images.find((value) => value.reference === releaseReference), + releaseImage = + release && download + ? ( + await inspectConfigurationReleaseImages({ + downloads: release.downloads, + image: download.image, + execFile: options.execFile, + signal, + }) + )[0] + : undefined, + expectedImageId = + releaseImage?.imageId ?? frozenImage?.imageId ?? original[0]!.provenance.imageId; + if ( + (frozenImage && spec.image !== frozenImage.imageReference) || + (release && (!download || !releaseImage || spec.image !== releaseReference)) + ) + throw Error("Frozen release/coordinator image reference changed"); + const owned = current[0]!, + observed = container.parse( + JSON.parse( + (await run(["container", "inspect", "--format", "{{json .}}", owned.containerId])).trim(), + ), + ), + hashLine = (await run([...args, "config", "--hash", id])).trim().split(/\s+/), + hash = hashLine.at(-1); if ( - !inspectedRevision || - services.currentConfigurationGeneration(options.stack.infraDir) !== inspectedRevision + hashLine.length !== 2 || + hashLine[0] !== id || + !hash || + !/^[a-f0-9]{64}$/.test(hash) || + observed.Id !== owned.containerId || + observed.Config.Labels["com.docker.compose.config-hash"] !== hash || + observed.Config.Image !== spec.image ) - throw Error("Original inspected generation changed"); - const args = services.stackComposeArgs(options.stack), - expected = compose.parse(JSON.parse(await run([...args, "config", "--format", "json"]))); - for (const id of ids) { - const current = containers.filter((value) => value.serviceId === id), - original = owner.originalContainers.filter((value) => value.serviceId === id), - spec = expected.services[id]; - if (!spec || current.length !== 1 || original.length !== 1 || current[0]!.state !== "running") - throw Error("Original applied container expectation unavailable"); - const frozenImage = - id === "data-manager" && owner.receipt.generation === "committed" - ? owner.plan.coordinatorImage - : undefined, - release = owner.receipt.generation === "committed" ? owner.plan.release : undefined, - releaseReference = release?.serviceImages[id as keyof typeof release.serviceImages], - download = release?.downloads.images.find((value) => value.reference === releaseReference), - releaseImage = - release && download - ? ( - await inspectConfigurationReleaseImages({ - downloads: release.downloads, - image: download.image, - execFile: options.execFile, - signal, - }) - )[0] - : undefined, - expectedImageId = - releaseImage?.imageId ?? frozenImage?.imageId ?? original[0]!.provenance.imageId; + throw Error("Original applied runtime configuration differs"); + const actualEnv = Object.fromEntries( + observed.Config.Env.map((value) => { + const separator = value.indexOf("="); + if (separator < 1) throw Error("Malformed runtime environment"); + return [value.slice(0, separator), value.slice(separator + 1)]; + }), + ), + expectedEnv = Object.fromEntries( + Object.entries(spec.environment).map(([key, value]) => [ + key, + value === null ? "" : String(value), + ]), + ); + if (Object.keys(expectedEnv).some((key) => actualEnv[key] !== expectedEnv[key])) + throw Error("Original applied runtime environment differs"); + const resources = verifyConfigurationRuntimeResources(spec, observed); + checks.push({ + serviceId: id, + ordinal: null, + check: "configuration", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ + containerId: owned.containerId, + expectedConfigHash: hash, + actualConfigHash: observed.Config.Labels["com.docker.compose.config-hash"], + resources, + expectedEnvironmentDigest: keyed(canonicalJson(expectedEnv)), + actualEnvironmentDigest: keyed( + canonicalJson( + Object.fromEntries(Object.keys(expectedEnv).map((key) => [key, actualEnv[key]])), + ), + ), + }); + if (observed.Image !== expectedImageId || observed.Image !== owned.provenance.imageId) + throw Error("Original applied image differs"); + checks.push({ + serviceId: id, + ordinal: null, + check: "image", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ + containerId: owned.containerId, + imageId: observed.Image, + expectedImageId, + ...(releaseImage && download + ? { + releaseReference: releaseImage.reference, + releasePlatformManifestDigest: download.platformManifestDigest, + } + : {}), + }); + const mounts = []; + for (const volume of spec.volumes) { + const matches = observed.Mounts.filter((value) => value.Destination === volume.target), + actual = matches[0]; if ( - (frozenImage && spec.image !== frozenImage.imageReference) || - (release && (!download || !releaseImage || spec.image !== releaseReference)) + matches.length !== 1 || + !actual || + actual.Type !== volume.type || + actual.RW !== !volume.read_only ) - throw Error("Frozen release/coordinator image reference changed"); - const owned = current[0]!, - observed = container.parse( - JSON.parse( - ( - await run(["container", "inspect", "--format", "{{json .}}", owned.containerId]) - ).trim(), - ), - ), - hashLine = (await run([...args, "config", "--hash", id])).trim().split(/\s+/), - hash = hashLine.at(-1); + throw Error("Original applied mount differs"); if ( - hashLine.length !== 2 || - hashLine[0] !== id || - !hash || - !/^[a-f0-9]{64}$/.test(hash) || - observed.Id !== owned.containerId || - observed.Config.Labels["com.docker.compose.config-hash"] !== hash || - observed.Config.Image !== spec.image + volume.type === "bind" + ? !volume.source || + !actual.Source || + (await bindSource(volume.source)) !== (await bindSource(actual.Source)) + : volume.type === "volume" && + (!volume.source || + actual.Name !== (expected.volumes[volume.source]?.name ?? volume.source)) ) - throw Error("Original applied runtime configuration differs"); - const actualEnv = Object.fromEntries( - observed.Config.Env.map((value) => { - const separator = value.indexOf("="); - if (separator < 1) throw Error("Malformed runtime environment"); - return [value.slice(0, separator), value.slice(separator + 1)]; - }), - ), - expectedEnv = Object.fromEntries( - Object.entries(spec.environment).map(([key, value]) => [ - key, - value === null ? "" : String(value), - ]), - ); - if (Object.keys(expectedEnv).some((key) => actualEnv[key] !== expectedEnv[key])) - throw Error("Original applied runtime environment differs"); - const resources = verifyConfigurationRuntimeResources(spec, observed); - checks.push({ - serviceId: id, - ordinal: null, - check: "configuration", - expectedRevisionId: inspectedRevision, - expectedImageId, + throw Error("Original applied mount source differs"); + mounts.push({ + targetDigest: keyed(volume.target), + type: actual.Type, + sourceDigest: volume.type === "bind" ? keyed(await bindSource(volume.source!)) : null, + nameDigest: actual.Name ? keyed(actual.Name) : null, + readOnly: !actual.RW, }); - details.push({ - containerId: owned.containerId, - expectedConfigHash: hash, - actualConfigHash: observed.Config.Labels["com.docker.compose.config-hash"], - resources, - expectedEnvironmentDigest: keyed(canonicalJson(expectedEnv)), - actualEnvironmentDigest: keyed( - canonicalJson( - Object.fromEntries(Object.keys(expectedEnv).map((key) => [key, actualEnv[key]])), - ), - ), - }); - if (observed.Image !== expectedImageId || observed.Image !== owned.provenance.imageId) - throw Error("Original applied image differs"); - checks.push({ - serviceId: id, - ordinal: null, - check: "image", - expectedRevisionId: inspectedRevision, - expectedImageId, - }); - details.push({ - containerId: owned.containerId, - imageId: observed.Image, - expectedImageId, - ...(releaseImage && download - ? { - releaseReference: releaseImage.reference, - releasePlatformManifestDigest: download.platformManifestDigest, - } - : {}), - }); - const mounts = []; - for (const volume of spec.volumes) { - const matches = observed.Mounts.filter((value) => value.Destination === volume.target), - actual = matches[0]; + } + const mountedFiles = [ + ...spec.secrets.map((secret) => ({ + path: expected.secrets[secret.source]?.file, + target: `/run/secrets/${secret.target ?? secret.source}`, + })), + ...spec.configs.map((config) => ({ + path: expected.configs[config.source]?.file, + target: config.target?.startsWith("/") + ? config.target + : `/${config.target ?? config.source}`, + })), + ]; + for (const { path, target } of mountedFiles) { + const matches = observed.Mounts.filter((value) => value.Destination === target), + actual = matches[0]; + if ( + !path || + matches.length !== 1 || + !actual || + actual.Type !== "bind" || + actual.RW || + !actual.Source || + (await realpath(path)) !== (await realpath(actual.Source)) + ) + throw Error("Original applied configuration/secret mount differs"); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await file.stat(), + named = await lstat(path); if ( - matches.length !== 1 || - !actual || - actual.Type !== volume.type || - actual.RW !== !volume.read_only + !before.isFile() || + before.nlink !== 1 || + before.size > 65536 * 4 || + before.dev !== named.dev || + before.ino !== named.ino ) - throw Error("Original applied mount differs"); + throw Error("Original configuration/secret source unavailable"); + const expectedBytes = await file.readFile(), + encoded = (await run(["container", "exec", observed.Id, "base64", target])).replace( + /[\r\n]/g, + "", + ), + actualBytes = Buffer.from(encoded, "base64"), + expectedDigest = keyed(expectedBytes), + actualDigest = keyed(actualBytes), + after = await file.stat(), + latest = await lstat(path); if ( - volume.type === "bind" - ? !volume.source || - !actual.Source || - (await bindSource(volume.source)) !== (await bindSource(actual.Source)) - : volume.type === "volume" && - (!volume.source || - actual.Name !== (expected.volumes[volume.source]?.name ?? volume.source)) + before.dev !== latest.dev || + before.ino !== latest.ino || + before.size !== after.size || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + encoded !== actualBytes.toString("base64") || + !timingSafeEqual(Buffer.from(expectedDigest, "hex"), Buffer.from(actualDigest, "hex")) ) - throw Error("Original applied mount source differs"); + throw Error("Original runtime configuration/secret content differs"); mounts.push({ - targetDigest: keyed(volume.target), + targetDigest: keyed(target), type: actual.Type, - sourceDigest: volume.type === "bind" ? keyed(await bindSource(volume.source!)) : null, - nameDigest: actual.Name ? keyed(actual.Name) : null, - readOnly: !actual.RW, + sourceDigest: keyed(await realpath(path)), + readOnly: true, + expectedDigest, + actualDigest, }); + } finally { + await file.close(); } - const mountedFiles = [ - ...spec.secrets.map((secret) => ({ - path: expected.secrets[secret.source]?.file, - target: `/run/secrets/${secret.target ?? secret.source}`, - })), - ...spec.configs.map((config) => ({ - path: expected.configs[config.source]?.file, - target: config.target?.startsWith("/") - ? config.target - : `/${config.target ?? config.source}`, - })), - ]; - for (const { path, target } of mountedFiles) { - const matches = observed.Mounts.filter((value) => value.Destination === target), - actual = matches[0]; - if ( - !path || - matches.length !== 1 || - !actual || - actual.Type !== "bind" || - actual.RW || - !actual.Source || - (await realpath(path)) !== (await realpath(actual.Source)) - ) - throw Error("Original applied configuration/secret mount differs"); - const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const before = await file.stat(), - named = await lstat(path); - if ( - !before.isFile() || - before.nlink !== 1 || - before.size > 65536 * 4 || - before.dev !== named.dev || - before.ino !== named.ino - ) - throw Error("Original configuration/secret source unavailable"); - const expectedBytes = await file.readFile(), - encoded = (await run(["container", "exec", observed.Id, "base64", target])).replace( - /[\r\n]/g, - "", - ), - actualBytes = Buffer.from(encoded, "base64"), - expectedDigest = keyed(expectedBytes), - actualDigest = keyed(actualBytes), - after = await file.stat(), - latest = await lstat(path); - if ( - before.dev !== latest.dev || - before.ino !== latest.ino || - before.size !== after.size || - before.mtimeMs !== after.mtimeMs || - before.ctimeMs !== after.ctimeMs || - encoded !== actualBytes.toString("base64") || - !timingSafeEqual(Buffer.from(expectedDigest, "hex"), Buffer.from(actualDigest, "hex")) - ) - throw Error("Original runtime configuration/secret content differs"); - mounts.push({ - targetDigest: keyed(target), - type: actual.Type, - sourceDigest: keyed(await realpath(path)), - readOnly: true, - expectedDigest, - actualDigest, - }); - } finally { - await file.close(); - } - } - checks.push({ - serviceId: id, - ordinal: null, - check: "mount", - expectedRevisionId: inspectedRevision, - expectedImageId, - }); - // Every mount and mounted-file byte comparison above still runs. Bind - // their complete canonical evidence without repeating it in the signed owner. - details.push({ - containerId: owned.containerId, - mountCount: mounts.length, - mountsDigest: keyed(canonicalJson(mounts)), - }); - if ( - observed.State.Status !== "running" || - !observed.State.Running || - (observed.State.Health && observed.State.Health.Status !== "healthy") || - (spec.healthcheck && - !spec.healthcheck.disable && - observed.State.Health?.Status !== "healthy") - ) - throw Error(`Original applied container not ready: ${id}`); - checks.push({ - serviceId: id, - ordinal: null, - check: "readiness", - expectedRevisionId: inspectedRevision, - expectedImageId, - }); - details.push({ - containerId: owned.containerId, - state: observed.State.Status, - running: observed.State.Running, - health: observed.State.Health?.Status ?? null, - }); } - if (services.currentConfigurationGeneration(options.stack.infraDir) !== inspectedRevision) - throw Error("Original inspected generation changed"); - return { checks, details }; - } finally { - lock.release(); + checks.push({ + serviceId: id, + ordinal: null, + check: "mount", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + // Every mount and mounted-file byte comparison above still runs. Bind + // their complete canonical evidence without repeating it in the signed owner. + details.push({ + containerId: owned.containerId, + mountCount: mounts.length, + mountsDigest: keyed(canonicalJson(mounts)), + }); + if ( + observed.State.Status !== "running" || + !observed.State.Running || + (observed.State.Health && observed.State.Health.Status !== "healthy") || + (spec.healthcheck && !spec.healthcheck.disable && observed.State.Health?.Status !== "healthy") + ) + throw Error(`Original applied container not ready: ${id}`); + checks.push({ + serviceId: id, + ordinal: null, + check: "readiness", + expectedRevisionId: inspectedRevision, + expectedImageId, + }); + details.push({ + containerId: owned.containerId, + state: observed.State.Status, + running: observed.State.Running, + health: observed.State.Health?.Status ?? null, + }); } + if (services.currentConfigurationGeneration(options.stack.infraDir) !== inspectedRevision) + throw Error("Original inspected generation changed"); + return { checks, details }; } diff --git a/apps/ops-agent/src/configuration-maintenance-store.ts b/apps/ops-agent/src/configuration-maintenance-store.ts index bb6182291..ec3153924 100644 --- a/apps/ops-agent/src/configuration-maintenance-store.ts +++ b/apps/ops-agent/src/configuration-maintenance-store.ts @@ -12,10 +12,6 @@ import { import { coordinatorPullSubmissionDigest, parseOpsJobStatus } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import type { ConfigurationReleaseSnapshot } from "../../../packages/core/src/build-artifacts/configuration-release-snapshot"; -import { - readConfigurationHandoffReleaseOverlay, - readConfigurationHandoffReleaseState, -} from "../../../packages/core/src/build-artifacts/configuration-source-witness"; import { type CoordinatorPullWitness, coordinatorPullObservationDomain, @@ -436,20 +432,34 @@ export function readPreparedMaintenanceRelease( installation: ControlInstallation, maintenanceId: string, ): ConfigurationReleaseSnapshot | undefined { - let raw: unknown; + let owner: ConfigurationMaintenanceOwnerRecord; try { - raw = readPrivateRecord( - installationState(installation), - "configuration-maintenance", - `${maintenanceId}.json`, - ); + owner = readMaintenanceOwner(installation, maintenanceId); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; } - const owner = verifyControl(installation.publicKey, domain, raw, ownerSchema); return owner.receipt.dmFence ? owner.plan.release : undefined; } +/** The release a maintenance pins in the generation it commits; undefined when it keeps the + * applied release. A maintenance that was never prepared is an error. */ +export function readMaintenanceRelease( + installation: ControlInstallation, + maintenanceId: string, +): ConfigurationReleaseSnapshot | undefined { + return readMaintenanceOwner(installation, maintenanceId).plan.release; +} +function readMaintenanceOwner( + installation: ControlInstallation, + maintenanceId: string, +): ConfigurationMaintenanceOwnerRecord { + const raw = readPrivateRecord( + installationState(installation), + "configuration-maintenance", + `${maintenanceId}.json`, + ); + return verifyControl(installation.publicKey, domain, raw, ownerSchema); +} /** Observation-only completion of the original prepared handoff. All sources * and physical descendants must already be settled; this runs no Docker I/O. */ async function observeOwnedConfigurationHandoff( @@ -3041,43 +3051,22 @@ export async function inspectFrozenMaintenanceRuntime( throw Error("Unchanged original owned runtime differs"); if (contained && canonicalJson(containers) !== canonicalJson(owner.originalContainers)) throw Error("Original noncommitted runtime changed"); - let currentRelease: { releaseId: string; digest: string; updateJobId: string } | undefined; if (!contained && owner.plan.release) { if (!owner.plan.handoff || owner.plan.apiParent.issuer !== "api") - throw Error("Original release helper/current-state owner required"); + throw Error("Original release helper owner required"); const original = readCompletedConfigurationHandoff( - state.installation.installation, - owner.plan.handoff.handoffId, - ), - published = original.completion.release, - overlay = readConfigurationHandoffReleaseOverlay(source.infraDir), - pointer = readConfigurationHandoffReleaseState(source.infraDir); - currentRelease = { - releaseId: owner.plan.release.releaseId, - digest: owner.plan.release.manifestDigest, - updateJobId: owner.plan.apiParent.apiJobId, - }; + state.installation.installation, + owner.plan.handoff.handoffId, + ); if ( canonicalJson(original.intent.ownerPrepare) !== canonicalJson(owner.plan.prepare) || original.intent.leaseId !== owner.leaseId || original.intent.operation.planDigest !== owner.plan.planDigest || canonicalJson(original.intent.release) !== canonicalJson(owner.plan.release) || - published?.publication !== "published" || - published.statePublication !== "published" || - canonicalJson(overlay) !== canonicalJson(published.overlay) || - canonicalJson(pointer) !== canonicalJson(published.state) || - !overlay || - !pointer || - Buffer.from(pointer.bytes, "base64").toString("utf8") !== JSON.stringify(currentRelease) || - Buffer.from(overlay.bytes, "base64").toString("utf8") !== - services.renderReleaseCompose( - services.parseReleaseManifest( - owner.plan.release.canonicalManifest, - owner.plan.release.imagePrefix, - ), - ) + services.readAppliedRelease(source.infraDir)?.manifestDigest !== + owner.plan.release.manifestDigest ) - throw Error("Original helper release publication changed before final inspection"); + throw Error("The applied generation does not pin the release this update froze"); } const docker = await inspectMaintenanceDockerRuntime({ stack: source.stack, @@ -3133,10 +3122,7 @@ export async function inspectFrozenMaintenanceRuntime( ...docker.checks, ]), details = [ - { - currentRevisionId: services.currentConfigurationGeneration(source.infraDir), - ...(currentRelease ? { currentRelease } : {}), - }, + { currentRevisionId: services.currentConfigurationGeneration(source.infraDir) }, { containers }, { guard }, ...owner.plan.steps.map((step) => ({ @@ -3760,7 +3746,6 @@ export async function prepareConfigurationMaintenanceAdmission( const release = platform?.release ? input.frozenRelease ? await verifyFrozenConfigurationRelease({ - infraDir: source.infraDir, selection: platform.release, frozen: input.frozenRelease, execFile: source.execFile, @@ -3818,6 +3803,9 @@ export async function prepareConfigurationMaintenanceAdmission( selectedRoots: candidate.payload.selectedRoots ?? desired.roots, }, allowMissingSelected: !candidate.payload.selectedRoots && desired.source === "default", + ...(release + ? { release: services.parseReleaseManifest(release.canonicalManifest, release.imagePrefix) } + : {}), }); const prospective = new Set(rendered.resolved.enabled.map((service) => service.manifest.id)); if (!prospective.has("data-manager") || recreateServiceIds.some((id) => !prospective.has(id))) diff --git a/apps/ops-agent/src/configuration-release-preflight.ts b/apps/ops-agent/src/configuration-release-preflight.ts index d0b4903ef..6d113c763 100644 --- a/apps/ops-agent/src/configuration-release-preflight.ts +++ b/apps/ops-agent/src/configuration-release-preflight.ts @@ -1,7 +1,5 @@ -/** Resolve public release inputs under the actual private release-store lock. - * This is a read-only preflight, not target ownership or permission to pull. */ -import { join } from "node:path"; -import { canonicalJson } from "@openmapx/core/ops"; +/** Resolve public release inputs from the agent's manifest cache. This is a + * read-only preflight, not target ownership or permission to pull. */ import { services } from "@openmapx/core/server"; import { type ConfigurationReleaseSnapshot, @@ -18,45 +16,32 @@ export async function preflightConfigurationRelease(input: { execFile: NonNullable; signal: AbortSignal; }) { - const lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(input.infraDir).store, - ); - try { - input.signal.throwIfAborted(); - const release = readCheckedConfigurationRelease(input.infraDir, input.selection), - downloads = await inspectConfigurationReleaseDownloads({ - release, - execFile: input.execFile, - signal: input.signal, - }), - current = readCheckedConfigurationRelease(input.infraDir, input.selection), - channel = services.releaseSelection(); - if ( - channel.kind === "local" || - services.canonicalReleaseManifest(current.manifest) !== - services.canonicalReleaseManifest(release.manifest) - ) - throw Error("Checked release changed during preflight"); - return configurationReleaseSnapshotSchema.parse({ - version: 1, - releaseId: release.releaseId, - manifestDigest: release.manifestDigest, - imagePrefix: channel.imagePrefix, - canonicalManifest: services.canonicalReleaseManifest(release.manifest), - serviceImages: release.serviceImages, - downloads, + input.signal.throwIfAborted(); + const channel = services.releaseSelection(); + if (channel.kind === "local") throw new services.ReleaseChannelDisabledError(); + // A cached manifest is named by its digest, so it cannot change under this read. + const release = readCheckedConfigurationRelease(input.infraDir, input.selection), + downloads = await inspectConfigurationReleaseDownloads({ + release, + execFile: input.execFile, + signal: input.signal, }); - } finally { - lock.release(); - } + return configurationReleaseSnapshotSchema.parse({ + version: 1, + releaseId: release.releaseId, + manifestDigest: release.manifestDigest, + imagePrefix: channel.imagePrefix, + canonicalManifest: services.canonicalReleaseManifest(release.manifest), + serviceImages: release.serviceImages, + downloads, + }); } /** Re-checks a release that maintenance already froze, without the registry. - * It confirms the checked release and channel still match the snapshot and that - * every pinned image is on the host, so an outage that starts after the images - * were downloaded cannot fail a maintenance already in progress. */ + * It confirms the channel still matches the snapshot and that every pinned + * image is on the host, so an outage that starts after the images were + * downloaded cannot fail a maintenance already in progress. */ export async function verifyFrozenConfigurationRelease(input: { - infraDir: string; selection: { releaseId: string; manifestDigest: string }; frozen: ConfigurationReleaseSnapshot; execFile: NonNullable; @@ -68,27 +53,14 @@ export async function verifyFrozenConfigurationRelease(input: { frozen.manifestDigest !== input.selection.manifestDigest ) throw Error("Frozen release does not match the selection"); - const lock = await services.acquireReleaseStoreLock( - services.releaseStatePaths(input.infraDir).store, - ); - try { - input.signal.throwIfAborted(); - const release = readCheckedConfigurationRelease(input.infraDir, input.selection), - channel = services.releaseSelection(); - if ( - channel.kind === "local" || - channel.imagePrefix !== frozen.imagePrefix || - services.canonicalReleaseManifest(release.manifest) !== frozen.canonicalManifest || - canonicalJson(release.serviceImages) !== canonicalJson(frozen.serviceImages) - ) - throw Error("Checked release changed after it was frozen"); - await inspectConfigurationReleaseImages({ - downloads: frozen.downloads, - execFile: input.execFile, - signal: input.signal, - }); - return frozen; - } finally { - lock.release(); - } + input.signal.throwIfAborted(); + const channel = services.releaseSelection(); + if (channel.kind === "local" || channel.imagePrefix !== frozen.imagePrefix) + throw Error("The configured release repository changed after the release was frozen"); + await inspectConfigurationReleaseImages({ + downloads: frozen.downloads, + execFile: input.execFile, + signal: input.signal, + }); + return frozen; } diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts index 20622a70d..0db8c3a41 100644 --- a/apps/ops-agent/src/configuration-release.test.ts +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -15,8 +15,7 @@ beforeEach(() => { vi.stubEnv("OPENMAPX_RELEASE", "main"); vi.stubEnv("OPENMAPX_RELEASE_MANIFEST_REPOSITORY", "ghcr.io/openmapx/release-manifest"); root = mkdtempSync(join(tmpdir(), "openmapx-configuration-release-")); - directory = services.releaseStatePaths(root).store; - mkdirSync(directory, { recursive: true, mode: 0o700 }); + directory = services.releaseCacheDirectory(root); raw = services.canonicalReleaseManifest( services.parseReleaseManifest( JSON.stringify({ @@ -33,8 +32,9 @@ beforeEach(() => { ), ); manifestDigest = createHash("sha256").update(raw).digest("hex"); - writeFileSync(join(directory, `${releaseId}.json`), raw, { mode: 0o600 }); + services.cacheReleaseManifest(root, services.parseReleaseManifest(raw)); }); +const cached = (digest = manifestDigest) => join(directory, `${digest}.json`); afterEach(() => { rmSync(root, { recursive: true, force: true }); vi.unstubAllEnvs(); @@ -66,11 +66,11 @@ it("rejects digest drift and noncanonical cached bytes", () => { expect(() => readCheckedConfigurationRelease(root, { releaseId, manifestDigest: "0".repeat(64) }), ).toThrow(); - writeFileSync(join(directory, `${releaseId}.json`), `${raw}\n`); + writeFileSync(cached(), `${raw}\n`); expect(() => readCheckedConfigurationRelease(root, { releaseId, manifestDigest })).toThrow(); }); it.each(["symlink", "hardlink"])("rejects a %s cached manifest", (kind) => { - const path = join(directory, `${releaseId}.json`), + const path = cached(), original = join(root, "original.json"); writeFileSync(original, raw, { mode: 0o600 }); rmSync(path); @@ -93,8 +93,8 @@ it.each(["foreign", "tag", "oversized", "local"])("rejects ineligible checked da event: "push", }; const modified = services.canonicalReleaseManifest(manifest); - writeFileSync(join(directory, `${releaseId}.json`), modified); const digest = createHash("sha256").update(modified).digest("hex"); + writeFileSync(cached(digest), modified, { mode: 0o600 }); expect(() => readCheckedConfigurationRelease(root, { releaseId, manifestDigest: digest }), ).toThrow(); @@ -262,7 +262,7 @@ it("inspects all six images concurrently and retains release order under reverse }); it.each(["sibling failure", "caller abort"])( - "keeps the release lock until every metadata child settles after %s", + "settles only after every metadata child settles after %s", async (failure) => { const { preflightConfigurationRelease } = await import("./configuration-release-preflight"); const held = heldMetadata(); @@ -297,9 +297,6 @@ it.each(["sibling failure", "caller abort"])( }); await flushMetadata(); expect(settled).toBe(false); - await expect( - services.acquireReleaseStoreLock(directory, {}, { failIfBusy: true }), - ).rejects.toThrow("Release store is busy"); // A late valid index must not launch a platform lookup after sibling failure. for (const call of held.calls) call.resolve(platformIndex()); expect(await result).toEqual( @@ -310,8 +307,6 @@ it.each(["sibling failure", "caller abort"])( ), ); expect(held.calls).toHaveLength(failure === "sibling failure" ? 6 : 12); - const lock = await services.acquireReleaseStoreLock(directory, {}, { failIfBusy: true }); - lock.release(); } finally { controller.abort(); held.drain(); @@ -374,53 +369,40 @@ it.each([ if (kind === "aborted") expect(execFile).not.toHaveBeenCalled(); }); -// The checked preflight joins the real private cache/lock to registry reads; +// The checked preflight joins the private manifest cache to registry reads; // this is not a target grant, pull or release runtime acceptance test. -it.each([false, true])( - "binds one checked preflight snapshot and rejects metadata-time source drift: %s", - async (drift) => { - const { preflightConfigurationRelease } = await import("./configuration-release-preflight"); - let changed = false; - const execFile = vi.fn(async (_file: string, args: readonly string[]) => { - if (args[0] === "info") return { stdout: "linux/amd64", stderr: "" }; - expect(args.slice(0, 2)).toEqual(["manifest", "inspect"]); - if (drift && !changed) { - changed = true; - writeFileSync(join(directory, `${releaseId}.json`), `${raw}\n`, { mode: 0o600 }); - } - return { - stdout: JSON.stringify({ - schemaVersion: 2, - config: { digest: `sha256:${"c".repeat(64)}`, size: 32 }, - layers: [{ digest: `sha256:${"d".repeat(64)}`, size: 64 }], - }), - stderr: "", - }; - }); - const run = () => - preflightConfigurationRelease({ - infraDir: root, - selection: { releaseId, manifestDigest }, - execFile, - signal: new AbortController().signal, - }); - if (drift) await expect(run()).rejects.toThrow("Checked configuration release changed"); - else - expect(await run()).toMatchObject({ - releaseId, - manifestDigest, - canonicalManifest: raw, - imagePrefix: "ghcr.io/openmapx", - downloads: { declaredBytes: 6 * 96 }, - }); - expect( - execFile.mock.calls.every(([, args]) => args[0] === "info" || args[0] === "manifest"), - ).toBe(true); - writeFileSync(join(directory, `${releaseId}.json`), raw, { mode: 0o600 }); - // Original lock is released on either return path; no cleanup/admission is needed. - expect(await run()).toMatchObject({ releaseId, manifestDigest }); - }, -); +it("freezes the cached manifest with its registry metadata", async () => { + const { preflightConfigurationRelease } = await import("./configuration-release-preflight"); + const execFile = vi.fn(async (_file: string, args: readonly string[]) => { + if (args[0] === "info") return { stdout: "linux/amd64", stderr: "" }; + expect(args.slice(0, 2)).toEqual(["manifest", "inspect"]); + return { + stdout: JSON.stringify({ + schemaVersion: 2, + config: { digest: `sha256:${"c".repeat(64)}`, size: 32 }, + layers: [{ digest: `sha256:${"d".repeat(64)}`, size: 64 }], + }), + stderr: "", + }; + }); + expect( + await preflightConfigurationRelease({ + infraDir: root, + selection: { releaseId, manifestDigest }, + execFile, + signal: new AbortController().signal, + }), + ).toMatchObject({ + releaseId, + manifestDigest, + canonicalManifest: raw, + imagePrefix: "ghcr.io/openmapx", + downloads: { declaredBytes: 6 * 96 }, + }); + expect( + execFile.mock.calls.every(([, args]) => args[0] === "info" || args[0] === "manifest"), + ).toBe(true); +}); function frozenIo(options: { missing?: boolean } = {}) { const calls: string[][] = []; @@ -482,7 +464,6 @@ it("re-checks a frozen release from local state only, so a registry outage canno expect( await verifyFrozenConfigurationRelease({ - infraDir: root, selection, frozen, execFile: local.execFile, @@ -494,7 +475,7 @@ it("re-checks a frozen release from local state only, so a registry outage canno frozenIo.registryDown = false; }); -it.each(["a missing image", "a changed checked release"])( +it.each(["a missing image", "another configured repository"])( "refuses a frozen release when %s", async (problem) => { const { preflightConfigurationRelease, verifyFrozenConfigurationRelease } = await import( @@ -507,12 +488,14 @@ it.each(["a missing image", "a changed checked release"])( execFile: frozenIo().execFile, signal: new AbortController().signal, }); - if (problem === "a changed checked release") - writeFileSync(join(directory, `${releaseId}.json`), `${raw}\n`, { mode: 0o600 }); + if (problem === "another configured repository") + vi.stubEnv( + "OPENMAPX_RELEASE_MANIFEST_REPOSITORY", + "registry.example.org/fork/release-manifest", + ); await expect( verifyFrozenConfigurationRelease({ - infraDir: root, selection, frozen, execFile: frozenIo({ missing: problem === "a missing image" }).execFile, diff --git a/apps/ops-agent/src/configuration-release.ts b/apps/ops-agent/src/configuration-release.ts index e77c63741..b972060d7 100644 --- a/apps/ops-agent/src/configuration-release.ts +++ b/apps/ops-agent/src/configuration-release.ts @@ -1,4 +1,4 @@ -/** Reuse canonical checked data from the agent's own release store. */ +/** Reuse canonical checked data from the agent's own manifest cache. */ import { services } from "@openmapx/core/server"; export function readCheckedConfigurationRelease( infraDir: string, diff --git a/apps/ops-agent/src/data-transit-resource-source.test.ts b/apps/ops-agent/src/data-transit-resource-source.test.ts index d0c911b92..bb27299d0 100644 --- a/apps/ops-agent/src/data-transit-resource-source.test.ts +++ b/apps/ops-agent/src/data-transit-resource-source.test.ts @@ -132,7 +132,6 @@ async function fixture( stack: { infraDir: root, composePath: join(root, "compose.yml"), - composeReleasePath: join(root, "release.yml"), }, fileExists: () => false, execFile: async (_file, args, options) => { diff --git a/apps/ops-agent/src/docker-runtime.test.ts b/apps/ops-agent/src/docker-runtime.test.ts index dc507d126..815294d8f 100644 --- a/apps/ops-agent/src/docker-runtime.test.ts +++ b/apps/ops-agent/src/docker-runtime.test.ts @@ -65,14 +65,13 @@ async function preparedStack(ids = ["motis", "postgis", "redis"]) { stack: { infraDir, composePath: services.currentConfigurationFile(infraDir, services.GENERATED_COMPOSE_FILE), - composeReleasePath: join(infraDir, "docker-compose.release.yml"), }, }; } /** The unit adapter boundary returns Docker's config read from the real generated inputs. */ function readPreparedDockerConfig( - stack: { composePath: string; composeReleasePath: string }, + stack: { composePath: string }, file: string, args: readonly string[], ) { @@ -82,11 +81,9 @@ function readPreparedDockerConfig( for (let index = 0; index < args.length; index++) { if (args[index] !== "-f") continue; const path = args[++index]; - expect( - [stack.composePath, stack.composeReleasePath] - .filter(existsSync) - .map((path) => realpathSync(path)), - ).toContain(realpathSync(path!)); + expect([stack.composePath].filter(existsSync).map((path) => realpathSync(path))).toContain( + realpathSync(path!), + ); const configuration = parseYaml(readFileSync(path!, "utf8")) as { services: Record; }; @@ -660,7 +657,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done { timeout: 5_000 }, ); const runtime = createDockerRuntime({ - stack: { infraDir: infra, composePath: compose, composeReleasePath: release }, + stack: { infraDir: infra, composePath: compose }, fileExists: existsSync, }); const diagnosticPaths = { live, staging, previous: `${live}.previous` }; @@ -678,7 +675,7 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done ).toBe("0"); if (scenario === "failed-staging-stop") { const failing = createDockerRuntime({ - stack: { infraDir: infra, composePath: compose, composeReleasePath: release }, + stack: { infraDir: infra, composePath: compose }, fileExists: existsSync, execFile: async (file, args, options) => { if (args[0] === "stop" && args[1] === "motis-staging") @@ -936,14 +933,9 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done ); }); - it("names only the frozen generated closure when a release overlay declares disabled services", async () => { + it("names only the frozen generated closure", async () => { const prepared = await preparedStack(), calls: string[][] = []; - writeFileSync( - prepared.stack.composeReleasePath, - "services:\n app-api:\n image: example/api:1\n data-manager:\n image: example/data:1\n", - { mode: 0o600 }, - ); const runtime = createDockerRuntime({ stack: prepared.stack, fileExists: existsSync, @@ -1037,7 +1029,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1056,7 +1047,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done it("maps lifecycle effects to fixed Compose argv without accepting caller argv or paths", async () => { const prepared = await preparedStack(); - writeFileSync(prepared.stack.composeReleasePath, "services: {}\n", { mode: 0o644 }); writeFileSync(join(prepared.stack.infraDir, ".env"), "", { mode: 0o644 }); const calls: Array<{ file: string; args: readonly string[] }> = []; const runtime = createDockerRuntime({ @@ -1104,7 +1094,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1156,7 +1145,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1297,7 +1285,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1341,7 +1328,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1357,7 +1343,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1456,7 +1441,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, trafficDataRoot: dataRoot, @@ -1509,7 +1493,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, trafficDataRoot: dataRoot, @@ -1543,7 +1526,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, trafficDataRoot: dataRoot, @@ -1578,7 +1560,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1630,7 +1611,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1713,7 +1693,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1761,7 +1740,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, trafficDataRoot: dataRoot, @@ -1819,7 +1797,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -1863,7 +1840,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, execFile: async (_file, args) => { @@ -1958,7 +1934,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -2039,7 +2014,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -2080,7 +2054,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -2104,7 +2077,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -2241,7 +2213,6 @@ while :; do printf "WRITE %s\\n" "$name" >&3; sleep 0.1; done stack: { infraDir: "/trusted", composePath: "/trusted/compose.yml", - composeReleasePath: "/trusted/release.yml", }, fileExists: () => false, ownedServices, @@ -2401,7 +2372,6 @@ it("streams the actual default fixed Docker human output without the JSON collec stack: { infraDir: root, composePath: join(root, "compose.yml"), - composeReleasePath: join(root, "release.yml"), }, fileExists: () => false, }); diff --git a/apps/ops-agent/src/docker-runtime.ts b/apps/ops-agent/src/docker-runtime.ts index 251576e7b..eef2c38ef 100644 --- a/apps/ops-agent/src/docker-runtime.ts +++ b/apps/ops-agent/src/docker-runtime.ts @@ -746,48 +746,39 @@ export function createDockerRuntime(options: DockerRuntimeOptions): OpsRuntime { }, }, }); - const store = services.releaseStatePaths(options.stack.infraDir).store; - await mkdir(store, { mode: 0o700, recursive: true }); - const release = await services.acquireReleaseStoreLock(store); - try { - const command = args(); - // Direct DM Start is fixed --no-recreate; guarded DM recreation is - // checked by the actual frozen-step verifier before this adapter. - // All implicit coordinator creation stays outside this generic path. - if (operation.serviceId !== "data-manager" && !command.includes("--no-deps")) { - const configuration = JSON.parse( - ( - await run( - [...composePrefix(), "config", "--format", "json"], - context.signal, - 15000, - ) - ).stdout, - ), - targets = operation.serviceId - ? [operation.serviceId] - : services.readAppliedComposeServiceIds( - options.stack.infraDir, - operation.expectedConfigurationRevisionId, - ); - if ( - services - .composeCreationDependencyIds(configuration, targets) - .includes("data-manager") - ) - throw Error("Coordinator maintenance required before service creation"); - } - const result = await changed(command, context, LIFECYCLE_TIMEOUT_MS); - const next = continuation?.(); - if (next) { - if (context.artifactLease) assertArtifactTargetLeaseForward(context.artifactLease); - if (context.controlAuthority) assertControlExecutionForward(context.controlAuthority); - return await changed(next, context, LIFECYCLE_TIMEOUT_MS); - } - return result; - } finally { - release.release(); + const command = args(); + // Direct DM Start is fixed --no-recreate; guarded DM recreation is + // checked by the actual frozen-step verifier before this adapter. + // All implicit coordinator creation stays outside this generic path. + if (operation.serviceId !== "data-manager" && !command.includes("--no-deps")) { + const configuration = JSON.parse( + ( + await run( + [...composePrefix(), "config", "--format", "json"], + context.signal, + 15000, + ) + ).stdout, + ), + targets = operation.serviceId + ? [operation.serviceId] + : services.readAppliedComposeServiceIds( + options.stack.infraDir, + operation.expectedConfigurationRevisionId, + ); + if ( + services.composeCreationDependencyIds(configuration, targets).includes("data-manager") + ) + throw Error("Coordinator maintenance required before service creation"); } + const result = await changed(command, context, LIFECYCLE_TIMEOUT_MS); + const next = continuation?.(); + if (next) { + if (context.artifactLease) assertArtifactTargetLeaseForward(context.artifactLease); + if (context.controlAuthority) assertControlExecutionForward(context.controlAuthority); + return await changed(next, context, LIFECYCLE_TIMEOUT_MS); + } + return result; }, context.signal, ); diff --git a/apps/ops-agent/src/image-stage-runtime.ts b/apps/ops-agent/src/image-stage-runtime.ts index 053b0fb2f..48cedb019 100644 --- a/apps/ops-agent/src/image-stage-runtime.ts +++ b/apps/ops-agent/src/image-stage-runtime.ts @@ -80,110 +80,102 @@ export function installImageStageRuntime( return await services.withConfigurationGenerationWriter( options.infraDir, async () => { - const directory = services.releaseStatePaths(options.infraDir).store, - lock = await services.acquireReleaseStoreLock(directory); - try { - const source = await (async () => { - try { - assertControlExecutionForward(authority); - if ( - services.currentConfigurationGeneration(options.infraDir) !== - operation.expectedConfigurationRevisionId - ) - throw Error("Image-stage configuration changed"); - const registry = await options.loadAuthority(); - if ( - !registry.services.some((service) => service.manifest.id === operation.serviceId) - ) - throw Error("Registered image source required"); - const selected = services.readServiceReleaseImageSelection( - options.infraDir, + const source = await (async () => { + try { + assertControlExecutionForward(authority); + if ( + services.currentConfigurationGeneration(options.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw Error("Image-stage configuration changed"); + const registry = await options.loadAuthority(); + if (!registry.services.some((service) => service.manifest.id === operation.serviceId)) + throw Error("Registered image source required"); + const selected = services.readServiceReleaseImageSelection( + options.infraDir, + operation.serviceId, + ), + requestedReference = + selected.imageReference ?? + services.configurationImageReference( + strictReadFile( + services.configurationGenerationComposeFile( + options.infraDir, + operation.expectedConfigurationRevisionId, + ), + 1024 * 1024, + ), operation.serviceId, - ), - requestedReference = - selected.imageReference ?? - services.configurationImageReference( - strictReadFile( - services.configurationGenerationComposeFile( - options.infraDir, - operation.expectedConfigurationRevisionId, - ), + ); + const normalized = services.coordinatorPullReferenceIdentity(requestedReference), + floating = !normalized.includes("@"); + const platform = services.imageStagePlatform( + (await read(["info", "--format", "{{.OSType}}/{{.Architecture}}"])).stdout, + ); + const metadata = floating + ? JSON.parse( + ( + await read( + ["manifest", "inspect", "--verbose", requestedReference], 1024 * 1024, - ), - operation.serviceId, - ); - const normalized = services.coordinatorPullReferenceIdentity(requestedReference), - floating = !normalized.includes("@"); - const platform = services.imageStagePlatform( - (await read(["info", "--format", "{{.OSType}}/{{.Architecture}}"])).stdout, - ); - const metadata = floating - ? JSON.parse( - ( - await read( - ["manifest", "inspect", "--verbose", requestedReference], - 1024 * 1024, - ) - ).stdout, - ) - : null; - const stagedReference = services.immutableImageStageReference( - requestedReference, - metadata, - platform, - ), - before = floating ? await tagIdentity(requestedReference) : null; - if ( - (await options.loadAuthority()).revisionId !== registry.revisionId || - services.currentConfigurationGeneration(options.infraDir) !== - operation.expectedConfigurationRevisionId - ) - throw Error("Image-stage source changed"); - assertImageStageLeaseCompatibility(authority); - assertControlExecutionForward(authority); - return { + ) + ).stdout, + ) + : null; + const stagedReference = services.immutableImageStageReference( requestedReference, - stagedReference, - floating, - before, - releaseSelectionDigest: selected.releaseSelectionDigest, - }; - } catch (error) { - retainConfigurationPreEffectRefusal(authority); - throw error; - } - })(); - beginControlEffect(authority); - const args = ["image", "pull", source.stagedReference]; - if (options.execFile) - await run("docker", args, { - signal: context.signal, - timeout: remaining(), - maxBuffer: 1024 * 1024, - }); - else - await runContainedStreamingProcess("docker", args, { - signal: context.signal, - timeout: remaining(), - onLine: context.emitLog, - redact: safeProcessDiagnostic, - }); - const stagedId = await inspect(source.stagedReference); - if ( - !stagedId || - (source.floating && (await tagIdentity(source.requestedReference)) !== source.before) - ) - throw new OpsExecutionFailure("recovery_required", {}); - return { - revisionId: operation.expectedConfigurationRevisionId, - requestedReference: source.requestedReference, - stagedReference: source.stagedReference, - imageId: stagedId, - releaseSelectionDigest: source.releaseSelectionDigest, - }; - } finally { - lock.release(); - } + metadata, + platform, + ), + before = floating ? await tagIdentity(requestedReference) : null; + if ( + (await options.loadAuthority()).revisionId !== registry.revisionId || + services.currentConfigurationGeneration(options.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw Error("Image-stage source changed"); + assertImageStageLeaseCompatibility(authority); + assertControlExecutionForward(authority); + return { + requestedReference, + stagedReference, + floating, + before, + releaseSelectionDigest: selected.releaseSelectionDigest, + }; + } catch (error) { + retainConfigurationPreEffectRefusal(authority); + throw error; + } + })(); + beginControlEffect(authority); + const args = ["image", "pull", source.stagedReference]; + if (options.execFile) + await run("docker", args, { + signal: context.signal, + timeout: remaining(), + maxBuffer: 1024 * 1024, + }); + else + await runContainedStreamingProcess("docker", args, { + signal: context.signal, + timeout: remaining(), + onLine: context.emitLog, + redact: safeProcessDiagnostic, + }); + const stagedId = await inspect(source.stagedReference); + if ( + !stagedId || + (source.floating && (await tagIdentity(source.requestedReference)) !== source.before) + ) + throw new OpsExecutionFailure("recovery_required", {}); + return { + revisionId: operation.expectedConfigurationRevisionId, + requestedReference: source.requestedReference, + stagedReference: source.stagedReference, + imageId: stagedId, + releaseSelectionDigest: source.releaseSelectionDigest, + }; }, context.signal, ); diff --git a/apps/ops-agent/src/owned-services.test.ts b/apps/ops-agent/src/owned-services.test.ts index 0acb14abe..43bb80236 100644 --- a/apps/ops-agent/src/owned-services.test.ts +++ b/apps/ops-agent/src/owned-services.test.ts @@ -138,10 +138,7 @@ it.runIf(installedCompose)( async () => { const fixture = await ownedComposePathRuntime(); roots.push(fixture.host.root); - fixture.row.Labels["com.docker.compose.project.config_files"] = [ - fixture.host.stack.composePath, - fixture.host.stack.composeReleasePath, - ].join(","); + fixture.row.Labels["com.docker.compose.project.config_files"] = fixture.host.stack.composePath; fixture.row.Labels["com.docker.compose.project.working_dir"] = dirname( fixture.host.stack.composePath, ); diff --git a/apps/ops-agent/src/owned-services.ts b/apps/ops-agent/src/owned-services.ts index d8fbf5222..f8d77eedf 100644 --- a/apps/ops-agent/src/owned-services.ts +++ b/apps/ops-agent/src/owned-services.ts @@ -2,7 +2,6 @@ import { createHash } from "node:crypto"; import { closeSync, constants, - existsSync, fstatSync, lstatSync, openSync, @@ -501,11 +500,7 @@ export function createOwnedServiceResolver(options: { !generationFile || first !== files.split(",")[0] || workingDirectory !== dirname(first) || - configFiles.length > 2 || - new Set(configFiles).size !== configFiles.length || - (configFiles[1] !== undefined && - (configFiles[1] !== services.releaseStatePaths(infra).overlay || - configFiles[1] !== canonical(options.stack.composeReleasePath))) + configFiles.length !== 1 ) continue; try { @@ -534,11 +529,7 @@ export function createOwnedServiceResolver(options: { const key = JSON.stringify(configFiles); let configuration = configurations.get(key); if (!configuration) { - const releasePath = services.releaseStatePaths(infra).overlay; - const args = services.stackComposeArgs( - { infraDir: infra, composePath: first, composeReleasePath: releasePath }, - (path) => (path === releasePath ? configFiles.includes(releasePath) : existsSync(path)), - ); + const args = services.stackComposeArgs({ infraDir: infra, composePath: first }); if (!args.includes("--project-name")) args.splice(1, 0, "--project-name", services.STACK_PROJECT); const config = JSON.parse(await run([...args, "config", "--format", "json"])) as { diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index aacf61165..cfe4fde0e 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -108,9 +108,8 @@ describe("assembled production service authority", () => { }, ); - it("clears only the exact selected pin through the production runtime without Docker mutation", async () => { - const host = await fixture(), - path = services.releaseStatePaths(host.infraDir).overlay; + it("clears only the exact pinned release by rendering a generation without it, without Docker mutation", async () => { + const host = await fixture(); const manifest: services.ReleaseManifest = { schemaVersion: 1, release: "checked-pin", @@ -122,9 +121,19 @@ describe("assembled production service authority", () => { ) as services.ReleaseManifest["images"], ...releaseProvenance(), }; - services.writeReleaseComposeArtifacts(manifest, host.infraDir); + await services.commitConfigurationGeneration({ + infraDir: host.infraDir, + services: host.loaded, + input: { + ...services.readAppliedConfiguration(host.infraDir), + domain: "example.test", + selectedRoots: ["motis"], + }, + release: manifest, + }); expect(await productionReleaseEffects(host.runtime).inspect()).toMatchObject({ selectedReleaseId: "checked-pin", + currentReleaseId: "checked-pin", }); const inspection = await productionReleaseEffects(host.runtime).inspectSystem({ signal: new AbortController().signal, @@ -133,19 +142,23 @@ describe("assembled production service authority", () => { expect(parseOpsResult("system.inspect", inspection).release.selectedReleaseId).toBe( "checked-pin", ); - const before = readFileSync(path, "utf8"), - pointer = services.currentConfigurationGeneration(host.infraDir); + const pinned = services.currentConfigurationGeneration(host.infraDir); const changed = { kind: "release.clear", releaseId: "different-pin" } as OpsOperation; await expect( dispatchOpsOperation(host.runtime, changed, await host.context(changed)), ).rejects.toThrow(/changed/); - expect(readFileSync(path, "utf8")).toBe(before); + expect(services.currentConfigurationGeneration(host.infraDir)).toBe(pinned); const operation = { kind: "release.clear", releaseId: "checked-pin" } as OpsOperation; expect( await dispatchOpsOperation(host.runtime, operation, await host.context(operation)), ).toEqual({ releaseId: "checked-pin", cleared: true }); - expect(existsSync(path)).toBe(false); - expect(services.currentConfigurationGeneration(host.infraDir)).toBe(pointer); + const cleared = services.currentConfigurationGeneration(host.infraDir); + expect(cleared).not.toBe(pinned); + expect(services.readAppliedRelease(host.infraDir)).toBeNull(); + expect(services.readPreviousConfigurationGeneration(host.infraDir, cleared!)).toBe(pinned); + expect(services.readGenerationRelease(host.infraDir, pinned!)?.manifest.release).toBe( + "checked-pin", + ); expect( host.calls.some( (args) => diff --git a/apps/ops-agent/src/production-runtime.ts b/apps/ops-agent/src/production-runtime.ts index 884a4c62b..be71b5fb8 100644 --- a/apps/ops-agent/src/production-runtime.ts +++ b/apps/ops-agent/src/production-runtime.ts @@ -77,7 +77,7 @@ export async function productionRegisteredServiceIds(runtime: OpsRuntime): Promi const resolvers = new WeakMap(); const releaseAdapters = new WeakMap(); -/** Release resolution and the selection clear run under the configuration +/** Release resolution and the pin clear run under the configuration * generation writer, after the staged-coordinator release-write check. */ function installReleaseCapabilityBoundary( runtime: OpsRuntime, @@ -86,20 +86,18 @@ function installReleaseCapabilityBoundary( base: AdministrativeReleaseEffects, owned: OwnedServiceResolver, execFile: NonNullable, + loadAuthority: ProductionOpsRuntimeOptions["loadAuthority"], ): AdministrativeReleaseEffects { - const directory = services.releaseStatePaths(infraDir).store; - const withSource = async (signal: AbortSignal, action: () => Promise): Promise => { + const withSource = async ( + signal: AbortSignal, + action: (writer: services.ConfigurationGenerationWriter) => Promise, + ): Promise => { try { return await services.withConfigurationGenerationWriter( infraDir, - async () => { - const lock = await services.acquireReleaseStoreLock(directory); - try { - await assertStagedCoordinatorReleaseWrite(runtime, signal); - } finally { - lock.release(); - } - return action(); + async (writer) => { + await assertStagedCoordinatorReleaseWrite(runtime, signal); + return action(writer); }, signal, ); @@ -113,14 +111,31 @@ function installReleaseCapabilityBoundary( throw error; } }; + // Clearing the pin renders the applied generation again without it; running + // containers keep their images until they are next recreated. runtime["release.clear"] = async (operation, context) => - withSource(context.signal, async () => { + withSource(context.signal, async (writer) => { context.signal.throwIfAborted(); - const result = await services.clearReleaseComposeSelection({ - path: services.releaseStatePaths(infraDir).overlay, - expectedReleaseId: operation.releaseId, + const current = services.currentConfigurationGeneration(infraDir); + if ( + !current || + services.readGenerationRelease(infraDir, current)?.manifest.release !== operation.releaseId + ) + throw Error("Release selection changed"); + const { release: _pinned, ...input } = services.readGenerationConfiguration( + infraDir, + current, + ); + const authority = await loadAuthority(); + await writer.commit({ + services: authority.services, + integrationSchemas: authority.integrationSchemas, + input, + allowMissingSelected: true, + expectedCurrentRevisionId: current, + release: null, + signal: context.signal, }); - if (!result.cleared) throw Error("Release selection changed"); return { releaseId: operation.releaseId, cleared: true as const }; }); return { @@ -205,12 +220,7 @@ export async function createProductionOpsRuntime( }); const runtime = createDockerRuntime({ stack: options.stack, - fileExists: - options.docker?.fileExists ?? - ((path) => - path === options.stack.composeReleasePath - ? services.ownedStateFilePresent(path) - : existsSync(path)), + fileExists: options.docker?.fileExists ?? existsSync, execFile: options.docker?.execFile, followLogs: options.docker?.followLogs, trafficDataRoot: join(options.stack.infraDir, "data"), @@ -285,6 +295,7 @@ export async function createProductionOpsRuntime( createDefaultReleaseEffects(options.rootDir, options.releaseRuntimeOptions), owned, options.docker?.execFile ?? runContainedProcess, + options.loadAuthority, ); createTransitousLockRuntime(runtime, { rootDir: options.rootDir }); installTransitCredentialsRuntime(runtime, { diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index fea5badc1..b19a81832 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -208,7 +208,7 @@ export const OPS_AGENT_RETENTION_LIMITS = { maxAggregateJournalBytes: 24 * 1024 * 1024, maxAggregateEventBytes: 2 * DEFAULT_JOB_ENTRIES * MAX_RETAINED_EVENT_BYTES_PER_JOB, } as const; -// Conservative read-only surface while the release store cannot be read safely. +// Conservative read-only surface while the release cache cannot be read safely. /** * Operations that change containers claim their targets in the lease store * before any effect. Pulls, builds, execs and probes leave running containers diff --git a/apps/ops-agent/src/service-pull-runtime.ts b/apps/ops-agent/src/service-pull-runtime.ts index a7bb5946c..0db711203 100644 --- a/apps/ops-agent/src/service-pull-runtime.ts +++ b/apps/ops-agent/src/service-pull-runtime.ts @@ -226,201 +226,190 @@ export function installServicePullRuntime(runtime: OpsRuntime, options: ServiceP operation.expectedConfigurationRevisionId ) throw Error("Pull configuration revision changed"); - const directory = services.releaseStatePaths(options.infraDir).store; - await mkdir(directory, { mode: 0o700, recursive: true }); - const lock = await services.acquireReleaseStoreLock(directory); - try { - const registry = await options.loadAuthority(); - let imageReference: string, releaseSelectionDigest: string | null; - if (operation.candidateId) { - if (!options.candidateSource) throw Error("Original pull candidate unavailable"); - const candidate = await readConfigurationCandidate( - options.candidateSource, - operation.candidateId, - ), - source = renderCoordinatorCandidate(candidate, registry, options.infraDir), - fixed = source.fixed, - actual = verifyCoordinatorPullAdmissionSource( - admitted.installation.installation, - candidate.record, - ), - parent = actual.parent; - if ( - fixed.action !== "update" || - !parent || - parent.issuer !== "api" || - canonicalJson(fixed.pull.operation) !== canonicalJson(operation) || - fixed.pull.operationKey !== context.execution.operationKey || - fixed.pull.operationFingerprint !== context.execution.operationFingerprint || - candidate.record.apiJobId !== parent.apiJobId || - fixed.pull.attempt.apiJobId !== parent.apiJobId || - fixed.pull.attempt.apiOwnerVersion !== parent.apiOwnerVersion || - fixed.pull.attempt.attemptId !== parent.attemptId || - fixed.pull.attempt.parentOperationKey !== parent.operationKey || - fixed.pull.attempt.parentFingerprint !== parent.fingerprint || - fixed.pull.attempt.parentDeadlineAt !== parent.deadlineAt || - fixed.pull.requestedDeadlineAt !== actual.requestedDeadlineAt - ) - throw Error("Original coordinator pull submission changed"); - verifyConfigurationReleaseSourceWitness({ - installation: admitted.installation.installation, - infraDir: options.infraDir, - sourceRevision: candidate.record.expectedSourceRevision, - expectedCurrentRevisionId: operation.expectedConfigurationRevisionId, - authorityRevision: registry.revisionId, - }); - ({ imageReference, releaseSelectionDigest } = source); - if ( - actual.coordinatorPullBaseline.configurationAuthorityRevision !== - registry.revisionId || - actual.coordinatorPullBaseline.policyRevision !== - context.claim.capability.revisionId || - actual.coordinatorPullBaseline.configImageReference !== imageReference - ) - throw Error("Original pull baseline source changed"); - await confirmCoordinatorPullOwnership( - options.ownedServices, - operation, - context.execution.operationFingerprint, - actual.coordinatorPullBaseline, - context.signal, - ); - if ( - (await options.ownedServices.inspectOwnedServices(context.signal)).find( - (s) => s.serviceId === "data-manager", - )?.state !== "running" - ) - throw Error("Ready old coordinator required before image pull"); - await assertConfigurationCoordinatorReady(context.signal); - // Same actual current native Compose/release sources plus the full - // readonly prospective render/registered manifest sources. Aliases - // are a refusal condition; no sibling OLD-image exception exists. - const args = services.stackComposeArgs({ - infraDir: options.infraDir, - composePath: services.configurationGenerationComposeFile( + const registry = await options.loadAuthority(); + let imageReference: string, releaseSelectionDigest: string | null; + if (operation.candidateId) { + if (!options.candidateSource) throw Error("Original pull candidate unavailable"); + const candidate = await readConfigurationCandidate( + options.candidateSource, + operation.candidateId, + ), + source = renderCoordinatorCandidate(candidate, registry, options.infraDir), + fixed = source.fixed, + actual = verifyCoordinatorPullAdmissionSource( + admitted.installation.installation, + candidate.record, + ), + parent = actual.parent; + if ( + fixed.action !== "update" || + !parent || + parent.issuer !== "api" || + canonicalJson(fixed.pull.operation) !== canonicalJson(operation) || + fixed.pull.operationKey !== context.execution.operationKey || + fixed.pull.operationFingerprint !== context.execution.operationFingerprint || + candidate.record.apiJobId !== parent.apiJobId || + fixed.pull.attempt.apiJobId !== parent.apiJobId || + fixed.pull.attempt.apiOwnerVersion !== parent.apiOwnerVersion || + fixed.pull.attempt.attemptId !== parent.attemptId || + fixed.pull.attempt.parentOperationKey !== parent.operationKey || + fixed.pull.attempt.parentFingerprint !== parent.fingerprint || + fixed.pull.attempt.parentDeadlineAt !== parent.deadlineAt || + fixed.pull.requestedDeadlineAt !== actual.requestedDeadlineAt + ) + throw Error("Original coordinator pull submission changed"); + verifyConfigurationReleaseSourceWitness({ + installation: admitted.installation.installation, + infraDir: options.infraDir, + sourceRevision: candidate.record.expectedSourceRevision, + expectedCurrentRevisionId: operation.expectedConfigurationRevisionId, + authorityRevision: registry.revisionId, + }); + ({ imageReference, releaseSelectionDigest } = source); + if ( + actual.coordinatorPullBaseline.configurationAuthorityRevision !== + registry.revisionId || + actual.coordinatorPullBaseline.policyRevision !== + context.claim.capability.revisionId || + actual.coordinatorPullBaseline.configImageReference !== imageReference + ) + throw Error("Original pull baseline source changed"); + await confirmCoordinatorPullOwnership( + options.ownedServices, + operation, + context.execution.operationFingerprint, + actual.coordinatorPullBaseline, + context.signal, + ); + if ( + (await options.ownedServices.inspectOwnedServices(context.signal)).find( + (s) => s.serviceId === "data-manager", + )?.state !== "running" + ) + throw Error("Ready old coordinator required before image pull"); + await assertConfigurationCoordinatorReady(context.signal); + // Same actual current native Compose sources plus the full + // readonly prospective render/registered manifest sources. Aliases + // are a refusal condition; no sibling OLD-image exception exists. + const args = services.stackComposeArgs({ + infraDir: options.infraDir, + composePath: services.configurationGenerationComposeFile( + options.infraDir, + operation.expectedConfigurationRevisionId, + ), + }); + const config = JSON.parse( + ( + await (options.execFile ?? runContainedProcess)( + "docker", + [...args, "config", "--format", "json"], + { signal: context.signal, timeout: 15000, maxBuffer: 1024 * 1024 }, + ) + ).stdout, + ) as { services?: Record }; + if ( + !config.services || + typeof config.services !== "object" || + Array.isArray(config.services) || + Object.keys(config.services).length > 256 + ) + throw Error("Current coordinator image inventory unavailable"); + const identity = services.coordinatorPullReferenceIdentity(imageReference), + enabled = new Set(source.rendered.resolved.enabled.map((value) => value.manifest.id)); + for (const service of registry.services) { + if (service.manifest.id === "data-manager") continue; + const selected = services.readServiceReleaseImageSelection( options.infraDir, - operation.expectedConfigurationRevisionId, + service.manifest.id, ), - composeReleasePath: services.releaseStatePaths(options.infraDir).overlay, - }); - const config = JSON.parse( - ( - await (options.execFile ?? runContainedProcess)( - "docker", - [...args, "config", "--format", "json"], - { signal: context.signal, timeout: 15000, maxBuffer: 1024 * 1024 }, - ) - ).stdout, - ) as { services?: Record }; + prospective = + selected.imageReference ?? + (enabled.has(service.manifest.id) + ? services.configurationImageReference( + source.rendered.rendered.composeYaml, + service.manifest.id, + ) + : services.serviceContainerImageReference(service.manifest.container)); + const current = + config.services[service.manifest.id]?.image ?? + selected.imageReference ?? + services.serviceContainerImageReference(service.manifest.container); if ( - !config.services || - typeof config.services !== "object" || - Array.isArray(config.services) || - Object.keys(config.services).length > 256 + [current, prospective].some( + (value) => services.coordinatorPullReferenceIdentity(value) === identity, + ) ) - throw Error("Current coordinator image inventory unavailable"); - const identity = services.coordinatorPullReferenceIdentity(imageReference), - enabled = new Set( - source.rendered.resolved.enabled.map((value) => value.manifest.id), - ); - for (const service of registry.services) { - if (service.manifest.id === "data-manager") continue; - const selected = services.readServiceReleaseImageSelection( + throw Error("Coordinator image reference shared by another registered service"); + } + } else { + const selected = services.readServiceReleaseImageSelection( + options.infraDir, + operation.serviceId, + ); + imageReference = + selected.imageReference ?? + services.configurationImageReference( + readFileSync( + services.configurationGenerationComposeFile( options.infraDir, - service.manifest.id, + operation.expectedConfigurationRevisionId, ), - prospective = - selected.imageReference ?? - (enabled.has(service.manifest.id) - ? services.configurationImageReference( - source.rendered.rendered.composeYaml, - service.manifest.id, - ) - : services.serviceContainerImageReference(service.manifest.container)); - const current = - config.services[service.manifest.id]?.image ?? - selected.imageReference ?? - services.serviceContainerImageReference(service.manifest.container); - if ( - [current, prospective].some( - (value) => services.coordinatorPullReferenceIdentity(value) === identity, - ) - ) - throw Error("Coordinator image reference shared by another registered service"); - } - } else { - const selected = services.readServiceReleaseImageSelection( - options.infraDir, + "utf8", + ), operation.serviceId, ); - imageReference = - selected.imageReference ?? - services.configurationImageReference( - readFileSync( - services.configurationGenerationComposeFile( - options.infraDir, - operation.expectedConfigurationRevisionId, - ), - "utf8", - ), - operation.serviceId, - ); - releaseSelectionDigest = selected.releaseSelectionDigest; - } - const remaining = () => { - const value = context.execution.deadlineAt - Date.now(); - if (value <= 0 || context.signal.aborted) - throw new OpsExecutionFailure("timeout", {}); - return value; - }; - if (operation.serviceId === "data-manager") { - try { - assertCoordinatorPullLeaseCompatibility(authority); - } catch { - retainConfigurationPreEffectRefusal(authority); - throw new OpsExecutionFailure("busy", {}); - } - beginControlEffect(authority); + releaseSelectionDigest = selected.releaseSelectionDigest; + } + const remaining = () => { + const value = context.execution.deadlineAt - Date.now(); + if (value <= 0 || context.signal.aborted) throw new OpsExecutionFailure("timeout", {}); + return value; + }; + if (operation.serviceId === "data-manager") { + try { + assertCoordinatorPullLeaseCompatibility(authority); + } catch { + retainConfigurationPreEffectRefusal(authority); + throw new OpsExecutionFailure("busy", {}); } - const pullArgs = ["image", "pull", imageReference]; - if (options.execFile) - await options.execFile("docker", pullArgs, { - signal: context.signal, - timeout: Math.min(15 * 60000, remaining()), - maxBuffer: 1024 * 1024, - }); - else - await runContainedStreamingProcess("docker", pullArgs, { - signal: context.signal, - timeout: Math.min(15 * 60000, remaining()), - onLine: context.emitLog, - redact: safeProcessDiagnostic, - }); - const imageId = ( - await (options.execFile ?? runContainedProcess)( - "docker", - ["image", "inspect", "--format", "{{.Id}}", imageReference], - { signal: context.signal, timeout: Math.min(15000, remaining()), maxBuffer: 4096 }, - ) - ).stdout.trim(); - context.signal.throwIfAborted(); - assertControlExecutionForward(authority); - if ( - (await options.loadAuthority()).revisionId !== registry.revisionId || - services.currentConfigurationGeneration(options.infraDir) !== - operation.expectedConfigurationRevisionId - ) - throw Error("Pull source changed"); - return servicePullResultSchema.parse({ - changed: true, - revisionId: operation.expectedConfigurationRevisionId, - imageReference, - imageId, - releaseSelectionDigest, - }); - } finally { - lock.release(); + beginControlEffect(authority); } + const pullArgs = ["image", "pull", imageReference]; + if (options.execFile) + await options.execFile("docker", pullArgs, { + signal: context.signal, + timeout: Math.min(15 * 60000, remaining()), + maxBuffer: 1024 * 1024, + }); + else + await runContainedStreamingProcess("docker", pullArgs, { + signal: context.signal, + timeout: Math.min(15 * 60000, remaining()), + onLine: context.emitLog, + redact: safeProcessDiagnostic, + }); + const imageId = ( + await (options.execFile ?? runContainedProcess)( + "docker", + ["image", "inspect", "--format", "{{.Id}}", imageReference], + { signal: context.signal, timeout: Math.min(15000, remaining()), maxBuffer: 4096 }, + ) + ).stdout.trim(); + context.signal.throwIfAborted(); + assertControlExecutionForward(authority); + if ( + (await options.loadAuthority()).revisionId !== registry.revisionId || + services.currentConfigurationGeneration(options.infraDir) !== + operation.expectedConfigurationRevisionId + ) + throw Error("Pull source changed"); + return servicePullResultSchema.parse({ + changed: true, + revisionId: operation.expectedConfigurationRevisionId, + imageReference, + imageId, + releaseSelectionDigest, + }); }, context.signal, ); diff --git a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts index 25ea6b4eb..774be2a8b 100644 --- a/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts +++ b/apps/ops-agent/src/test-fixtures/data-owned-http-process.ts @@ -56,7 +56,6 @@ const runtime = createDockerRuntime({ stack: { infraDir: root, composePath: join(root, "compose.yml"), - composeReleasePath: join(root, "release.yml"), }, fileExists: existsSync, // Only external Docker I/O is substituted. Production handlers, admission, diff --git a/apps/ops-agent/src/traffic-target-ownership.test.ts b/apps/ops-agent/src/traffic-target-ownership.test.ts index 6ddefe487..766a035da 100644 --- a/apps/ops-agent/src/traffic-target-ownership.test.ts +++ b/apps/ops-agent/src/traffic-target-ownership.test.ts @@ -46,7 +46,6 @@ it.each(["succeeded", "canceled"] as const)( stack: { infraDir: root, composePath: join(root, "compose.yml"), - composeReleasePath: join(root, "release.yml"), }, execFile: async (_file, args, options) => { if (args.includes("valhalla_ways_to_edges")) { diff --git a/apps/ops-agent/src/transit-credentials-runtime.test.ts b/apps/ops-agent/src/transit-credentials-runtime.test.ts index 505e8cc2c..06b0f3f0c 100644 --- a/apps/ops-agent/src/transit-credentials-runtime.test.ts +++ b/apps/ops-agent/src/transit-credentials-runtime.test.ts @@ -124,7 +124,6 @@ async function fixture() { stack: { infraDir, composePath: join(infraDir, "docker-compose.generated.yml"), - composeReleasePath: join(infraDir, "docker-compose.release.yml"), }, ownedServices: { inspectOwnedServices: async () => diff --git a/apps/ops-agent/src/trusted-config-runtime.ts b/apps/ops-agent/src/trusted-config-runtime.ts index 7aeac7b84..e427dd3f0 100644 --- a/apps/ops-agent/src/trusted-config-runtime.ts +++ b/apps/ops-agent/src/trusted-config-runtime.ts @@ -31,6 +31,7 @@ import { assertConfigurationCoordinatorReady } from "./configuration-hardlinks-r import { assertStagedCoordinatorConfigurationWrite, inspectOwnedMaintenanceGenerationRuntime, + readMaintenanceRelease, readPreparedMaintenanceRelease, } from "./configuration-maintenance-store"; import { @@ -235,8 +236,29 @@ async function commitGeneration( coreServices.prepareAgentStateDirectories(options.infraDir); // And its stores, which the generation mounts into app-api. coreServices.prepareAgentStores(options.infraDir); + // A release update's maintenance pins its frozen release in the generation it commits. + let release: coreServices.ReleaseManifest | undefined; + if (operation.kind === "stack.render" && operation.maintenanceId) { + const reading = openControlInstallation({ + infraDir: options.infraDir, + role: "ops-agent", + serviceIds: services.map((service) => service.manifest.id), + container: existsSync("/run/secrets/openmapx-control-verifiers"), + }); + try { + const frozen = readMaintenanceRelease(reading, operation.maintenanceId); + if (frozen) + release = coreServices.parseReleaseManifest( + frozen.canonicalManifest, + frozen.imagePrefix, + ); + } finally { + closeControlInstallation(reading); + } + } const mounts = await options.containerBindMounts?.(); const committed = await writer.commit({ + ...(release ? { release } : {}), ...(mounts ? { containerBindMounts: mounts } : {}), services, ownedServiceGenerations, @@ -452,7 +474,6 @@ export function installTrustedConfigurationRuntime( } if (releaseSelection && frozenRelease) await verifyFrozenConfigurationRelease({ - infraDir: options.infraDir, selection: releaseSelection, frozen: frozenRelease, execFile: options.execFile ?? runContainedProcess, diff --git a/packages/cli/__tests__/cli-build-artifacts.test.ts b/packages/cli/__tests__/cli-build-artifacts.test.ts index d083bb347..aeff9f0e0 100644 --- a/packages/cli/__tests__/cli-build-artifacts.test.ts +++ b/packages/cli/__tests__/cli-build-artifacts.test.ts @@ -137,9 +137,23 @@ describe("build provenance", () => { return git("rev-parse", "HEAD"); } - it("records the checkout's commit and the selected release for a checkout's CLI", async () => { + const pinRelease = () => + services.commitConfigurationGeneration({ + infraDir: join(buildHost, "infra", "docker"), + services: [], + input: { + domain: "maps.example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + release, + }); + + it("records the checkout's commit and the pinned release for a checkout's CLI", async () => { const commit = checkoutAtCommit(); - services.writeReleaseComposeArtifacts(release, join(buildHost, "infra", "docker")); + await pinRelease(); await expect(buildProvenance(buildHost, { bundled: false })).resolves.toEqual({ revision: commit, @@ -149,7 +163,7 @@ describe("build provenance", () => { it("records only the release for the operations agent's CLI, which is not the checkout's code", async () => { checkoutAtCommit(); - services.writeReleaseComposeArtifacts(release, join(buildHost, "infra", "docker")); + await pinRelease(); await expect(buildProvenance(buildHost, { bundled: true })).resolves.toEqual({ release: "abc123-1-1", diff --git a/packages/cli/__tests__/cli-control-rollback.test.ts b/packages/cli/__tests__/cli-control-rollback.test.ts new file mode 100644 index 000000000..ee06255ab --- /dev/null +++ b/packages/cli/__tests__/cli-control-rollback.test.ts @@ -0,0 +1,254 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { leaseStoreDirectory, openLeaseStore } from "@openmapx/core/build-artifacts"; +import { services } from "@openmapx/core/server"; +import { Command } from "commander"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { releaseProvenance } from "../../core/src/services/__fixtures__/release-provenance"; +import { renderComposeForRepo } from "../src/commands/compose"; +import { registerControlCommands } from "../src/commands/control"; +import { dockerCompose, dockerComposeStream } from "../src/lib/docker"; +import { applyGeneratedHardlinksBeforeStart } from "../src/lib/hardlinks"; +import { assertDatabaseAllowsRollback, planRollback, type RollbackPlan } from "../src/lib/rollback"; + +vi.setConfig({ testTimeout: 60000 }); + +// The Docker and data-manager edges; generations, leases and the renderer are real. +vi.mock("../src/lib/docker", async (original) => ({ + ...(await original()), + dockerCompose: vi.fn(), + dockerComposeStream: vi.fn(), +})); +vi.mock("../src/lib/hardlinks", async (original) => ({ + ...(await original()), + applyGeneratedHardlinksBeforeStart: vi.fn(async () => undefined), +})); + +const ids = ["alpha", "app-api", "data-manager", "ops-agent", "postgis"]; +let root: string; +let infraDir: string; +let printed: string[]; + +function manifest(release: string, digit: string, when?: number): services.ReleaseManifest { + return { + schemaVersion: 1, + release, + images: Object.fromEntries( + services.RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${digit.repeat(64)}`, + ]), + ) as services.ReleaseManifest["images"], + ...releaseProvenance(), + ...(when ? { databaseMigrationHead: { tag: `00${digit}0_release`, when } } : {}), + }; +} + +function newestAppliedMigration(value: number | "fails") { + vi.mocked(dockerCompose).mockResolvedValue( + value === "fails" + ? { exitCode: 1, stdout: "", stderr: "service postgis is not running" } + : { exitCode: 0, stdout: `${value}\n`, stderr: "" }, + ); +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "openmapx-rollback-")); + infraDir = join(root, "infra/docker"); + writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); + mkdirSync(infraDir, { recursive: true }); + for (const id of ids) { + const directory = join(root, "services", id); + mkdirSync(directory, { recursive: true }); + writeFileSync( + join(directory, "service.json"), + JSON.stringify({ + id, + name: id, + version: "1.0.0", + quality: "built-in", + container: { image: `fixture/${id}`, tag: "1" }, + }), + ); + } + for (const [key, value] of Object.entries({ + OPENMAPX_ROOT_DIR: root, + OPENMAPX_RELEASE: "local", + DOMAIN: "example.test", + OPENMAPX_ENABLED_SERVICES: "", + POSTGRES_PASSWORD: "p".repeat(24), + BETTER_AUTH_SECRET: "a".repeat(32), + })) + vi.stubEnv(key, value); + printed = []; + for (const stream of ["log", "error", "warn"] as const) + vi.spyOn(console, stream).mockImplementation((line: string) => { + printed.push(String(line)); + }); + vi.spyOn(process, "exit").mockImplementation((code) => { + throw Error(`exit:${code}`); + }); + vi.mocked(dockerComposeStream).mockReset().mockResolvedValue(0); + vi.mocked(dockerCompose).mockReset(); + vi.mocked(applyGeneratedHardlinksBeforeStart).mockClear(); +}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); +}); + +async function render(selected: string[], release: services.ReleaseManifest | null) { + services.writeServiceSelectionFile(infraDir, selected); + return (await renderComposeForRepo({ rootDir: root, domain: "example.test", release })) + .revisionId; +} +async function rollback(...args: string[]) { + const program = new Command().exitOverride(); + registerControlCommands(program); + return program.parseAsync(["node", "openmapx", "control", "rollback", ...args]); +} + +describe("rollback plan", () => { + it("names the applied generation and the one applied before it", async () => { + expect(() => planRollback(infraDir)).toThrow(/No configuration/); + const first = await render(["alpha"], manifest("release-1", "1", 100)); + expect(() => planRollback(infraDir)).toThrow(/first one/); + const second = await render(["alpha", "app-api"], manifest("release-2", "2", 200)); + const plan = planRollback(infraDir); + expect(plan).toMatchObject({ currentRevisionId: second, targetRevisionId: first }); + expect(plan.current?.manifest.release).toBe("release-2"); + expect(plan.target?.manifest.release).toBe("release-1"); + }); +}); + +describe("the database check", () => { + const plan = ( + target: services.ReleaseManifest | null, + current: services.ReleaseManifest | null = manifest("release-2", "2", 200), + ): RollbackPlan => ({ + currentRevisionId: `cfg1_${"b".repeat(43)}`, + targetRevisionId: `cfg1_${"a".repeat(43)}`, + current: current && { + manifest: current, + manifestDigest: services.releaseManifestDigest(current), + }, + target: target && { manifest: target, manifestDigest: services.releaseManifestDigest(target) }, + }); + + it("refuses when the database has a migration newer than the target release's newest", () => { + expect(() => assertDatabaseAllowsRollback(plan(manifest("release-1", "1", 100)), 200)).toThrow( + /newer than release release-1 \(its newest is 0010_release\)/, + ); + }); + + it("allows a database at or before the target release's newest migration", () => { + expect(() => + assertDatabaseAllowsRollback(plan(manifest("release-1", "1", 100)), 100), + ).not.toThrow(); + expect(() => + assertDatabaseAllowsRollback(plan(manifest("release-1", "1", 100)), 50), + ).not.toThrow(); + expect(() => + assertDatabaseAllowsRollback(plan(manifest("release-1", "1", 100)), null), + ).not.toThrow(); + }); + + it("refuses a target that pins no release or records no migration head", () => { + expect(() => assertDatabaseAllowsRollback(plan(null), 1)).toThrow(/pins no release/); + expect(() => assertDatabaseAllowsRollback(plan(manifest("release-1", "1")), 1)).toThrow( + /records no database migration head/, + ); + }); + + it("needs no database when the release does not change", () => { + const same = manifest("release-2", "2", 200); + expect(() => assertDatabaseAllowsRollback(plan(same, same), 999)).not.toThrow(); + expect(() => assertDatabaseAllowsRollback(plan(null, null), 999)).not.toThrow(); + }); +}); + +describe("openmapx control rollback", () => { + it("prints the plan and the configuration it reverts, and changes nothing without --yes", async () => { + await render(["alpha"], manifest("release-1", "1", 100)); + const current = await render(["alpha", "app-api"], manifest("release-2", "2", 200)); + newestAppliedMigration(100); + await expect(rollback()).rejects.toThrow("exit:0"); + const output = printed.join("\n"); + expect(output).toContain("(release release-2)"); + expect(output).toContain("(release release-1)"); + expect(output).toContain("also reverts every configuration change made since then"); + expect(output).toContain("Run again with --yes"); + expect(services.currentConfigurationGeneration(infraDir)).toBe(current); + expect(dockerComposeStream).not.toHaveBeenCalled(); + }); + + it("refuses when the database is newer than the previous release and changes nothing", async () => { + await render(["alpha"], manifest("release-1", "1", 100)); + const current = await render(["alpha"], manifest("release-2", "2", 200)); + newestAppliedMigration(200); + await expect(rollback("--yes")).rejects.toThrow(/newer than release release-1/); + expect(services.currentConfigurationGeneration(infraDir)).toBe(current); + expect(dockerComposeStream).not.toHaveBeenCalled(); + }); + + it("refuses when the database's migrations cannot be read", async () => { + await render(["alpha"], manifest("release-1", "1", 100)); + await render(["alpha"], manifest("release-2", "2", 200)); + newestAppliedMigration("fails"); + await expect(rollback("--yes")).rejects.toThrow(/postgis must be running/); + expect(dockerComposeStream).not.toHaveBeenCalled(); + }); + + it("renders the previous configuration and release again and recreates in self-update order", async () => { + const previous = await render(["alpha", "data-manager"], manifest("release-1", "1", 100)); + const current = await render( + ["alpha", "app-api", "data-manager", "ops-agent", "postgis"], + manifest("release-2", "2", 200), + ); + newestAppliedMigration(90); + await expect(rollback("--yes")).rejects.toThrow("exit:0"); + const applied = services.currentConfigurationGeneration(infraDir); + expect(applied).not.toBe(current); + expect(services.readPreviousConfigurationGeneration(infraDir, applied!)).toBe(current); + expect(services.readAppliedRelease(infraDir)?.manifest.release).toBe("release-1"); + expect(services.readGenerationConfiguration(infraDir, applied!).selectedRoots).toEqual( + services.readGenerationConfiguration(infraDir, previous).selectedRoots, + ); + expect(services.readDesiredSelection(infraDir).roots).toEqual(["alpha", "data-manager"]); + const compose = readFileSync( + join(infraDir, services.CURRENT_CONFIGURATION, "docker-compose.generated.yml"), + "utf8", + ); + expect(compose).toContain(manifest("release-1", "1").images["data-manager"]); + expect(applyGeneratedHardlinksBeforeStart).toHaveBeenCalled(); + expect(vi.mocked(dockerComposeStream).mock.calls.map((call) => call[0])).toEqual([ + ["up", "-d", "--no-deps", "alpha"], + ["up", "-d", "--no-deps", "data-manager"], + ]); + const audit = readFileSync(join(infraDir, "state/ops-agent/control-audit.jsonl"), "utf8"); + expect(JSON.parse(audit.trim().split("\n").at(-1)!)).toMatchObject({ + action: "rollback", + from: current, + to: previous, + exitCode: 0, + }); + }); + + it("stops at the first service that fails to recreate and releases the lease", async () => { + await render(["alpha", "postgis"], manifest("release-1", "1", 100)); + await render(["alpha", "postgis", "ops-agent"], manifest("release-2", "2", 200)); + newestAppliedMigration(100); + vi.mocked(dockerComposeStream).mockResolvedValueOnce(0).mockResolvedValueOnce(3); + await expect(rollback("--yes")).rejects.toThrow("exit:3"); + expect(vi.mocked(dockerComposeStream).mock.calls.map((call) => call[0])).toEqual([ + ["up", "-d", "--no-deps", "alpha"], + ["up", "-d", "--no-deps", "postgis"], + ]); + expect(printed.join("\n")).toContain("Recreating postgis failed (exit 3)"); + expect( + openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: "test" }).list(), + ).toEqual([]); + }); +}); diff --git a/packages/cli/__tests__/cli-release-check.test.ts b/packages/cli/__tests__/cli-release-check.test.ts index f26622221..3576a15bd 100644 --- a/packages/cli/__tests__/cli-release-check.test.ts +++ b/packages/cli/__tests__/cli-release-check.test.ts @@ -1,5 +1,4 @@ -import { createHash } from "node:crypto"; -import { existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { services } from "@openmapx/core/server"; @@ -31,15 +30,25 @@ function manifest(id: string): services.ReleaseManifest { ...releaseProvenance(), }; } -function fixture(onCopy?: (path: string) => void) { +async function fixture(onCopy?: (path: string) => void) { const root = mkdtempSync(join(tmpdir(), "openmapx-cli-check-")); roots.push(root); writeFileSync(join(root, "pnpm-workspace.yaml"), "packages: []\n"); const infraDir = join(root, "infra", "docker"); - const paths = services.ensureReleaseStateDirectories(infraDir); - services.writeReleaseComposeArtifacts(manifest("running"), infraDir); - const before = readFileSync(paths.overlay, "utf8"); - writeFileSync(join(paths.store, "current.json"), "running-state", { mode: 0o600 }); + mkdirSync(infraDir, { recursive: true }); + await services.commitConfigurationGeneration({ + infraDir, + services: [], + input: { + domain: "maps.example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + release: manifest("running"), + }); + const before = services.currentConfigurationGeneration(infraDir); const candidate = manifest("candidate"); const docker: release.ReleaseDockerRunner = vi.fn(async (args) => { const ok = (stdout = "") => ({ stdout, exitCode: 0, stderr: "" }); @@ -59,7 +68,7 @@ function fixture(onCopy?: (path: string) => void) { if (args.join(" ") === `rm -f ${"f".repeat(64)}`) return ok(); throw Error(`Unexpected Docker command in metadata-only check: ${JSON.stringify(args)}`); }); - return { root, infraDir, paths, before, candidate, docker }; + return { root, infraDir, before, candidate, docker }; } beforeEach(() => vi.stubEnv("OPENMAPX_RELEASE", "main")); afterEach(() => { @@ -69,76 +78,60 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +/** The pins stay those of the applied generation, which a check never replaces. */ +function expectPinsUnchanged(f: Awaited>) { + expect(services.currentConfigurationGeneration(f.infraDir)).toBe(f.before); + expect(services.readAppliedRelease(f.infraDir)?.manifest.release).toBe("running"); +} + describe("CLI checked candidates without selection", () => { - it("caches exact metadata while preserving selected pins and recorded running state", async () => { - const f = fixture(); + it("caches exact metadata by digest while preserving the applied pins", async () => { + const f = await fixture(); const result = await release.checkRelease({ infraDir: f.infraDir, docker: f.docker }); - const raw = readFileSync(join(f.paths.store, "candidate.json"), "utf8"); + const digest = services.releaseManifestDigest(f.candidate); expect(result).toMatchObject({ manifest: f.candidate, ociDigest: `sha256:${"d".repeat(64)}`, - manifestDigest: createHash("sha256").update(raw).digest("hex"), + releaseId: "candidate", + manifestDigest: digest, configuredSelector: "main", }); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.before); - expect(readFileSync(join(f.paths.store, "current.json"), "utf8")).toBe("running-state"); - }); - - it("normalizes unset and explicit default selectors identically", () => { - const f = fixture(); - vi.stubEnv("OPENMAPX_RELEASE", undefined); - const implicit = services.releaseConfigurationIdentity( - services.readAppliedReleaseSelection(f.infraDir), - ); - vi.stubEnv("OPENMAPX_RELEASE", "main"); expect( - services.releaseConfigurationIdentity(services.readAppliedReleaseSelection(f.infraDir)), - ).toBe(implicit); + readFileSync(join(services.releaseCacheDirectory(f.infraDir), `${digest}.json`), "utf8"), + ).toBe(services.canonicalReleaseManifest(f.candidate)); + expectPinsUnchanged(f); }); it.each(["oversized", "symlink"])( - "rejects %s copied metadata without publication", + "rejects %s copied metadata without caching it", async (kind) => { - const f = fixture((path) => { - if (kind === "oversized") - writeFileSync(path, "x".repeat(services.RELEASE_STORE_LIMITS.manifestBytes + 1)); + const f = await fixture((path) => { + if (kind === "oversized") writeFileSync(path, "x".repeat(32 * 1024 + 1)); else { rmSync(path); - symlinkSync(join(f.paths.store, "current.json"), path); + symlinkSync(join(f.root, "pnpm-workspace.yaml"), path); } }); await expect( release.checkRelease({ infraDir: f.infraDir, docker: f.docker }), ).rejects.toThrow(); - expect(existsSync(join(f.paths.store, "candidate.json"))).toBe(false); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.before); + expect(services.cachedReleases(f.infraDir, "ghcr.io/openmapx")).toEqual([]); + expectPinsUnchanged(f); expect(f.docker).toHaveBeenLastCalledWith(["rm", "-f", "f".repeat(64)]); }, ); - it("refuses a configured selector that changes during resolution", async () => { - const f = fixture(() => vi.stubEnv("OPENMAPX_RELEASE", "preview")); + it("refuses a disabled channel before any registry call", async () => { + const f = await fixture(); + vi.stubEnv("OPENMAPX_RELEASE", "local"); await expect(release.checkRelease({ infraDir: f.infraDir, docker: f.docker })).rejects.toThrow( - /configuration|selection.*changed/i, + /disabled/, ); - expect(existsSync(join(f.paths.store, "candidate.json"))).toBe(false); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.before); - }); - - it("refuses a different store owner before any write or registry call", async () => { - const f = fixture(); - await expect( - release.checkRelease({ - infraDir: f.infraDir, - docker: f.docker, - uid: (process.geteuid?.() ?? 0) + 1, - }), - ).rejects.toThrow(/owned|uid|unsafe/i); expect(f.docker).not.toHaveBeenCalled(); }); - it("reports a checked candidate from the command without reporting it selected", async () => { - const f = fixture(); + it("reports a checked candidate from the command without pinning it", async () => { + const f = await fixture(); commandDocker = f.docker; vi.stubEnv("OPENMAPX_ROOT_DIR", f.root); const log = vi.spyOn(console, "log").mockImplementation(() => {}); @@ -149,8 +142,8 @@ describe("CLI checked candidates without selection", () => { registerComposeCommands(program); await program.parseAsync(["compose", "release", "--check"], { from: "user" }); expect(log.mock.calls.flat().join("\n")).toContain("Checked release: candidate"); - expect(log.mock.calls.flat().join("\n")).not.toContain("Atomic release selection candidate"); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.before); + expect(log.mock.calls.flat().join("\n")).not.toContain("Pinned release candidate"); + expectPinsUnchanged(f); }); it.each(["--status", "--clear"])( diff --git a/packages/cli/__tests__/cli-release-coordinator.test.ts b/packages/cli/__tests__/cli-release-coordinator.test.ts index b52f7c7e2..9925f4db9 100644 --- a/packages/cli/__tests__/cli-release-coordinator.test.ts +++ b/packages/cli/__tests__/cli-release-coordinator.test.ts @@ -1,16 +1,15 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { services } from "@openmapx/core/server"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { releaseProvenance } from "../../core/src/services/__fixtures__/release-provenance"; import { - clearReleaseSelection, - ensureReleaseOverlay, + assertReleaseClearable, type ReleaseDockerRunner, type ReleaseManifest, + releaseForStart, selectRelease, - writeReleaseOverlay, } from "../src/lib/release"; const digest = (letter: string) => `sha256:${letter.repeat(64)}`; @@ -32,15 +31,24 @@ function manifest(release: string, dmDigest = "c"): ReleaseManifest { }; } -function fixture() { +/** An installation whose applied generation pins `before`, or nothing. */ +async function fixture(pinned = true) { const infraDir = mkdtempSync(join(tmpdir(), "openmapx-release-coordinator-")); roots.push(infraDir); const before = manifest("before"); - writeReleaseOverlay(before, { infraDir }); - const paths = services.releaseStatePaths(infraDir); - services.ensureReleaseStateDirectory(paths.store); - const overlay = readFileSync(paths.overlay, "utf8"); - return { infraDir, before, paths, overlay }; + await services.commitConfigurationGeneration({ + infraDir, + services: [], + input: { + domain: "maps.example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + release: pinned ? before : null, + }); + return { infraDir, before }; } function dockerFixture( @@ -118,8 +126,8 @@ afterEach(() => { }); describe("coordinator-safe release selection", () => { - it("preserves the live pin when the candidate changes Data Manager", async () => { - const f = fixture(); + it("refuses a candidate that changes the running Data Manager and caches nothing", async () => { + const f = await fixture(); const next = manifest("after", "0"); await expect( selectRelease({ @@ -128,7 +136,8 @@ describe("coordinator-safe release selection", () => { docker: dockerFixture(next), }), ).rejects.toThrow(/coordinator|Data Manager/i); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.overlay); + expect(services.readAppliedRelease(f.infraDir)?.manifest.release).toBe("before"); + expect(services.cachedReleases(f.infraDir, "ghcr.io/openmapx")).toEqual([]); }); it.each([ @@ -138,7 +147,7 @@ describe("coordinator-safe release selection", () => { { malformedInspect: true }, { cachedImageId: digest("0") }, ])("refuses an ambiguous or unreadable coordinator (%j)", async (observation) => { - const f = fixture(); + const f = await fixture(); const next = manifest("after"); await expect( selectRelease({ @@ -147,151 +156,102 @@ describe("coordinator-safe release selection", () => { docker: dockerFixture(next, observation), }), ).rejects.toThrow(); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.overlay); }); it.each([{ ids: [] }, { ids: [containerId] }])( - "permits bootstrap or the exact running pin (%j)", + "permits bootstrap or the exact running pin and reports the change (%j)", async ({ ids }) => { - const f = fixture(); + const f = await fixture(); const next = manifest("after"); + const lines: string[] = []; await expect( selectRelease({ infraDir: f.infraDir, resolve: async () => next, docker: dockerFixture(next, { ids }), + report: (line) => lines.push(line), }), ).resolves.toMatchObject({ release: "after" }); + expect(lines).toContain("Previous release: before"); + expect(lines).toContain(`data-manager: reused (${digest("c")} → ${digest("c")})`); + expect(services.cachedReleases(f.infraDir, "ghcr.io/openmapx")[0]?.manifest.release).toBe( + "after", + ); + // Selection renders nothing by itself; the command commits the generation. + expect(services.readAppliedRelease(f.infraDir)?.manifest.release).toBe("before"); }, ); - - it("can restore the immutable pin of the actual container after an unsafe reselection", async () => { - const f = fixture(); - writeReleaseOverlay(manifest("mismatched", "0"), { infraDir: f.infraDir }); - await selectRelease({ - infraDir: f.infraDir, - resolve: async () => f.before, - docker: dockerFixture(f.before), - }); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.overlay); - }); }); -describe("automatic selection and clearing", () => { - it.each([{ reference: manifest("after", "0").images["data-manager"] }, { listFailure: true }])( - "reports coordinator refusal as held without publishing automatic pins (%j)", - async (observation) => { - const f = fixture(); - unlinkSync(f.paths.overlay); - const result = await ensureReleaseOverlay({ - infraDir: f.infraDir, - docker: dockerFixture(f.before, observation), - }); - expect(result.status).toBe("held"); - expect(existsSync(f.paths.overlay)).toBe(false); - }, - ); - - it("does not publish automatic pins while another process owns the release store", async () => { - const f = fixture(); - unlinkSync(f.paths.overlay); - const lock = await services.acquireReleaseStoreLock(f.paths.store, {}, { failIfBusy: true }); - try { - const result = await ensureReleaseOverlay({ - infraDir: f.infraDir, - docker: dockerFixture(f.before, { ids: [] }), - }); - expect(result.status).toBe("held"); - expect(existsSync(f.paths.overlay)).toBe(false); - } finally { - lock.release(); - } - }); - - it("uses the lifecycle's actual held release lock without reacquiring it", async () => { - const f = fixture(); - unlinkSync(f.paths.overlay); - const releaseStoreLock = await services.acquireReleaseStoreLock( - f.paths.store, - {}, - { failIfBusy: true }, - ); - try { - const result = await ensureReleaseOverlay({ - infraDir: f.infraDir, - releaseStoreLock, - docker: dockerFixture(f.before, { ids: [] }), - }); - expect(result.status).toBe("resolved"); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.overlay); - } finally { - releaseStoreLock.release(); - } +describe("the release a start renders", () => { + it("keeps the applied pins without asking the registry", async () => { + const f = await fixture(); + const docker = dockerFixture(manifest("after")); + await expect(releaseForStart({ infraDir: f.infraDir, docker })).resolves.toEqual({ + manifest: f.before, + resolved: false, + }); + expect(docker).not.toHaveBeenCalled(); }); - it("keeps the overlay needed by an existing pinned coordinator", async () => { - const f = fixture(); + it("resolves and caches a release at bootstrap", async () => { + const f = await fixture(false); + const next = manifest("after"); await expect( - clearReleaseSelection({ infraDir: f.infraDir, docker: dockerFixture(f.before) }), - ).rejects.toThrow(/coordinator|Data Manager/i); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(f.overlay); + releaseForStart({ infraDir: f.infraDir, docker: dockerFixture(next, { ids: [] }) }), + ).resolves.toEqual({ manifest: next, resolved: true }); + expect(services.cachedReleases(f.infraDir, "ghcr.io/openmapx")).toHaveLength(1); }); - it.each(["malformed", "mismatched"])( - "can clear an already %s overlay for recovery", - async (kind) => { - const f = fixture(); - if (kind === "malformed") writeFileSync(f.paths.overlay, "services: [", { mode: 0o600 }); - const docker = dockerFixture(f.before, { - reference: manifest("other", "0").images["data-manager"], - }); - await expect(clearReleaseSelection({ infraDir: f.infraDir, docker })).resolves.toMatchObject({ - cleared: true, - }); - expect(existsSync(f.paths.overlay)).toBe(false); - }, - ); - - it("does not clear a different selection published after the guarded observation", async () => { - const f = fixture(); - const newer = manifest("concurrent", "0"); - const docker = dockerFixture(f.before, { - reference: newer.images["data-manager"], - onInspect: () => writeReleaseOverlay(newer, { infraDir: f.infraDir }), + it("refuses to start release images without pins when the registry cannot be reached", async () => { + const f = await fixture(false); + const docker: ReleaseDockerRunner = async () => ({ + stdout: "", + stderr: "no network", + exitCode: 1, }); - await expect(clearReleaseSelection({ infraDir: f.infraDir, docker })).rejects.toThrow( - /changed/i, + await expect(releaseForStart({ infraDir: f.infraDir, docker })).rejects.toThrow( + /pins no release.*OPENMAPX_RELEASE=local/s, ); - expect( - services.parseReleaseComposeSelection(readFileSync(f.paths.overlay, "utf8")).release, - ).toBe("concurrent"); }); - it("does not replace malformed-overlay recovery with deletion of a newly valid selection", async () => { - const f = fixture(); - writeFileSync(f.paths.overlay, "services: [", { mode: 0o600 }); - const pending = clearReleaseSelection({ - infraDir: f.infraDir, - docker: dockerFixture(f.before), + it("renders no pins under OPENMAPX_RELEASE=local", async () => { + vi.stubEnv("OPENMAPX_RELEASE", "local"); + const f = await fixture(false); + const docker = dockerFixture(manifest("after")); + await expect(releaseForStart({ infraDir: f.infraDir, docker })).resolves.toEqual({ + manifest: null, + resolved: false, }); - writeReleaseOverlay(manifest("concurrent"), { infraDir: f.infraDir }); - await expect(pending).rejects.toThrow(/changed/i); - expect( - services.parseReleaseComposeSelection(readFileSync(f.paths.overlay, "utf8")).release, - ).toBe("concurrent"); + expect(docker).not.toHaveBeenCalled(); }); +}); - it("does not clear a newly selected release after observing an absent overlay", async () => { - const f = fixture(); - unlinkSync(f.paths.overlay); - const pending = clearReleaseSelection({ - infraDir: f.infraDir, - docker: dockerFixture(f.before), - }); - writeReleaseOverlay(manifest("concurrent"), { infraDir: f.infraDir }); - await expect(pending).rejects.toThrow(/changed/i); - expect( - services.parseReleaseComposeSelection(readFileSync(f.paths.overlay, "utf8")).release, - ).toBe("concurrent"); +describe("clearing the pins", () => { + it("refuses while the running Data Manager uses the pinned image", async () => { + const f = await fixture(); + await expect( + assertReleaseClearable({ infraDir: f.infraDir, docker: dockerFixture(f.before) }), + ).rejects.toThrow(/System → Updates/); + }); + + it("names the pinned release when the coordinator runs something else or nothing", async () => { + const f = await fixture(); + await expect( + assertReleaseClearable({ + infraDir: f.infraDir, + docker: dockerFixture(f.before, { + reference: manifest("other", "0").images["data-manager"], + }), + }), + ).resolves.toBe("before"); + await expect( + assertReleaseClearable({ + infraDir: f.infraDir, + docker: dockerFixture(f.before, { ids: [] }), + }), + ).resolves.toBe("before"); + const unpinned = await fixture(false); + await expect(assertReleaseClearable({ infraDir: unpinned.infraDir })).resolves.toBeNull(); }); }); diff --git a/packages/cli/__tests__/cli-release.test.ts b/packages/cli/__tests__/cli-release.test.ts index 6caa8d974..b6d2d4996 100644 --- a/packages/cli/__tests__/cli-release.test.ts +++ b/packages/cli/__tests__/cli-release.test.ts @@ -1,30 +1,17 @@ -import { - chmodSync, - existsSync, - mkdirSync, - mkdtempSync, - readdirSync, - readFileSync, - rmSync, - symlinkSync, - writeFileSync, -} from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { join } from "node:path"; import { services } from "@openmapx/core/server"; import { Command } from "commander"; import { afterEach, describe, expect, it, vi } from "vitest"; import { releaseProvenance } from "../../core/src/services/__fixtures__/release-provenance"; import { registerComposeCommands } from "../src/commands/compose"; import { - clearReleaseSelection as clearReleaseSelectionFromCli, - ensureReleaseOverlay, type ReleaseDockerRunner, + releaseForStart, releaseStatusLines, resolveReleaseManifest, - selectRelease as selectReleaseFromCli, touchesReleasePinnedServices, - writeReleaseOverlay, } from "../src/lib/release"; // Direct command-entry tests have no Docker daemon; other tests supply their @@ -83,100 +70,31 @@ function fakeDocker(manifest = manifestJson): ReleaseDockerRunner { }); } -// A named zero-container Docker fixture, never a production-policy bypass. -function selectRelease(opts: Parameters[0] = {}) { - return selectReleaseFromCli({ ...opts, docker: opts.docker ?? fakeDocker() }); -} -function clearReleaseSelection(opts: Parameters[0] = {}) { - return clearReleaseSelectionFromCli({ ...opts, docker: opts.docker ?? fakeDocker() }); -} - let temp: string | null = null; afterEach(() => { if (temp) rmSync(temp, { recursive: true, force: true }); temp = null; + vi.unstubAllEnvs(); }); -/** A temporary `infra/docker` with its release state directory, and the overlay path in it. */ -function releaseInfra(prefix = "omx-release-") { - temp = mkdtempSync(join(tmpdir(), prefix)); - const infraDir = temp; - const path = services.releaseStatePaths(infraDir).overlay; - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - return { infraDir, path }; +/** A temporary `infra/docker` whose applied generation pins `release`, or nothing. */ +async function pinnedInfra(release: services.ReleaseManifest | null) { + temp = mkdtempSync(join(tmpdir(), "omx-release-")); + await services.commitConfigurationGeneration({ + infraDir: temp, + services: [], + input: { + domain: "maps.example.test", + selectedRoots: [], + serviceConfigs: [], + integrationConfigs: [], + serviceSecrets: [], + }, + release, + }); + return temp; } -describe("ensureReleaseOverlay", () => { - it("writes the overlay from the resolved manifest when it is missing", async () => { - const { infraDir, path } = releaseInfra(); - const docker = fakeDocker(); - const result = await ensureReleaseOverlay({ docker, infraDir }); - expect(result).toEqual({ status: "resolved", path, release: "deadbeef" }); - expect(readFileSync(path, "utf-8")).toContain(`image: ghcr.io/openmapx/api@${digest("a")}`); - expect(readFileSync(path, "utf-8")).toContain( - `image: ghcr.io/openmapx/ops-agent@${digest("d")}`, - ); - expect(readFileSync(path, "utf-8")).toContain( - `image: ghcr.io/openmapx/transitous-runner@${digest("e")}`, - ); - expect(readFileSync(path, "utf-8")).not.toMatch(/privacy/i); - }); - - it("never overwrites an existing overlay", async () => { - const { infraDir, path } = releaseInfra(); - writeReleaseOverlay(JSON.parse(manifestJson), { infraDir }); - const docker = fakeDocker(); - expect(await ensureReleaseOverlay({ docker, infraDir })).toEqual({ status: "present", path }); - expect(docker).not.toHaveBeenCalled(); - expect(readFileSync(path, "utf-8")).toContain("deadbeef"); - }); - - it("reports an unpinned stack when the registry is unreachable", async () => { - const { infraDir, path } = releaseInfra(); - const docker: ReleaseDockerRunner = async () => ({ - stdout: "", - stderr: "no network", - exitCode: 1, - }); - const result = await ensureReleaseOverlay({ docker, infraDir }); - expect(result.status).toBe("unpinned"); - expect(existsSync(path)).toBe(false); - }); -}); - -describe("ensureReleaseOverlay with a disabled channel", () => { - it("reports disabled without touching docker or writing a file", async () => { - const { infraDir, path } = releaseInfra(); - const docker = fakeDocker(); - process.env.OPENMAPX_RELEASE = "local"; - try { - expect(await ensureReleaseOverlay({ docker, infraDir })).toEqual({ status: "disabled" }); - } finally { - delete process.env.OPENMAPX_RELEASE; - } - expect(docker).not.toHaveBeenCalled(); - expect(existsSync(path)).toBe(false); - }); - - it("pulls a fork's manifest image and accepts its image prefix", async () => { - const { infraDir, path } = releaseInfra(); - const forkManifest = manifestJson.replaceAll("ghcr.io/openmapx", "registry.example.org/fork"); - const docker = fakeDocker(forkManifest); - process.env.OPENMAPX_RELEASE_MANIFEST_REPOSITORY = "registry.example.org/fork/release-manifest"; - try { - const result = await ensureReleaseOverlay({ docker, infraDir }); - expect(result.status).toBe("resolved"); - } finally { - delete process.env.OPENMAPX_RELEASE_MANIFEST_REPOSITORY; - } - expect(docker).toHaveBeenCalledWith([ - "pull", - "registry.example.org/fork/release-manifest:main", - ]); - expect(readFileSync(path, "utf-8")).toContain("registry.example.org/fork/api@"); - }); -}); - describe("touchesReleasePinnedServices", () => { it("detects every release-pinned runtime service", () => { expect(touchesReleasePinnedServices(["motis", "app-api"])).toBe(true); @@ -187,204 +105,46 @@ describe("touchesReleasePinnedServices", () => { }); describe("local release visibility", () => { - it("reports selected pins locally and compares digests before selecting a new release", async () => { - const { infraDir, path } = releaseInfra(); - const previous = JSON.parse(manifestJson); - writeReleaseOverlay(previous, { infraDir }); - const before = readFileSync(path, "utf8"); - expect(releaseStatusLines(path).join("\n")).toContain("Selected release: deadbeef"); - expect(readFileSync(path, "utf8")).toBe(before); - const next = JSON.parse(manifestJson); - next.release = "next"; - next.images.api = `ghcr.io/openmapx/api@${digest("0")}`; - next.images.web = next.images.web.replace("ghcr.io/openmapx", "mirror.example/openmapx"); - const lines: string[] = []; - await selectRelease({ - infraDir, - resolve: async () => next, - report: (line) => { - expect(readFileSync(path, "utf8")).toBe(before); - lines.push(line); - }, - }); - expect(lines.join("\n")).toContain("Previous release: deadbeef"); - expect(lines.join("\n")).toContain("Candidate release: next"); - expect(lines.join("\n")).toContain("api: changed"); - expect(lines.join("\n")).toContain("web: reused"); - expect(lines.join("\n")).not.toContain("docs:"); - expect(releaseStatusLines(path).join("\n")).toContain("Selected release: next"); + it("reports the applied configuration's pins without the registry", async () => { + const manifest = services.parseReleaseManifest(manifestJson); + const infraDir = await pinnedInfra(manifest); + const lines = releaseStatusLines(infraDir).join("\n"); + expect(lines).toContain( + `Pinned release: deadbeef (manifest ${services.releaseManifestDigest(manifest)})`, + ); + expect(lines).toContain(`api: ${manifest.images.api}`); + expect(lines).not.toContain("docs:"); }); - it("distinguishes disabled resolution from an existing active overlay", () => { - const { infraDir, path } = releaseInfra(); + it("distinguishes disabled resolution from pins that still apply", async () => { vi.stubEnv("OPENMAPX_RELEASE", "local"); - try { - expect(releaseStatusLines(path).join("\n")).toContain("Release pinning disabled"); - writeReleaseOverlay(JSON.parse(manifestJson), { infraDir }); - expect(releaseStatusLines(path).join("\n")).toContain("existing overlay still applies"); - expect(releaseStatusLines(path).join("\n")).toContain("Selected release: deadbeef"); - } finally { - vi.unstubAllEnvs(); - } - }); - - it("leaves selected pins intact when resolution fails", async () => { - const { infraDir, path } = releaseInfra(); - writeReleaseOverlay(JSON.parse(manifestJson), { infraDir }); - const before = readFileSync(path, "utf8"); - await expect( - selectRelease({ - infraDir, - resolve: async () => { - throw new Error("offline"); - }, - }), - ).rejects.toThrow("offline"); - expect(readFileSync(path, "utf8")).toBe(before); + expect(releaseStatusLines(await pinnedInfra(null)).join("\n")).toContain( + "Release pinning disabled", + ); + const pinned = releaseStatusLines( + await pinnedInfra(services.parseReleaseManifest(manifestJson)), + ).join("\n"); + expect(pinned).toContain("the applied pins still apply"); + expect(pinned).toContain("Pinned release: deadbeef"); }); -}); -it("exposes read-only release status on the compose command", () => { - const program = new Command(); - registerComposeCommands(program); - const release = program.commands - .find((command) => command.name() === "compose") - ?.commands.find((command) => command.name() === "release"); - expect(release?.options.some((option) => option.long === "--status")).toBe(true); -}); - -it("compares legacy pins and rejects malformed local overlays before resolution", async () => { - const { infraDir, path } = releaseInfra(); - const manifest = JSON.parse(manifestJson); - writeFileSync(path, `services:\n app-api:\n image: ${manifest.images.api}\n`, { - mode: 0o600, - }); - const lines: string[] = []; - await selectRelease({ - infraDir, - resolve: async () => manifest, - report: (line) => lines.push(line), + it("asks for a selection when nothing is pinned", async () => { + expect(releaseStatusLines(await pinnedInfra(null)).join("\n")).toContain( + "pins no release. Run `pnpm openmapx compose release`", + ); }); - expect(lines.join("\n")).toContain("Previous release: unknown (modified overlay)"); - expect(lines.join("\n")).toContain("api: reused"); - writeFileSync(path, "services: ["); - const resolve = vi.fn(async () => manifest); - await expect(selectRelease({ infraDir, resolve })).rejects.toThrow(); - expect(resolve).not.toHaveBeenCalled(); - expect(readFileSync(path, "utf8")).toBe("services: ["); - expect(() => releaseStatusLines(path)).toThrow(); }); -it("does not report a candidate as selected when atomic overlay publication fails", async () => { - const { infraDir, path } = releaseInfra(); - const previous = JSON.parse(manifestJson); - writeReleaseOverlay(previous, { infraDir }); - const before = readFileSync(path, "utf8"); - const next = { ...previous, release: "next" }; - const lines: string[] = []; - // Once the store lock is held, an owner-only release directory without write - // permission refuses the temporary overlay. - const resolve = async () => { - chmodSync(dirname(path), 0o500); - return next; - }; - try { - await expect( - selectRelease({ infraDir, resolve, report: (line) => lines.push(line) }), - ).rejects.toThrow(); - } finally { - chmodSync(dirname(path), 0o700); - } - expect(lines.join("\n")).toContain("Candidate release: next"); - expect(lines.join("\n")).not.toContain("Selected release: next"); - expect(readFileSync(path, "utf8")).toBe(before); -}); - -describe("clearReleaseSelection", () => { - function fixture() { - const { infraDir, path } = releaseInfra("omx-release-clear-"); - const store = services.releaseStatePaths(infraDir).store; - mkdirSync(store, { mode: 0o700 }); - writeReleaseOverlay(JSON.parse(manifestJson), { infraDir }); - writeFileSync(join(store, "current.json"), "running-state"); - return { infraDir, path, store }; - } - - it("requires the exact checked release when clearing from a remote preview", async () => { - const { infraDir, path, store } = fixture(), - before = readFileSync(path, "utf8"); - await expect( - clearReleaseSelection({ infraDir, expectedReleaseId: "different-release" }), - ).rejects.toThrow(/changed/); - expect(readFileSync(path, "utf8")).toBe(before); - expect(await clearReleaseSelection({ infraDir, expectedReleaseId: "deadbeef" })).toEqual({ - path, - cleared: true, - }); - expect(readFileSync(join(store, "current.json"), "utf8")).toBe("running-state"); - await expect( - clearReleaseSelection({ infraDir, expectedReleaseId: "deadbeef" }), - ).rejects.toThrow(/changed/); - }); - it("clears only the local overlay idempotently and retains the running release state", async () => { - const { infraDir, path, store } = fixture(); - vi.stubEnv("OPENMAPX_RELEASE", "local"); - try { - expect(await clearReleaseSelection({ infraDir })).toEqual({ path, cleared: true }); - expect(await clearReleaseSelection({ infraDir })).toEqual({ path, cleared: false }); - expect(process.env.OPENMAPX_RELEASE).toBe("local"); - } finally { - vi.unstubAllEnvs(); - } - expect(existsSync(path)).toBe(false); - expect(readFileSync(join(store, "current.json"), "utf8")).toBe("running-state"); - expect(readdirSync(store)).toEqual(["current.json"]); - }); - - it("retains explicit host clearing of malformed overlays without reading unbounded bytes", async () => { - const { infraDir, path } = fixture(); - writeFileSync(path, Buffer.alloc(100000, 255)); - await expect(clearReleaseSelection({ infraDir })).resolves.toEqual({ path, cleared: true }); - }); - it("refuses a held or abandoned lock without changing it", async () => { - const { infraDir, path, store } = fixture(); - const lock = join(store, ".release-store.lock"); - mkdirSync(lock); - writeFileSync(join(lock, "owner.json"), '{"acquiredAtMs":0}'); - await expect(clearReleaseSelection({ infraDir })).rejects.toThrow(/busy/); - expect(existsSync(path)).toBe(true); - expect(readFileSync(join(lock, "owner.json"), "utf8")).toBe('{"acquiredAtMs":0}'); - }); - - it.each(["overlay", "store"])("refuses a symlinked %s", async (target) => { - const { infraDir, path, store } = fixture(); - const link = target === "overlay" ? path : store; - rmSync(link, { recursive: true }); - const outside = join(dirname(path), "outside"); - mkdirSync(outside); - writeFileSync(join(outside, "sentinel"), "retain"); - symlinkSync(outside, link); - await expect(clearReleaseSelection({ infraDir })).rejects.toThrow(/unsafe/i); - expect(readFileSync(join(outside, "sentinel"), "utf8")).toBe("retain"); - }); - - it("removes only the agent state's overlay, never a same-named file in infra/docker", async () => { - const { infraDir, path } = fixture(); - const sameName = join(infraDir, "docker-compose.release.yml"); - writeFileSync(sameName, "unrelated file\n", { mode: 0o600 }); - expect(await clearReleaseSelection({ infraDir })).toEqual({ path, cleared: true }); - expect(readFileSync(sameName, "utf8")).toBe("unrelated file\n"); - }); - - it("refuses an overlay that is group-writable or someone else's", async () => { - const { infraDir, path } = fixture(); - chmodSync(path, 0o620); - await expect(clearReleaseSelection({ infraDir })).rejects.toThrow(/unsafe/i); - chmodSync(path, 0o600); - // Tests cannot chown: inject the uid the state must belong to instead. - const foreign = (process.geteuid?.() ?? 0) + 1; - await expect(clearReleaseSelection({ infraDir, uid: foreign })).rejects.toThrow(/unsafe/i); - expect(existsSync(path)).toBe(true); +describe("the compose release command", () => { + it("exposes read-only status", () => { + const program = new Command(); + registerComposeCommands(program); + const release = program.commands + .find((command) => command.name() === "compose") + ?.commands.find((command) => command.name() === "release"); + expect(release?.options.map((option) => option.long)).toEqual( + expect.arrayContaining(["--status", "--check", "--clear"]), + ); }); it("rejects --clear together with --status before taking action", async () => { @@ -396,43 +156,14 @@ describe("clearReleaseSelection", () => { }); }); -it("retains the selected pin when the command cannot inspect the coordinator", async () => { - temp = mkdtempSync(join(tmpdir(), "omx-release-clear-command-")); - const directory = join(temp, "infra", "docker"); - mkdirSync(directory, { recursive: true }); - const path = services.releaseStatePaths(directory).overlay; - writeReleaseOverlay(JSON.parse(manifestJson), { infraDir: directory }); - vi.stubEnv("OPENMAPX_ROOT_DIR", temp); - vi.stubEnv("OPENMAPX_RELEASE", "invalid"); - vi.stubEnv("PATH", ""); - const output = vi.spyOn(console, "log").mockImplementation(() => {}); - const errors = vi.spyOn(console, "error").mockImplementation(() => {}); - const exit = vi.spyOn(process, "exit").mockImplementation(() => { - throw new Error("unexpected CLI failure"); - }); - try { - const program = new Command(); - registerComposeCommands(program); - const outcome = await program - .parseAsync(["compose", "release", "--clear"], { from: "user" }) - .then( - () => "accepted", - (error) => String(error), - ); - expect({ - outcome, - overlayPresent: existsSync(path), - output: output.mock.calls.flat(), - errors: errors.mock.calls.flat(), - }).toMatchObject({ outcome: "Error: unexpected CLI failure", overlayPresent: true }); - expect(errors.mock.calls.flat().join("\n")).toContain("Release lockfile operation failed"); - expect(process.env.OPENMAPX_RELEASE).toBe("invalid"); - } finally { - output.mockRestore(); - errors.mockRestore(); - exit.mockRestore(); - vi.unstubAllEnvs(); - } +it("pulls a fork's manifest image and accepts its image prefix", async () => { + temp = mkdtempSync(join(tmpdir(), "omx-release-fork-")); + const forkManifest = manifestJson.replaceAll("ghcr.io/openmapx", "registry.example.org/fork"); + const docker = fakeDocker(forkManifest); + vi.stubEnv("OPENMAPX_RELEASE_MANIFEST_REPOSITORY", "registry.example.org/fork/release-manifest"); + const manifest = await resolveReleaseManifest(docker, temp); + expect(manifest.images.api).toMatch(/^registry\.example\.org\/fork\/api@/); + expect(docker).toHaveBeenCalledWith(["pull", "registry.example.org/fork/release-manifest:main"]); }); it("extracts the manifest from the frozen local image ID after a mutable pointer moves", async () => { @@ -459,34 +190,6 @@ it("extracts the manifest from the frozen local image ID after a mutable pointer ]); }); -it.each(["group-writable", "foreign"] as const)( - "refuses a %s overlay for status, selection and start instead of reading it", - async (kind) => { - const { infraDir, path } = releaseInfra(); - writeReleaseOverlay(JSON.parse(manifestJson), { infraDir }); - const before = readFileSync(path, "utf8"); - if (kind === "group-writable") chmodSync(path, 0o620); - const uid = kind === "foreign" ? (process.geteuid?.() ?? 0) + 1 : undefined; - const docker = fakeDocker(); - expect(() => releaseStatusLines(path, { uid })).toThrow(); - await expect(ensureReleaseOverlay({ infraDir, docker, uid })).rejects.toThrow(); - const resolve = vi.fn(async () => JSON.parse(manifestJson)); - await expect(selectRelease({ infraDir, uid, resolve })).rejects.toThrow(); - expect(resolve).not.toHaveBeenCalled(); - expect(docker).not.toHaveBeenCalled(); - expect(readFileSync(path, "utf8")).toBe(before); - }, -); - -it("rejects a malformed existing overlay without replacing it or reaching the registry", async () => { - const { infraDir, path } = releaseInfra("omx-invalid-overlay-"); - writeFileSync(path, "services: {}\n", { mode: 0o600 }); - const docker = fakeDocker(); - await expect(ensureReleaseOverlay({ infraDir, docker })).rejects.toThrow(/malformed/); - expect(readFileSync(path, "utf8")).toBe("services: {}\n"); - expect(docker).not.toHaveBeenCalled(); -}); - it("uses the applied UI selector on each invocation without changing process environment", async () => { const { services } = await import("@openmapx/core/server"); temp = mkdtempSync(join(tmpdir(), "omx-applied-release-")); @@ -511,7 +214,7 @@ it("uses the applied UI selector on each invocation without changing process env }, }); const docker = fakeDocker(); - await expect(ensureReleaseOverlay({ docker })).resolves.toEqual({ status: "disabled" }); + await expect(releaseForStart({ docker })).resolves.toEqual({ manifest: null, resolved: false }); expect(docker).not.toHaveBeenCalled(); await services.commitConfigurationGeneration({ infraDir, diff --git a/packages/cli/__tests__/cli-service-lifecycle.test.ts b/packages/cli/__tests__/cli-service-lifecycle.test.ts index 7c307dfeb..ce21f6270 100644 --- a/packages/cli/__tests__/cli-service-lifecycle.test.ts +++ b/packages/cli/__tests__/cli-service-lifecycle.test.ts @@ -19,7 +19,6 @@ import { renderComposeForRepo } from "../src/commands/compose"; import { registerServicesCommands } from "../src/commands/services"; import { migrateBeforeApiUpdate } from "../src/lib/api-migrations"; import { dockerComposeStream } from "../src/lib/docker"; -import { selectRelease } from "../src/lib/release"; // These exercise fsynced cross-process authority/retention stores; concurrent // package checks can exceed the generic 15s unit budget on this filesystem. @@ -89,14 +88,6 @@ function expectConfigurationWriterFree() { expect(existsSync(join(root, "infra/docker/state/ops-agent/configuration/writer.lock"))).toBe( false, ); - expect( - existsSync( - join( - services.releaseStatePaths(join(root, "infra/docker")).store, - services.RELEASE_STORE_LOCK_NAME, - ), - ), - ).toBe(false); } async function commitCompetingGeneration() { services.writeServiceSelectionFile(join(root, "infra/docker"), ["alpha"]); @@ -404,7 +395,7 @@ describe("actual CLI service lifecycle preparation", () => { failure: { reason: "pull", failedServiceIds: ["alpha"] }, }); }); - it("refuses recreation when the release selection changes during pull", async () => { + it("refuses recreation when a new release is pinned during pull", async () => { const manifest = services.parseReleaseManifest( JSON.stringify({ schemaVersion: 1, @@ -420,21 +411,19 @@ describe("actual CLI service lifecycle preparation", () => { "ghcr.io/openmapx", ); const infraDir = join(root, "infra/docker"); - const path = services.releaseStatePaths(infraDir).overlay; + let pinned: string | undefined; vi.mocked(dockerComposeStream).mockImplementation(async () => { - await selectRelease({ - infraDir, - resolve: async () => manifest, - docker: async () => ({ stdout: "", stderr: "", exitCode: 0 }), - }); + pinned = ( + await renderComposeForRepo({ rootDir: root, domain: "example.test", release: manifest }) + ).revisionId; return 0; }); await expect(command("update")).rejects.toThrow("exit:1"); - expect(existsSync(path)).toBe(true); expect(vi.mocked(dockerComposeStream).mock.calls.map((call) => call[0])).toEqual([ ["pull", "alpha"], ]); - expect(services.currentConfigurationGeneration(join(root, "infra/docker"))).toBe(seed); + expect(services.currentConfigurationGeneration(infraDir)).toBe(pinned); + expect(services.readAppliedRelease(infraDir)?.manifest.release).toBe("synthetic-next"); expect(await retainedReceipt()).toMatchObject({ phase: "failed", finalRevisionId: null, @@ -592,11 +581,12 @@ describe("actual CLI service lifecycle preparation", () => { }); const diagnostics = JSON.stringify(vi.mocked(console.error).mock.calls); expect(diagnostics).toContain( - `recover-update --operation-key ${receipt.execution.operationKey}`, + `openmapx control resolve --op ${receipt.execution.operationKey} --as committed|failed`, ); - expect(diagnostics).not.toContain("No automated release exists"); + expect(diagnostics).toContain("infra/docker/state/ops-agent/lifecycle/"); + expect(diagnostics).not.toContain("recover-update"); }); - it("does not offer recover-update when Start fails after its hardlink child committed", async () => { + it("points to control resolve when Start fails after its hardlink child committed", async () => { await renderWithHardlinkPlan(); const stop = await unreachableHardlinkExecutor(); vi.mocked(control.executeOwnedGenerationHardlinks).mockImplementationOnce( @@ -629,10 +619,10 @@ describe("actual CLI service lifecycle preparation", () => { }); const diagnostics = JSON.stringify(vi.mocked(console.error).mock.calls); expect(diagnostics).toContain(`Operation ${receipt.execution.operationKey} keeps`); - expect(diagnostics).toContain("a hardlink child was issued to data-manager"); - expect(diagnostics).toContain("No automated release exists yet"); - expect(diagnostics).not.toContain("recover-update --operation-key"); - expect(diagnostics).not.toContain("may still be running"); + expect(diagnostics).toContain( + `openmapx control resolve --op ${receipt.execution.operationKey} --as committed|failed`, + ); + expect(diagnostics).not.toContain("recover-update"); }); it("a lease held by a live operation still refuses a second start", async () => { let refusal: unknown; @@ -651,29 +641,16 @@ describe("actual CLI service lifecycle preparation", () => { expect(targetLocks()).toEqual([]); }); it("releases every target when the lifecycle fails before its receipt opens", async () => { - const paths = services.releaseStatePaths(join(root, "infra/docker")); - services.ensureReleaseStateDirectory(paths.directory); - services.ensureReleaseStateDirectory(paths.store); - const lock = await services.acquireReleaseStoreLock(paths.store, {}, { failIfBusy: true }); + const release = await services.acquireConfigurationOwnership(join(root, "infra/docker")); try { - await expect(command("start")).rejects.toThrow("Release store is busy"); + await expect(command("start")).rejects.toThrow(/Configuration writer is busy/); } finally { - lock.release(); + release(); } expect(dockerComposeStream).not.toHaveBeenCalled(); expect(targetLocks()).toEqual([]); await expect(command("start")).rejects.toThrow("exit:0"); }); - it("waits for a momentary release-store holder once containers have started", async () => { - vi.mocked(dockerComposeStream).mockImplementationOnce(async () => { - const paths = services.releaseStatePaths(join(root, "infra/docker")); - const lock = await services.acquireReleaseStoreLock(paths.store, {}, { failIfBusy: true }); - setTimeout(() => lock.release(), 300); - return 0; - }); - await expect(command("start")).rejects.toThrow("exit:0"); - expect(await retainedReceipt()).toMatchObject({ phase: "complete", followUp: "complete" }); - }); it("refuses to replace services when the generation moves during the migration", async () => { const directory = join(root, "services/app-api"); mkdirSync(directory, { recursive: true }); diff --git a/packages/cli/__tests__/cli-services-update.test.ts b/packages/cli/__tests__/cli-services-update.test.ts index 80ca6ac8d..a99167723 100644 --- a/packages/cli/__tests__/cli-services-update.test.ts +++ b/packages/cli/__tests__/cli-services-update.test.ts @@ -9,7 +9,6 @@ import { renderComposeForRepo, renderComposeWithWriter } from "../src/commands/c import { registerServicesCommands } from "../src/commands/services"; import { migrateBeforeApiUpdate } from "../src/lib/api-migrations"; import { dockerComposeStream } from "../src/lib/docker"; -import { ensureReleaseOverlay } from "../src/lib/release"; vi.setConfig({ testTimeout: 60000 }); @@ -40,10 +39,6 @@ vi.mock("../src/lib/api-migrations", () => ({ API_SERVICE_ID: "app-api", migrateBeforeApiUpdate: vi.fn(async () => 0), })); -vi.mock("../src/lib/release", async (importOriginal) => ({ - ...(await importOriginal()), - ensureReleaseOverlay: vi.fn(), -})); let root: string, seed: string; beforeEach(async () => { @@ -78,7 +73,6 @@ beforeEach(async () => { services.writeServiceSelectionFile(join(root, "infra/docker"), ids); seed = (await renderComposeForRepo({ rootDir: root, domain: "example.test" })).revisionId; vi.mocked(dockerComposeStream).mockReset().mockResolvedValue(0); - vi.mocked(ensureReleaseOverlay).mockReset(); vi.mocked(migrateBeforeApiUpdate).mockReset().mockResolvedValue(0); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); @@ -100,24 +94,15 @@ function update(...ids: string[]) { } describe("services update pull failures", () => { - it.each(["present", "resolved"] as const)( - "stops before recreation when a %s release cannot be pulled", - async (status) => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ - status, - path: "release.yml", - release: "abc", - }); - vi.mocked(dockerComposeStream).mockResolvedValueOnce(7); - await expect(update("ops-agent", "transitous-runner")).rejects.toThrow("exit:7"); - expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([ - [["pull", "ops-agent", "transitous-runner"]], - ]); - }, - ); + it("stops before recreation when a release cannot be pulled", async () => { + vi.mocked(dockerComposeStream).mockResolvedValueOnce(7); + await expect(update("ops-agent", "transitous-runner")).rejects.toThrow("exit:7"); + expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([ + [["pull", "ops-agent", "transitous-runner"]], + ]); + }); it("recreates a pinned release after a successful pull", async () => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); await expect(update("ops-agent")).rejects.toThrow("exit:0"); expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([ [["pull", "ops-agent"]], @@ -126,7 +111,6 @@ describe("services update pull failures", () => { }); it("passes --no-deps only to ordinary target recreation", async () => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); await expect(update("--no-deps", "app-web", "transitous-runner", "app-api")).rejects.toThrow( "exit:0", ); @@ -137,17 +121,13 @@ describe("services update pull failures", () => { }); it("refuses an explicitly unpinned local image after a failed pull", async () => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "disabled" }); vi.mocked(dockerComposeStream).mockResolvedValueOnce(7); await expect(update("ops-agent")).rejects.toThrow("exit:7"); expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "ops-agent"]]]); }); - it("does not pull or recreate when release resolution fails", async () => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ - status: "unpinned", - reason: "unavailable", - }); + it("does not pull or recreate release images the applied configuration does not pin", async () => { + vi.stubEnv("OPENMAPX_RELEASE", "main"); await expect(update("ops-agent")).rejects.toThrow("exit:1"); expect(dockerComposeStream).not.toHaveBeenCalled(); }); @@ -155,25 +135,10 @@ describe("services update pull failures", () => { it("refuses recreation of a locally built service after a failed pull", async () => { vi.mocked(dockerComposeStream).mockResolvedValueOnce(7); await expect(update("valhalla")).rejects.toThrow("exit:7"); - expect(ensureReleaseOverlay).not.toHaveBeenCalled(); expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "valhalla"]]]); }); - it("validates pins and pulls before full generation preparation, hardlinks and recreation", async () => { + it("pulls before full generation preparation, hardlinks and recreation", async () => { const events: string[] = []; - vi.mocked(ensureReleaseOverlay).mockImplementation(async (options) => { - expect(options?.releaseStoreLock).toEqual( - expect.objectContaining({ release: expect.any(Function) }), - ); - await expect( - services.acquireReleaseStoreLock( - services.releaseStatePaths(join(root, "infra/docker")).store, - {}, - { failIfBusy: true }, - ), - ).rejects.toThrow(/busy/); - events.push("pins"); - return { status: "present", path: "release.yml" }; - }); vi.mocked(dockerComposeStream).mockImplementation(async (args) => { const current = services.currentConfigurationGeneration(join(root, "infra/docker")); if (args[0] === "pull") { @@ -197,13 +162,12 @@ describe("services update pull failures", () => { return 0; }); await expect(update("ops-agent")).rejects.toThrow("exit:0"); - expect(events).toEqual(["pins", "pull", "recreate"]); + expect(events).toEqual(["pull", "recreate"]); }); }); describe("services update and the operations agent", () => { it("does not recreate when rendering after the pull cannot resolve the deployment user", async () => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); vi.mocked(renderComposeWithWriter).mockRejectedValueOnce( new Error( "The checkout /srv/openmapx is owned by root, so the deployment's user cannot be taken from it: set UID and GID in /srv/openmapx/infra/docker/.env to the account that runs the stack", @@ -216,10 +180,6 @@ describe("services update and the operations agent", () => { }); describe("services update and app-api's database migrations", () => { - beforeEach(() => { - vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); - }); - it("migrates after the pull and before recreation when app-api is replaced", async () => { const order: string[] = []; vi.mocked(dockerComposeStream).mockImplementation(async (args) => { diff --git a/packages/cli/__tests__/cli-system-update.test.ts b/packages/cli/__tests__/cli-system-update.test.ts index beb22074c..6f3a3cd93 100644 --- a/packages/cli/__tests__/cli-system-update.test.ts +++ b/packages/cli/__tests__/cli-system-update.test.ts @@ -17,7 +17,6 @@ afterEach(async () => { async function fixture(mode = "success") { const root = mkdtempSync(join(tmpdir(), "openmapx-update-client-")); cleanup.push(() => rmSync(root, { recursive: true, force: true })); - const paths = services.ensureReleaseStateDirectories(root); vi.stubEnv("OPENMAPX_RELEASE", "main"); vi.stubEnv("OPENMAPX_LOCAL_ADMIN_TOKEN", "must-never-be-sent"); const manifest: services.ReleaseManifest = { @@ -31,17 +30,7 @@ async function fixture(mode = "success") { ) as services.ReleaseManifest["images"], ...releaseProvenance(), }; - const lock = await services.acquireReleaseStoreLock(paths.store, {}, { failIfBusy: true }); - const checked = services.cacheCheckedReleaseManifest({ - infraDir: root, - manifest, - ociDigest: `sha256:${"c".repeat(64)}`, - configurationIdentity: services.releaseConfigurationIdentity( - services.readAppliedReleaseSelection(root), - ), - lock, - }); - lock.release(); + const checked = services.cacheReleaseManifest(root, manifest); const sessionFile = join(root, "session"); writeFileSync(sessionFile, "better-auth.session_token=synthetic.signed", { mode: 0o600 }); const requests: Array<{ diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index 2c5121724..b09a716b8 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -31,13 +31,12 @@ import { } from "../lib/platform-secret-files"; import { combineServiceSelection } from "../lib/preset-selection"; import { + assertReleaseClearable, checkRelease, - clearReleaseSelection, - ensureReleaseOverlay, + releaseForStart, releaseStatusLines, selectRelease, touchesReleasePinnedServices, - unpinnedReleaseWarning, } from "../lib/release"; import { selectedReleaseServiceFiles } from "../lib/release-service-files"; import { applyServiceSelection } from "../lib/service-selection"; @@ -62,6 +61,10 @@ export interface RenderRepoOptions { expectedCurrentRevisionId?: string | null; revisionId?: string; redisAuthHooks?: RedisAuthReconciliationHooks; + /** The release to pin; omitted, the applied generation's pins are kept. */ + release?: coreServices.ReleaseManifest | null; + /** A retained generation whose saved configuration to render instead of the applied one's. */ + configurationFrom?: string; } export interface RenderRepoResult { @@ -318,7 +321,9 @@ export async function renderComposeWithWriter( : loadIntegrationSchemas(paths.root); // The database is the admin panel's: what it saved arrives through the // generation it last applied. The env layer is this host's. - const saved = readAppliedConfiguration(paths.infraDir); + const saved = opts.configurationFrom + ? coreServices.readGenerationConfiguration(paths.infraDir, opts.configurationFrom) + : readAppliedConfiguration(paths.infraDir); const savedConfigs = new Map(saved.serviceConfigs.map((entry) => [entry.serviceId, entry])); const serviceConfigs = services.flatMap((service) => { const { id } = service.manifest; @@ -354,13 +359,17 @@ export async function renderComposeWithWriter( return [{ serviceId: id, values, ...(envKeys.length > 0 ? { envKeys } : {}) }]; }); const mounts = await containerBindMounts(); - // Run from the checkout with a release selected, render uses that - // release's service files, read from its image, and refuses while the - // checkout's built-in services differ from them. - const selected = await selectedReleaseServiceFiles(paths.infraDir); + // Run from the checkout with a release pinned, render uses that release's + // service files, read from its image, and refuses while the checkout's + // built-in services differ from them. + const release = + opts.release === undefined + ? (coreServices.readAppliedRelease(paths.infraDir)?.manifest ?? null) + : opts.release; + const selected = await selectedReleaseServiceFiles(release); if (selected === "none") { log.dim( - "No release is selected yet: the services' files come from this checkout. Bootstrap can select with `pnpm openmapx compose release`; existing installations use System → Updates or `system update`.", + "No release is pinned: the services' files come from this checkout. Select one with `pnpm openmapx compose release`.", ); } else if (selected) { const overrides = coreServices.readServiceOverrides(paths.infraDir, selected.files); @@ -383,6 +392,7 @@ export async function renderComposeWithWriter( } assertCliExecutionAvailable(); const committed = await writer.commit({ + release, ...(mounts ? { containerBindMounts: mounts } : {}), ...(selected && selected !== "none" ? { releaseServiceFiles: selected.files } : {}), services, @@ -498,7 +508,16 @@ export function registerComposeCommands(program: Command): void { let enabledServiceIds: string[] = []; try { const services = combineServiceSelection(options.services, options.preset); - const r = await renderComposeForRepo({ domain: options.domain, services }); + const release = await releaseForStart(); + if (release.resolved && release.manifest) + log.ok(`Pinned release ${release.manifest.release}`); + else if (!release.manifest) + log.dim("Release pinning disabled (OPENMAPX_RELEASE=local); using manifest image tags."); + const r = await renderComposeForRepo({ + domain: options.domain, + services, + release: release.manifest, + }); enabledServiceIds = r.enabledServiceIds; log.ok(`Rendered ${r.servicesRendered} services → ${r.revisionId}`); for (const warning of r.selectionWarnings) log.warn(warning); @@ -508,20 +527,6 @@ export function registerComposeCommands(program: Command): void { log.err(`Render failed: ${(err as Error).message}`); process.exit(1); } - const overlay = await ensureReleaseOverlay(); - if (overlay.status === "resolved") { - log.ok(`Atomic release selection ${overlay.release} → ${overlay.path}`); - } else if (overlay.status === "present") { - for (const line of releaseStatusLines(overlay.path)) log.dim(line); - } else if (overlay.status === "unpinned") { - log.err(unpinnedReleaseWarning(overlay.reason)); - process.exit(1); - } else if (overlay.status === "held") { - log.err(overlay.reason); - process.exit(1); - } else if (overlay.status === "disabled") { - log.dim("Release pinning disabled (OPENMAPX_RELEASE=local); using manifest image tags."); - } if (enabledServiceIds.length === 0) { log.dim("No enabled services to start"); process.exit(0); @@ -535,20 +540,17 @@ export function registerComposeCommands(program: Command): void { compose .command("release") - .description( - "Resolve the release lockfile for atomic release selection in docker-compose.release.yml", - ) - .option("--status", "Show locally selected release and image pins without registry access") + .description("Pin the release the configured selection resolves to in a new configuration") + .option("--status", "Show the applied configuration's release pins without registry access") .addOption( - new Option( - "--check", - "Check and cache release metadata without changing selected pins", - ).conflicts(["status", "clear"]), + new Option("--check", "Check and cache release metadata without changing the pins").conflicts( + ["status", "clear"], + ), ) .addOption( new Option( "--clear", - "Clear an unused or mismatched release selection; refuses a matching Data Manager", + "Render the configuration without release pins; refuses while Data Manager runs the pinned image", ).conflicts(["status", "check"]), ) .action(async (options: { status?: boolean; clear?: boolean; check?: boolean }) => { @@ -564,31 +566,28 @@ export function registerComposeCommands(program: Command): void { log.info(`Metadata image digest: ${checked.ociDigest}`); for (const name of coreServices.RELEASE_IMAGE_NAMES) log.info(`${name}: ${checked.manifest.images[name]}`); - log.dim("Selected pins and running services were not changed."); + log.dim("The pins and running services were not changed."); return; } if (options.clear) { - const result = await clearReleaseSelection(); - log.ok( - result.cleared - ? `Cleared local release selection → ${result.path}` - : "No local release selection to clear.", - ); - log.dim("Running containers and recorded running state are preserved."); - log.dim( - process.env.OPENMAPX_RELEASE === "local" - ? "Release resolution is disabled; the next start/update uses manifest image tags." - : "Automatic release resolution will select a release on the next start/update. To use local images, separately set OPENMAPX_RELEASE=local.", - ); + const cleared = await assertReleaseClearable(); + if (!cleared) { + log.ok("The applied configuration pins no release."); + return; + } + const r = await renderComposeForRepo({ release: null }); + log.ok(`Cleared the pins of ${cleared} → ${r.revisionId}`); + log.dim("Running containers keep their images until they are next recreated."); return; } - const selected = await selectRelease({ report: log.info }); - log.ok(`Atomic release selection ${selected.release} → ${selected.path}`); + const manifest = await selectRelease({ report: log.info }); + const r = await renderComposeForRepo({ release: manifest }); + log.ok(`Pinned release ${manifest.release} → ${r.revisionId}`); log.dim( - "Bootstrap: start the selected release with `pnpm openmapx compose up`. Existing installations: use System → Updates or authenticated `system update` after `compose release --check`.", + "Start it with `pnpm openmapx compose up`. Existing installations update through System → Updates or authenticated `system update`.", ); } catch (err) { - log.err(`Release lockfile operation failed: ${(err as Error).message}`); + log.err(`Release operation failed: ${(err as Error).message}`); process.exit(1); } }); @@ -609,9 +608,8 @@ export function registerComposeCommands(program: Command): void { .description("Pull selected images (no args = all services)") .action(async (ids: string[]) => { if (ids.length === 0 || touchesReleasePinnedServices(ids)) { - const overlay = await ensureReleaseOverlay(); - if (overlay.status === "unpinned") throw new Error(unpinnedReleaseWarning(overlay.reason)); - if (overlay.status === "held") throw new Error(overlay.reason); + const release = await releaseForStart(); + if (release.resolved) await renderComposeForRepo({ release: release.manifest }); } const code = await dockerComposeStream(["pull", ...ids]); process.exit(code); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index 9ca716b96..edcce89fa 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -11,9 +11,16 @@ import { services } from "@openmapx/core/server"; import type { Command } from "commander"; import { completeCliCommand } from "../lib/control-context"; import { settleMaintenanceFence } from "../lib/control-resolve"; -import { dockerCompose } from "../lib/docker"; +import { dockerCompose, dockerComposeStream } from "../lib/docker"; +import { applyGeneratedHardlinksBeforeStart } from "../lib/hardlinks"; +import { leasedHostEffect } from "../lib/host-effect"; import { log } from "../lib/output"; import { repoPaths } from "../lib/paths"; +import { + assertDatabaseAllowsRollback, + planRollback, + readNewestAppliedMigration, +} from "../lib/rollback"; /** * The operator exit for the update system: what the agent's store believes is @@ -260,4 +267,63 @@ export function registerControlCommands(program: Command): void { audit({ action: "resolve", operationKey: options.op, as: options.as, lease }); completeCliCommand(0); }); + + control + .command("rollback") + .description( + "Return to the configuration applied before the current one, with the release it pins", + ) + .option("--yes", "Roll back; without it the plan is printed and nothing changes") + .action(async (options: { yes?: boolean }) => { + const infraDir = repoPaths().infraDir; + const plan = planRollback(infraDir); + log.info( + `Applied configuration: ${plan.currentRevisionId} (release ${plan.current?.manifest.release ?? "none"})`, + ); + log.info( + `Previous configuration: ${plan.targetRevisionId} (release ${plan.target?.manifest.release ?? "none"})`, + ); + log.warn( + "Rolling back also reverts every configuration change made since then: the release selection, the selected services, service configuration and secrets return to the previous configuration's.", + ); + const sameRelease = plan.current?.manifestDigest === plan.target?.manifestDigest; + assertDatabaseAllowsRollback(plan, sameRelease ? null : await readNewestAppliedMigration()); + if (!options.yes) { + log.info("Nothing was changed. Run again with --yes to roll back."); + completeCliCommand(0); + } + const { renderComposeForRepo } = await import("./compose"); + const target = services.readGenerationConfiguration(infraDir, plan.targetRevisionId); + const code = await leasedHostEffect("control.rollback", "all", async () => { + const rendered = await renderComposeForRepo({ + services: target.selectedRoots, + release: target.release, + configurationFrom: plan.targetRevisionId, + expectedCurrentRevisionId: plan.currentRevisionId, + }); + log.ok(`Rendered the previous configuration again → ${rendered.revisionId}`); + // Later renders keep the restored selection. + if (services.readDesiredSelection(infraDir).source !== "env") + services.writeServiceSelectionFile(infraDir, target.selectedRoots); + await applyGeneratedHardlinksBeforeStart(); + // The self-update order: the database before its clients, the agent last. + for (const id of services.selfUpdateServiceOrder(rendered.enabledServiceIds)) { + const exit = await dockerComposeStream(["up", "-d", "--no-deps", id]); + if (exit !== 0) { + log.err( + `Recreating ${id} failed (exit ${exit}); the services after it were not touched.`, + ); + return exit; + } + } + return 0; + }); + audit({ + action: "rollback", + from: plan.currentRevisionId, + to: plan.targetRevisionId, + exitCode: code, + }); + completeCliCommand(code); + }); } diff --git a/packages/cli/src/lib/build-artifacts.ts b/packages/cli/src/lib/build-artifacts.ts index a3879951f..618162ea8 100644 --- a/packages/cli/src/lib/build-artifacts.ts +++ b/packages/cli/src/lib/build-artifacts.ts @@ -113,10 +113,7 @@ export async function buildProvenance( const paths = repoPaths(rootDir); let release: string | undefined; try { - const overlay = services.readReleaseOverlay(paths.infraDir); - release = - (overlay === null ? null : services.parseReleaseComposeSelection(overlay).release) ?? - undefined; + release = services.readAppliedRelease(paths.infraDir)?.manifest.release; } catch { release = undefined; } diff --git a/packages/cli/src/lib/motis-data.ts b/packages/cli/src/lib/motis-data.ts index 4e55f4a60..feeacc643 100644 --- a/packages/cli/src/lib/motis-data.ts +++ b/packages/cli/src/lib/motis-data.ts @@ -65,21 +65,17 @@ export const DEFAULT_TRANSITOUS_TOOLS_IMAGE = process.env.OPENMAPX_TRANSITOUS_TOOLS_IMAGE ?? "ghcr.io/openmapx/transitous-tools:main"; /** - * Resolve the helper image at call time: explicit env wins, then the digest - * pinned by an existing `docker-compose.release.yml` overlay (so host-side - * pipeline runs stay on the same release as the data-manager container), and - * only then the historical `:latest` tag. + * Resolve the helper image at call time: the digest the applied generation + * pins (so host-side pipeline runs stay on the same release as the + * data-manager container), else an explicit env value, and only then the + * historical `:latest` tag. */ export function resolveTransitousToolsImage(rootDir?: string): string { const fromEnv = process.env.OPENMAPX_TRANSITOUS_TOOLS_IMAGE; const selection = coreServices.readAppliedReleaseSelection(repoPaths(rootDir).infraDir); - const overlay = coreServices.readReleaseOverlay(repoPaths(rootDir).infraDir); - if (overlay !== null) { - const selected = coreServices.parseReleaseComposeSelection(overlay); - - const pinned = selected.images["transitous-tools"]; - if (!selected.release || !pinned) - throw new Error("Release overlay is malformed; explicitly reselect or clear it"); + const applied = coreServices.readAppliedRelease(repoPaths(rootDir).infraDir); + if (applied) { + const pinned = applied.manifest.images["transitous-tools"]; if (fromEnv && fromEnv !== pinned) throw new Error("Transitous helper override conflicts with selected release"); return pinned; diff --git a/packages/cli/src/lib/release-service-files.ts b/packages/cli/src/lib/release-service-files.ts index 947cccbad..e97d39890 100644 --- a/packages/cli/src/lib/release-service-files.ts +++ b/packages/cli/src/lib/release-service-files.ts @@ -1,10 +1,10 @@ import { services as coreServices } from "@openmapx/core/server"; import { execa } from "execa"; -/** The selected release's service files, as its operations-agent image embeds them. */ +/** A release's service files, as its operations-agent image embeds them. */ export interface SelectedReleaseServiceFiles { files: coreServices.ReleaseServiceFiles; - release: string | null; + release: string; image: string; /** The source revision the image was built from, from its OCI label. */ revision: string | null; @@ -14,27 +14,22 @@ export interface SelectedReleaseServiceFiles { export const RELEASE_FILES_COMMAND = ["node", "dist/cli.mjs", "compose", "release-files"]; /** - * The service files a host render uses: those of the release the overlay - * selects, read from its pinned operations-agent image (the host CLI runs + * The service files a host render uses: those of `release`, the release the + * render pins, read from its pinned operations-agent image (the host CLI runs * the checkout's code, which is not the release). `null` when this process * is the release's own CLI, or under `OPENMAPX_RELEASE=local`, where the - * checkout's files apply; `"none"` before a release is selected. + * checkout's files apply; `"none"` when no release is pinned. */ export async function selectedReleaseServiceFiles( - infraDir: string, + release: coreServices.ReleaseManifest | null, ): Promise { if (coreServices.readEmbeddedReleaseServiceFiles()) return null; if (coreServices.releaseChannel().kind === "disabled") return null; - const overlay = coreServices.readReleaseOverlayFile( - coreServices.releaseStatePaths(infraDir).overlay, - ); - if (overlay === null) return "none"; - const selection = coreServices.parseReleaseComposeSelection(overlay); - const image = selection.images["ops-agent"]; - if (!image) throw new Error("The release overlay pins no operations-agent image"); + if (release === null) return "none"; + const image = release.images["ops-agent"]; const unreadable = (reason: string) => new Error( - `The selected release's service files could not be read from its operations-agent image ${image}: ${reason}. Render uses the release's files, so it stops here. If the selected release predates this checkout, use System → Updates or authenticated system update after compose release --check. Bootstrap can select with compose release.`, + `The selected release's service files could not be read from its operations-agent image ${image}: ${reason}. Render uses the release's files, so it stops here.`, ); const run = await execa( "docker", @@ -69,7 +64,7 @@ export async function selectedReleaseServiceFiles( const revision = String(inspected.stdout ?? "").trim(); return { files, - release: selection.release, + release: release.release, image, revision: inspected.exitCode === 0 && /^[a-f0-9]{40}$/.test(revision) ? revision : null, }; diff --git a/packages/cli/src/lib/release.ts b/packages/cli/src/lib/release.ts index a723a592a..40ecc264c 100644 --- a/packages/cli/src/lib/release.ts +++ b/packages/cli/src/lib/release.ts @@ -1,16 +1,6 @@ -import { createHash } from "node:crypto"; -import { - closeSync, - constants, - fstatSync, - lstatSync, - mkdtempSync, - openSync, - readSync, - rmSync, -} from "node:fs"; +import { closeSync, constants, fstatSync, mkdtempSync, openSync, readSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { join } from "node:path"; import { services as coreServices } from "@openmapx/core/server"; import { execa } from "execa"; import { cliProcessOptions } from "./execution-deadline"; @@ -21,24 +11,10 @@ const { RELEASE_SELECTOR_ENV, RELEASE_PINNED_SERVICE_IDS, parseReleaseManifest, - writeReleaseComposeArtifacts, } = coreServices; export type ReleaseManifest = coreServices.ReleaseManifest; - -/** - * Where the release selection lives: the agent's release state under - * `infra/docker`, shared with the host CLI, which runs as the agent's user. - * `uid` is the owner that state must have; tests inject another. - */ -export interface ReleaseStateLocation { - infraDir?: string; - uid?: number; -} - -function releaseState(opts: ReleaseStateLocation) { - return coreServices.releaseStatePaths(opts.infraDir ?? repoPaths().infraDir); -} +const MAX_RELEASE_MANIFEST_BYTES = 32 * 1024; function appliedReleaseChannel( infraDir = repoPaths().infraDir, @@ -122,7 +98,10 @@ async function inspectCoordinator(docker: ReleaseDockerRunner) { return { imageId: row.Image, reference: row.ConfigImage }; } -async function assertCoordinatorSelection(manifest: ReleaseManifest, docker: ReleaseDockerRunner) { +export async function assertCoordinatorSelection( + manifest: ReleaseManifest, + docker: ReleaseDockerRunner, +) { const current = await inspectCoordinator(docker); if (!current) return; const reference = manifest.images["data-manager"]; @@ -143,7 +122,7 @@ function readManifestFromPrivateTemp(path: string): string { !before.isFile() || before.nlink !== 1 || before.size < 1 || - before.size > coreServices.RELEASE_STORE_LIMITS.manifestBytes + before.size > MAX_RELEASE_MANIFEST_BYTES ) throw Error("Release manifest exceeds its file bound"); const bytes = Buffer.alloc(before.size); @@ -226,251 +205,110 @@ export async function resolveReleaseManifest( return manifest; } -/** Cache a checked candidate; never publish an overlay or recorded running state. */ -export async function checkRelease( - opts: { docker?: ReleaseDockerRunner } & ReleaseStateLocation = {}, -) { +/** Check a candidate and cache its manifest; the pins and running services stay as they are. */ +export async function checkRelease(opts: { docker?: ReleaseDockerRunner; infraDir?: string } = {}) { const infraDir = opts.infraDir ?? repoPaths().infraDir; - const paths = coreServices.ensureReleaseStateDirectories(infraDir, { uid: opts.uid }); const selection = coreServices.readAppliedReleaseSelection(infraDir); if (selection.kind === "local") throw new coreServices.ReleaseChannelDisabledError(); - const configurationIdentity = coreServices.releaseConfigurationIdentity( - coreServices.readAppliedReleaseSelection(infraDir), - ); const candidate = await resolveReleaseCandidate(opts.docker, infraDir); - const lock = await coreServices.acquireReleaseStoreLock(paths.store, {}, { failIfBusy: true }); - try { - if ( - coreServices.releaseConfigurationIdentity( - coreServices.readAppliedReleaseSelection(infraDir), - ) !== configurationIdentity - ) - throw Error("Release configuration changed during check; check again"); - const cached = coreServices.cacheCheckedReleaseManifest({ - infraDir, - ...candidate, - configurationIdentity, - lock, - uid: opts.uid, - }); - return { - ...candidate, - ...cached, - configurationIdentity, - configuredSelector: selection.selector, - }; - } finally { - lock.release(); - } -} - -/** Write the overlay atomically into the agent's release state. */ -export function writeReleaseOverlay(manifest: ReleaseManifest, opts: ReleaseStateLocation = {}) { - return writeReleaseComposeArtifacts(manifest, opts.infraDir ?? repoPaths().infraDir, { - uid: opts.uid, - }).overlayPath; + const cached = coreServices.cacheReleaseManifest(infraDir, candidate.manifest); + return { ...candidate, ...cached, configuredSelector: selection.selector }; } export function touchesReleasePinnedServices(serviceIds: readonly string[]): boolean { return serviceIds.some((id) => (RELEASE_PINNED_SERVICE_IDS as readonly string[]).includes(id)); } -export type ReleaseOverlayState = - | { status: "present"; path: string } - | { status: "resolved"; path: string; release: string } - | { status: "disabled" } - | { status: "unpinned"; reason: string } - | { status: "held"; reason: string }; - /** - * Make sure `docker-compose.release.yml` exists before a command that would - * otherwise start release runtime images from their manifest `:latest` tags - * without the release lockfile's atomic release selection. Never - * overwrites an existing overlay — the admin updater and `compose release` - * own deliberate release changes. + * The release a start renders: the applied generation's pins, else, at + * bootstrap, the release the configured selection resolves to now. Null under + * `OPENMAPX_RELEASE=local`. Throws when release images would start without + * pins. */ -export async function ensureReleaseOverlay( - opts: { - docker?: ReleaseDockerRunner; - releaseStoreLock?: coreServices.ReleaseStoreLock; - } & ReleaseStateLocation = {}, -): Promise { +export async function releaseForStart( + opts: { docker?: ReleaseDockerRunner; infraDir?: string } = {}, +): Promise<{ manifest: ReleaseManifest | null; resolved: boolean }> { const infraDir = opts.infraDir ?? repoPaths().infraDir; - const path = releaseState(opts).overlay; - const selected = readSelection(path, opts); - if (selected) { - assertCompatibleSelection(selected, infraDir); - return { status: "present", path }; - } - // An operator running local images opts out once instead of paying a - // failed registry pull (and a warning) on every start. - if (appliedReleaseChannel(infraDir).kind === "disabled") return { status: "disabled" }; - let acquired: coreServices.ReleaseStoreLock | undefined; + const applied = coreServices.readAppliedRelease(infraDir); + if (applied) return { manifest: applied.manifest, resolved: false }; + if (appliedReleaseChannel(infraDir).kind === "disabled") + return { manifest: null, resolved: false }; + let manifest: ReleaseManifest; try { - const paths = releaseState(opts); - coreServices.ensureReleaseStateDirectory(paths.store, { uid: opts.uid }); - if (opts.releaseStoreLock) - coreServices.assertReleaseStoreLock(opts.releaseStoreLock, paths.store); - else - acquired = await coreServices.acquireReleaseStoreLock(paths.store, {}, { failIfBusy: true }); - // Up/pull may have observed absence before another owner selected a release. - const current = readSelection(path, opts); - if (current) { - assertCompatibleSelection(current, infraDir); - return { status: "present", path }; - } - let manifest: ReleaseManifest; - try { - manifest = await resolveReleaseManifest(opts.docker, infraDir); - } catch (error) { - return { status: "unpinned", reason: (error as Error).message }; - } - await assertCoordinatorSelection(manifest, opts.docker ?? defaultDocker); - const activeLock = opts.releaseStoreLock ?? acquired; - if (!activeLock) throw Error("Release store lock unavailable"); - coreServices.assertReleaseStoreLock(activeLock, paths.store); - writeReleaseOverlay(manifest, opts); - return { status: "resolved", path, release: manifest.release }; + manifest = await resolveReleaseManifest(opts.docker, infraDir); } catch (error) { - return { status: "held", reason: (error as Error).message }; - } finally { - acquired?.release(); + throw new Error(unpinnedReleaseWarning((error as Error).message)); } + await assertCoordinatorSelection(manifest, opts.docker ?? defaultDocker); + coreServices.cacheReleaseManifest(infraDir, manifest); + return { manifest, resolved: true }; } export function unpinnedReleaseWarning(reason: string): string { return ( - `No ${repoPaths().composeReleasePath} and the release lockfile could not be resolved (${reason}). ` + - "Refusing to start release runtime images without atomic release selection. " + - "For bootstrap, run `pnpm openmapx compose release` once the registry is reachable. Existing installations use System → Updates or authenticated system update after compose release --check. Alternatively set " + + `The applied configuration pins no release and the release manifest could not be resolved (${reason}). ` + + "Refusing to start release images without pins. " + + "Select a release with `pnpm openmapx compose release` once the registry is reachable, or set " + `${RELEASE_SELECTOR_ENV}=local to run local images deliberately.` ); } -/** The overlay's selection; an overlay anyone but the agent's user could write is refused. */ -function readSelection( - path: string, - opts: ReleaseStateLocation, -): coreServices.ReleaseComposeSelection | null { - const raw = coreServices.readReleaseOverlayFile(path, { uid: opts.uid }); - return raw === null ? null : coreServices.parseReleaseComposeSelection(raw); -} - -function assertCompatibleSelection( - selected: coreServices.ReleaseComposeSelection, - infraDir: string, -) { - const channel = appliedReleaseChannel(infraDir); - const prefix = - channel.kind === "enabled" ? channel.imagePrefix : selected.images.api?.split("/api@")[0]; - if ( - !selected.release || - !prefix || - !coreServices.RELEASE_IMAGE_NAMES.every((name) => - selected.images[name]?.startsWith(`${prefix}/${name}@`), - ) - ) - throw Error( - "Existing release overlay is malformed or belongs to another configured repository; explicitly reselect or clear it", - ); -} - -/** Read-only local status; an existing overlay still applies when resolution is disabled. */ -export function releaseStatusLines( - path = repoPaths().composeReleasePath, - opts: Pick = {}, -): string[] { - const selected = readSelection(path, opts); - const channel = appliedReleaseChannel(); - const disabled = channel.kind === "disabled"; - if (!selected) +/** What the applied generation pins, read locally without the registry. */ +export function releaseStatusLines(infraDir = repoPaths().infraDir): string[] { + const applied = coreServices.readAppliedRelease(infraDir); + const disabled = appliedReleaseChannel(infraDir).kind === "disabled"; + if (!applied) return [ disabled - ? "Release pinning disabled; no release overlay selected." - : "No release lockfile selected locally. Run `pnpm openmapx compose release`.", + ? "Release pinning disabled; the applied configuration pins no release." + : "The applied configuration pins no release. Run `pnpm openmapx compose release`.", ]; return [ - `Configured selection: ${process.env.OPENMAPX_RELEASE ?? "main"}; existing pins remain selected until explicit reselection.`, - ...(disabled ? ["Release resolution disabled; existing overlay still applies."] : []), - `Selected release: ${selected.release ?? "unknown (modified overlay)"}`, - ...coreServices.RELEASE_IMAGE_NAMES.map( - (name) => `${name}: ${selected.images[name] ?? "unknown"}`, - ), - "These are locally selected pins; running containers have not been inspected.", + `Configured selection: ${process.env.OPENMAPX_RELEASE ?? "main"}; the pins stay until a release is selected or cleared.`, + ...(disabled ? ["Release resolution disabled; the applied pins still apply."] : []), + `Pinned release: ${applied.manifest.release} (manifest ${applied.manifestDigest})`, + ...coreServices.RELEASE_IMAGE_NAMES.map((name) => `${name}: ${applied.manifest.images[name]}`), + "These are the applied configuration's pins; running containers have not been inspected.", ]; } +/** Resolves a release and reports how its pins differ from the applied ones; nothing is written. */ export async function selectRelease( opts: { resolve?: () => Promise; report?: (line: string) => void; docker?: ReleaseDockerRunner; - } & ReleaseStateLocation = {}, -): Promise<{ path: string; release: string }> { - const { directory, store, overlay: path } = releaseState(opts); - coreServices.ensureReleaseStateDirectory(directory, { uid: opts.uid }); - coreServices.ensureReleaseStateDirectory(store, { uid: opts.uid }); - const lock = await coreServices.acquireReleaseStoreLock(store, {}, { failIfBusy: true }); - try { - const previous = readSelection(path, opts); - const manifest = await ( - opts.resolve ?? (() => resolveReleaseManifest(defaultDocker, opts.infraDir)) - )(); - await assertCoordinatorSelection(manifest, opts.docker ?? defaultDocker); - const report = opts.report ?? (() => undefined); + infraDir?: string; + } = {}, +): Promise { + const infraDir = opts.infraDir ?? repoPaths().infraDir; + const previous = coreServices.readAppliedRelease(infraDir)?.manifest; + const manifest = await ( + opts.resolve ?? (() => resolveReleaseManifest(defaultDocker, infraDir)) + )(); + await assertCoordinatorSelection(manifest, opts.docker ?? defaultDocker); + coreServices.cacheReleaseManifest(infraDir, manifest); + const report = opts.report ?? (() => undefined); + report(`Previous release: ${previous?.release ?? "none"}`); + report(`Candidate release: ${manifest.release}`); + for (const name of coreServices.RELEASE_IMAGE_NAMES) { + const before = previous?.images[name].split("@")[1]; + const after = manifest.images[name].split("@")[1]; report( - `Previous release: ${previous ? (previous.release ?? "unknown (modified overlay)") : "none"}`, + `${name}: ${before ? (before === after ? "reused" : "changed") : "no previous pin"} (${before ?? "none"} → ${after})`, ); - report(`Candidate release: ${manifest.release}`); - for (const name of coreServices.RELEASE_IMAGE_NAMES) { - const before = previous?.images[name]?.split("@")[1]; - const after = manifest.images[name].split("@")[1]; - report( - `${name}: ${before ? (before === after ? "reused" : "changed") : "previous digest unknown"} (${before ?? "unknown"} → ${after})`, - ); - } - coreServices.assertReleaseStoreLock(lock, store); - writeReleaseOverlay(manifest, opts); - return { path, release: manifest.release }; - } finally { - lock.release(); } + return manifest; } -/** Clear only selection with the shared conditional release-store writer. */ -export async function clearReleaseSelection( - opts: ReleaseStateLocation & { expectedReleaseId?: string; docker?: ReleaseDockerRunner } = {}, -): Promise<{ path: string; cleared: boolean }> { - const path = releaseState(opts).overlay; - let raw: Buffer | undefined; - try { - lstatSync(path); - coreServices.assertReleaseStateDirectory(dirname(path), { uid: opts.uid }); - raw = coreServices.readReleaseStateBytes(path, 128 * 1024, { uid: opts.uid }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") - throw Error("Release selection overlay is unsafe", { cause: error }); - } - let selected: coreServices.ReleaseComposeSelection | undefined; - if (raw !== undefined) { - try { - selected = coreServices.parseReleaseComposeSelection( - new TextDecoder("utf-8", { fatal: true }).decode(raw), - ); - } catch { - // A malformed overlay is already unusable; retain conditional recovery. - } - if (selected?.images["data-manager"]) { - const current = await inspectCoordinator(opts.docker ?? defaultDocker); - if (current?.reference === selected.images["data-manager"]) throw Error(coordinatorGuidance); - } - } - return coreServices.clearReleaseComposeSelection({ - infraDir: opts.infraDir ?? repoPaths().infraDir, - uid: opts.uid, - path, - expectedReleaseId: opts.expectedReleaseId ?? selected?.release ?? undefined, - expectedOverlayDigest: - raw === undefined ? null : createHash("sha256").update(raw).digest("hex"), - }); +/** Refuses to clear pins whose data-manager image the running coordinator uses. */ +export async function assertReleaseClearable( + opts: { docker?: ReleaseDockerRunner; infraDir?: string } = {}, +): Promise { + const applied = coreServices.readAppliedRelease(opts.infraDir ?? repoPaths().infraDir); + if (!applied) return null; + const current = await inspectCoordinator(opts.docker ?? defaultDocker); + if (current?.reference === applied.manifest.images["data-manager"]) + throw Error(coordinatorGuidance); + return applied.manifest.release; } diff --git a/packages/cli/src/lib/rollback.ts b/packages/cli/src/lib/rollback.ts new file mode 100644 index 000000000..db2cb6111 --- /dev/null +++ b/packages/cli/src/lib/rollback.ts @@ -0,0 +1,88 @@ +import { services } from "@openmapx/core/server"; +import { dockerCompose } from "./docker"; + +export interface RollbackPlan { + currentRevisionId: string; + targetRevisionId: string; + current: services.GenerationRelease | null; + target: services.GenerationRelease | null; +} + +/** The applied generation and the one applied before it, with the releases they pin. */ +export function planRollback(infraDir: string): RollbackPlan { + const currentRevisionId = services.currentConfigurationGeneration(infraDir); + if (!currentRevisionId) throw new Error("No configuration has been applied yet"); + const targetRevisionId = services.readPreviousConfigurationGeneration( + infraDir, + currentRevisionId, + ); + if (!targetRevisionId) + throw new Error("The applied configuration is the first one; there is nothing to roll back to"); + try { + services.configurationGenerationComposeFile(infraDir, targetRevisionId); + } catch (error) { + throw new Error(`The previous configuration ${targetRevisionId} is no longer retained`, { + cause: error, + }); + } + return { + currentRevisionId, + targetRevisionId, + current: services.readGenerationRelease(infraDir, currentRevisionId), + target: services.readGenerationRelease(infraDir, targetRevisionId), + }; +} + +/** + * Refuses a rollback whose images could run against a newer database schema: + * the migrator only applies migrations newer than the newest applied one, so + * an older API would start against the newer schema without complaint. The + * newest applied migration must not be newer than the target release's + * recorded migration head. Rolling back within one release changes no image + * and needs no check. + */ +export function assertDatabaseAllowsRollback( + plan: RollbackPlan, + newestAppliedMigration: number | null, +): void { + if (plan.current?.manifestDigest === plan.target?.manifestDigest) return; + if (!plan.target) + throw new Error( + "The previous configuration pins no release, so the database migrations its images expect are unknown. Rollback refuses; restore a backup taken before the update instead.", + ); + const head = plan.target.manifest.databaseMigrationHead; + if (!head) + throw new Error( + `Release ${plan.target.manifest.release} records no database migration head, so rollback cannot prove its images match the database. Restore a backup taken before the update instead.`, + ); + if (newestAppliedMigration !== null && newestAppliedMigration > head.when) + throw new Error( + `The database has migrations newer than release ${plan.target.manifest.release} (its newest is ${head.tag}). Its images would run against a newer schema. Restore a backup taken before the update, or fix forward.`, + ); +} + +/** `created_at` of the newest migration the database recorded, or null when it has none. */ +export async function readNewestAppliedMigration(): Promise { + const result = await dockerCompose([ + "exec", + "-T", + "postgis", + "psql", + "-U", + "postgres", + "-d", + "openmapx", + "-v", + "ON_ERROR_STOP=1", + "-At", + "-c", + "SELECT coalesce(max(created_at), 0) FROM drizzle.__drizzle_migrations", + ]); + const value = result.stdout.trim(); + if (result.exitCode !== 0 || !/^\d+$/.test(value)) + throw new Error( + `The database's migrations could not be read (exit ${result.exitCode}${result.stderr.trim() ? `: ${result.stderr.trim().split("\n").slice(-1)[0]}` : ""}); postgis must be running`, + ); + const newest = Number(value); + return newest === 0 ? null : newest; +} diff --git a/packages/cli/src/lib/service-file-report.ts b/packages/cli/src/lib/service-file-report.ts index 49c546748..281e8bfd2 100644 --- a/packages/cli/src/lib/service-file-report.ts +++ b/packages/cli/src/lib/service-file-report.ts @@ -22,7 +22,9 @@ export async function serviceFileReport(): Promise< let release: coreServices.ReleaseServiceFiles; let selected: Awaited> = null; try { - selected = await selectedReleaseServiceFiles(paths.infraDir); + selected = await selectedReleaseServiceFiles( + coreServices.readAppliedRelease(paths.infraDir)?.manifest ?? null, + ); release = selected && selected !== "none" ? selected.files : coreServices.releaseServiceFiles(services); } catch (error) { diff --git a/packages/cli/src/lib/service-lifecycle.ts b/packages/cli/src/lib/service-lifecycle.ts index 7f019ceb9..9eaf75b9c 100644 --- a/packages/cli/src/lib/service-lifecycle.ts +++ b/packages/cli/src/lib/service-lifecycle.ts @@ -229,14 +229,11 @@ export async function runCliServiceLifecycle(options: { if (inherited && !inherited.lease) throw Error("Delegated lifecycle requires its original borrowed lease"); let owning = false; - let releaseStoreLock: services.ReleaseStoreLock | undefined; let store: services.ServiceLifecycleReceiptStore | undefined; let receipt: services.ServiceLifecycleReceipt | undefined; let lease: TargetLeaseAuthority | undefined; let finalCommit: ReturnType; let failureAttempted = false; - let releaseSelection: string | null | undefined; - let dockerEffectReached = false; const checkpoint = async ( owned: services.VerifiedLifecycleOwnership, patch: Partial, @@ -314,7 +311,7 @@ export async function runCliServiceLifecycle(options: { warnFailureNotRecorded(); } }; - /** Holds the writer and release-store lock only for `run`, which must not start a container. */ + /** Holds the writer only for `run`, which must not start a container. */ const withWriterStep = async ( run: (owned: services.VerifiedLifecycleOwnership) => Promise, signal?: AbortSignal, @@ -328,17 +325,6 @@ export async function runCliServiceLifecycle(options: { { writer, lease: held, authority }, async (owned) => { assertForward(signal); - const releasePaths = services.releaseStatePaths(paths.infraDir); - services.ensureReleaseStateDirectory(releasePaths.directory); - services.ensureReleaseStateDirectory(releasePaths.store); - // A brief holder such as release selection must not fail a lifecycle whose - // containers may already be running, so later steps wait for the store. - const lock = await services.acquireReleaseStoreLock( - releasePaths.store, - {}, - { failIfBusy: !dockerEffectReached }, - ); - releaseStoreLock = lock; try { return await run(owned); } catch (error) { @@ -347,22 +333,16 @@ export async function runCliServiceLifecycle(options: { } finally { finalCommit = services.configurationGenerationWriterCommit(owned.writer) ?? finalCommit; - releaseStoreLock = undefined; - lock.release(); } }, ), signal, ); }; - const readReleaseSelection = () => - services.readReleaseOverlayFile(services.releaseStatePaths(paths.infraDir).overlay); - // Selecting a release rewrites the overlay without moving the generation. + // The release pins are part of the generation, so its revision covers them. const assertConfigurationUnchanged = (revisionId: string, conflict: string) => { if (services.currentConfigurationGeneration(paths.infraDir) !== revisionId) throw new LifecycleConfigurationConflict(conflict); - if (releaseSelection !== undefined && readReleaseSelection() !== releaseSelection) - throw new LifecycleConfigurationConflict("Lifecycle release selection changed"); }; /** * Refuses the Docker effect that follows unless the configuration is still the @@ -373,7 +353,6 @@ export async function runCliServiceLifecycle(options: { const beforeDockerEffect = async (revisionId: string, conflict: string, signal?: AbortSignal) => { await withWriterStep(async () => assertConfigurationUnchanged(revisionId, conflict), signal); assertForward(signal); - dockerEffectReached = true; }; const committedRevisionId = () => { if (!receipt?.finalRevisionId) throw Error("Lifecycle generation is not committed"); @@ -438,15 +417,16 @@ export async function runCliServiceLifecycle(options: { async validatePins(selected, signal) { await withWriterStep(async (owned) => { assertConfigurationUnchanged(expected, "Lifecycle configuration revision conflict"); - if (release.touchesReleasePinnedServices(selected)) { - if (!releaseStoreLock) throw Error("Lifecycle release-store ownership unavailable"); - const overlay = await release.ensureReleaseOverlay({ releaseStoreLock }); - if (overlay.status === "held") throw new ServiceLifecycleCommandError(1, overlay.reason); - if (overlay.status === "unpinned") - throw new ServiceLifecycleCommandError(1, "Release selection unavailable"); - } + if ( + release.touchesReleasePinnedServices(selected) && + !services.readAppliedRelease(paths.infraDir) && + services.readAppliedReleaseSelection(paths.infraDir).kind !== "local" + ) + throw new ServiceLifecycleCommandError( + 1, + "The applied configuration pins no release; select one with `compose release`", + ); await assertNativeCreation(); - releaseSelection = readReleaseSelection(); await checkpoint(owned, { phase: "pins_validated" }); }, signal); }, @@ -587,9 +567,7 @@ export async function runCliServiceLifecycle(options: { // Recovery accepts only a failure at the hardlink step itself. if (attempt !== null) log.warn( - failed.failure?.reason === "hardlinks" && !failed.hardlinks - ? `Operation ${key} keeps its service locks because a hardlink request issued to data-manager may still be running. Inspect infra/docker/data/ops-agent/lifecycle/${key}.json; after its deadline, \`pnpm openmapx services recover-update --operation-key ${key}\` can release an unadopted request.` - : `Operation ${key} keeps its service locks because a hardlink child was issued to data-manager. No automated release exists yet for this case, and \`services recover-update\` does not apply. Its record is infra/docker/data/ops-agent/lifecycle/${key}.json.`, + `Operation ${key} keeps its service locks because a hardlink request issued to data-manager may still be running. Its record is infra/docker/state/ops-agent/lifecycle/${key}.json. Once data-manager has finished it, \`pnpm openmapx control status\` shows the lease and \`pnpm openmapx control resolve --op ${key} --as committed|failed\` ends it.`, ); return; } diff --git a/packages/cli/src/lib/system-update-receipt.ts b/packages/cli/src/lib/system-update-receipt.ts index 9cbfd2a77..4eedb7ccc 100644 --- a/packages/cli/src/lib/system-update-receipt.ts +++ b/packages/cli/src/lib/system-update-receipt.ts @@ -26,7 +26,7 @@ const identitySchema = z.strictObject({ origin: z.url(), actorId: z.string().min(1), release: z.strictObject({ - releaseId: z.string().regex(services.RELEASE_ID_PATTERN), + releaseId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/), manifestDigest: z.string().regex(/^[a-f0-9]{64}$/), }), configuredSelector: z.string().min(1), diff --git a/packages/cli/src/lib/system-update.ts b/packages/cli/src/lib/system-update.ts index b08a3a3a9..17b6eb752 100644 --- a/packages/cli/src/lib/system-update.ts +++ b/packages/cli/src/lib/system-update.ts @@ -1,4 +1,5 @@ /** Drives the existing real-admin producer. No host mutation or new principal. */ +import { createHash } from "node:crypto"; import { adminActivityJobDetailSchema, adminActivityJobsResponseSchema, @@ -29,6 +30,11 @@ class HttpFailure extends Error { } } const equal = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b); +/** The configured release selection, so a resumed update notices that it changed. */ +const selectionIdentity = (infraDir: string) => + createHash("sha256") + .update(JSON.stringify(services.readAppliedReleaseSelection(infraDir))) + .digest("hex"); const targets = ["app-api", "app-web", "data-manager", "ops-agent", "transitous-runner"]; export interface SystemUpdateOptions { infraDir: string; @@ -128,9 +134,7 @@ export async function runSystemUpdate(options: SystemUpdateOptions): Promise<{ j actorId: actor.user.id, release: { releaseId: options.releaseId, manifestDigest: options.manifestDigest }, configuredSelector: selection.selector, - configurationIdentity: services.releaseConfigurationIdentity( - services.readAppliedReleaseSelection(options.infraDir), - ), + configurationIdentity: selectionIdentity(options.infraDir), createBackup: options.createBackup !== false, }; let receipt: SystemUpdateReceipt; @@ -164,11 +168,7 @@ export async function runSystemUpdate(options: SystemUpdateOptions): Promise<{ j identity.release, services.readAppliedReleaseSelection(options.infraDir), ); - if ( - services.releaseConfigurationIdentity( - services.readAppliedReleaseSelection(options.infraDir), - ) !== identity.configurationIdentity - ) + if (selectionIdentity(options.infraDir) !== identity.configurationIdentity) throw Error("Configured release selection changed"); const configuration = await get( "/api/admin/platform/configuration", diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.test.ts b/packages/core/src/build-artifacts/configuration-handoff-records.test.ts index f0257dfee..0a9d7381f 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.test.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.test.ts @@ -73,13 +73,7 @@ it("does not confuse an attempted release pull/publication with verified local i phase: "pending", imageId: null, }; - const initial = { - images: [image], - publication: "not_attempted", - overlay: null, - statePublication: "not_attempted", - state: null, - }; + const initial = { images: [image] }; expect(configurationHandoffReleaseProgressSchema.parse(initial)).toEqual(initial); expect( configurationHandoffReleaseProgressSchema.safeParse({ @@ -97,6 +91,9 @@ it("does not confuse an attempted release pull/publication with verified local i configurationHandoffReleaseProgressSchema.safeParse({ ...initial, publication: "published" }) .success, ).toBe(false); + expect( + configurationHandoffReleaseProgressSchema.safeParse({ ...initial, overlay: null }).success, + ).toBe(false); expect( configurationHandoffReleaseProgressSchema.safeParse({ ...initial, images: [image, image] }) .success, diff --git a/packages/core/src/build-artifacts/configuration-handoff-records.ts b/packages/core/src/build-artifacts/configuration-handoff-records.ts index f55260adc..310a275da 100644 --- a/packages/core/src/build-artifacts/configuration-handoff-records.ts +++ b/packages/core/src/build-artifacts/configuration-handoff-records.ts @@ -13,27 +13,6 @@ import { controlOriginSchema } from "./control-records"; const digest = z.string().regex(/^[a-f0-9]{64}$/), containerId = z.string().regex(/^[a-f0-9]{64}$/), time = z.number().int().nonnegative().safe(); -/** Private original overlay bytes; never part of the cross-role public graph. */ -export const configurationHandoffOverlaySourceSchema = z - .strictObject({ - identity: z.strictObject({ - dev: z.number().int().nonnegative().safe(), - ino: z.number().int().nonnegative().safe(), - mtime: z.number().finite(), - ctime: z.number().finite(), - size: z - .number() - .int() - .nonnegative() - .max(128 * 1024), - }), - bytes: z.string().max(174764), - }) - .refine( - (value) => - Buffer.from(value.bytes, "base64").toString("base64") === value.bytes && - Buffer.from(value.bytes, "base64").length === value.identity.size, - ); export const configurationHandoffReleaseProgressSchema = z .strictObject({ images: z @@ -49,18 +28,11 @@ export const configurationHandoffReleaseProgressSchema = z }), ) .max(CONFIGURATION_RELEASE_IMAGE_NAMES.length), - publication: z.enum(["not_attempted", "attempted", "published"]), - overlay: configurationHandoffOverlaySourceSchema.nullable(), - statePublication: z.enum(["not_attempted", "attempted", "published"]), - state: configurationHandoffOverlaySourceSchema.nullable(), }) .superRefine((value, context) => { if ( new Set(value.images.map((image) => image.image)).size !== value.images.length || - value.images.some((image) => (image.phase === "verified") !== (image.imageId !== null)) || - (value.publication === "published") !== (value.overlay !== null) || - (value.statePublication === "published") !== (value.state !== null) || - (value.statePublication !== "not_attempted" && value.publication !== "published") + value.images.some((image) => (image.phase === "verified") !== (image.imageId !== null)) ) context.addIssue({ code: "custom", message: "Original verified release progress required" }); }); @@ -104,16 +76,12 @@ export const configurationHandoffIntentRecordSchema = z helperContainerName: z.string().regex(/^openmapx-configuration-handoff-[a-f0-9-]{36}$/), preparedAt: time, release: configurationReleaseSnapshotSchema.optional(), - previousReleaseOverlay: configurationHandoffOverlaySourceSchema.nullable().optional(), - previousReleaseState: configurationHandoffOverlaySourceSchema.nullable().optional(), }) .refine((value) => Buffer.byteLength(JSON.stringify(value)) <= 65536) .superRefine((value, context) => { const { mode } = value.operation; if ( (mode.kind === "release") !== (value.release !== undefined) || - (mode.kind === "release") !== (value.previousReleaseOverlay !== undefined) || - (mode.kind === "release") !== (value.previousReleaseState !== undefined) || (mode.kind === "release" && value.release && (mode.releaseId !== value.release.releaseId || @@ -213,9 +181,7 @@ export const configurationHandoffWorkerRecordSchema = configurationHandoffProgre (value) => !value.release || value.phase !== "applied" || - (value.release.publication === "published" && - value.release.statePublication === "published" && - value.release.images.length === CONFIGURATION_RELEASE_IMAGE_NAMES.length && + (value.release.images.length === CONFIGURATION_RELEASE_IMAGE_NAMES.length && value.release.images.every((image) => image.phase === "verified")), ) .refine( diff --git a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts index f44f9e1e4..734efd480 100644 --- a/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts +++ b/packages/core/src/build-artifacts/configuration-maintenance-final-records.ts @@ -508,19 +508,7 @@ function assertExecutedConfigurationMaintenanceFinalState( !same(inspection.requiredChecks, requiredChecks) || inspection.observations.length !== requiredChecks.length || !inspection.observations.every((value, index) => value.checkIndex === index && value.passed) || - !same(details[0], { - currentRevisionId: inspectedRevision, - ...(!contained && owner.plan.release - ? { - currentRelease: { - releaseId: owner.plan.release.releaseId, - digest: owner.plan.release.manifestDigest, - updateJobId: - owner.plan.apiParent.issuer === "api" ? owner.plan.apiParent.apiJobId : null, - }, - } - : {}), - }) || + !same(details[0], { currentRevisionId: inspectedRevision }) || !same(details[2], { guard: owner.preparedFence }) || !owner.plan.steps.every((step, index) => same(details[index + 3], { diff --git a/packages/core/src/build-artifacts/configuration-source-witness.test.ts b/packages/core/src/build-artifacts/configuration-source-witness.test.ts index 163ce2b17..314b8dc9b 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.test.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.test.ts @@ -4,11 +4,6 @@ import { join } from "node:path"; import { afterEach, beforeEach, expect, it } from "vitest"; import { commitConfigurationGeneration } from "../services/configuration-generation"; import { writeServiceSelectionFile } from "../services/deployment"; -import { - parseReleaseComposeSelection, - TRANSITOUS_TOOLS_IMAGE_ENV, -} from "../services/release-manifest"; -import { ensureReleaseStateDirectories } from "../services/release-state"; import type { LoadedService } from "../services/types"; import { configurationHostFields, @@ -28,60 +23,6 @@ beforeEach(async () => { await provisionControlInstallation(root); }); afterEach(() => rmSync(root, { recursive: true, force: true })); -it("retains the original private release selection instead of accepting a changed image pin", () => { - const installation = openControlInstallation({ - infraDir: root, - role: "ops-agent", - serviceIds: ["data-manager"], - }); - try { - const pins = Object.fromEntries( - ["api", "web", "data-manager", "ops-agent", "transitous-runner", "transitous-tools"].map( - (name) => [name, `ghcr.io/openmapx/${name}@sha256:${"b".repeat(64)}`], - ), - ); - const document = { - services: { - "app-api": { image: pins.api }, - "app-web": { image: pins.web }, - "data-manager": { - image: pins["data-manager"], - environment: { [TRANSITOUS_TOOLS_IMAGE_ENV]: pins["transitous-tools"] }, - }, - "ops-agent": { image: pins["ops-agent"] }, - "transitous-runner": { image: pins["transitous-runner"] }, - }, - "x-openmapx-release": { release: "public-release-fixture", images: pins }, - }; - const path = ensureReleaseStateDirectories(root).overlay; - writeFileSync(path, JSON.stringify(document), { mode: 0o600 }); - expect(parseReleaseComposeSelection(JSON.stringify(document)).release).toBe( - "public-release-fixture", - ); - const options = { - installation, - infraDir: root, - services: [] as LoadedService[], - authorityRevision: "v1", - env: {}, - }, - observed = observeConfigurationSources(options), - verify = () => - verifyConfigurationSourceWitness({ - ...options, - sourceRevision: observed.sourceRevision, - expectedCurrentRevisionId: null, - }); - expect(Object.keys(observed).sort()).toEqual(["revisionId", "services", "sourceRevision"]); - verify(); - pins["data-manager"] = `ghcr.io/openmapx/data-manager@sha256:${"c".repeat(64)}`; - document.services["data-manager"].image = pins["data-manager"]; - writeFileSync(path, JSON.stringify(document), { mode: 0o600 }); - expect(() => verify()).toThrow("Host sources changed"); - } finally { - closeControlInstallation(installation); - } -}); it("keeps host values private while detecting a changed persisted full selection", () => { const installation = openControlInstallation({ infraDir: root, diff --git a/packages/core/src/build-artifacts/configuration-source-witness.ts b/packages/core/src/build-artifacts/configuration-source-witness.ts index 239429ecb..965f5183e 100644 --- a/packages/core/src/build-artifacts/configuration-source-witness.ts +++ b/packages/core/src/build-artifacts/configuration-source-witness.ts @@ -15,14 +15,7 @@ import { readAppliedPlatformConfiguration, } from "../services/configuration-generation"; import { serviceSelectionPath } from "../services/deployment"; -import { parseReleaseManifest, renderReleaseCompose } from "../services/release-manifest"; -import { releaseStatePaths } from "../services/release-state"; import type { LoadedService } from "../services/types"; -import { - configurationHandoffRecordDomains, - configurationHandoffRecordName, - configurationHandoffWorkerRecordSchema, -} from "./configuration-handoff-records"; import { configurationMaintenanceOwnerRecordSchema } from "./configuration-maintenance-owner-records"; import { assertControlExecutionForward, @@ -171,34 +164,7 @@ export function assertConfigurationHandoffHostSources(authority: VerifiedExecuti timingSafeEqual(Buffer.from(value, "hex"), Buffer.from(intent.sourceDigest, "hex")); if (currentConfigurationGeneration(infraDir) !== intent.operation.expectedCurrentRevisionId) throw Error("Helper host generation changed"); - if (matches(keyed())) return; - if (!intent.release || intent.previousReleaseOverlay === undefined) - throw Error("Helper host sources changed"); - // The only allowed source transition is this original helper's indexed, - // signed publication of its exact frozen renderer output. All other files, - // environment values and the original generation remain under the old HMAC. - const file = configurationHandoffRecordName(intent.operation.handoffId, "completion"), - publication = readPrivateRecord(local, "observations", file), - worker = verifyControl( - intent.helperPublicKey, - configurationHandoffRecordDomains.completion, - publication, - configurationHandoffWorkerRecordSchema, - ), - files = currentSources(infraDir, {}); - if ( - worker.intentDigest !== controlDigest(intent) || - canonicalJson(worker.execution) !== canonicalJson(intent.helper) || - worker.release?.publication !== "published" || - canonicalJson(worker.release.overlay) !== canonicalJson(files.release) || - !files.release || - Buffer.from(files.release.bytes, "base64").toString("utf8") !== - renderReleaseCompose( - parseReleaseManifest(intent.release.canonicalManifest, intent.release.imagePrefix), - ) || - !matches(keyed({ ...files, release: intent.previousReleaseOverlay })) - ) - throw Error("Original helper release publication changed"); + if (!matches(keyed())) throw Error("Helper host sources changed"); } function hostEnv(infraDir: string, env: NodeJS.ProcessEnv) { const path = join(infraDir, ".env"); @@ -272,57 +238,10 @@ function hostEnv(infraDir: string, env: NodeJS.ProcessEnv) { return { env: { ...file, ...env }, identity, selection }; } -/** Bounded private data only. Callers own their existing writer/target checks. */ -function readHandoffSourceFile(path: string, maximumBytes: number) { - let fd: number | undefined; - try { - fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); - const before = fstatSync(fd); - if (!before.isFile() || before.nlink !== 1 || before.size > maximumBytes) - throw Error("Host sources unavailable"); - const bytes = readFileSync(fd), - after = fstatSync(fd), - current = lstatSync(path); - if ( - before.dev !== current.dev || - before.ino !== current.ino || - before.size !== after.size || - before.mtimeMs !== after.mtimeMs || - before.ctimeMs !== after.ctimeMs || - current.size !== after.size || - current.mtimeMs !== after.mtimeMs || - current.ctimeMs !== after.ctimeMs - ) - throw Error("Host sources unavailable"); - return { - identity: { - dev: before.dev, - ino: before.ino, - mtime: before.mtimeMs, - ctime: before.ctimeMs, - size: before.size, - }, - bytes: bytes.toString("base64"), - }; - } catch (error) { - if (fd !== undefined || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - return null; - } finally { - if (fd !== undefined) closeSync(fd); - } -} -/** Raw values stay exclusively in role-private helper records. */ -export function readConfigurationHandoffReleaseOverlay(infraDir: string) { - return readHandoffSourceFile(releaseStatePaths(infraDir).overlay, 128 * 1024); -} -export function readConfigurationHandoffReleaseState(infraDir: string) { - return readHandoffSourceFile(join(releaseStatePaths(infraDir).store, "current.json"), 4096); -} function currentSources(infraDir: string, env: NodeJS.ProcessEnv) { const platform = readAppliedPlatformConfiguration(infraDir); return { ...hostEnv(infraDir, env), - release: readConfigurationHandoffReleaseOverlay(infraDir), ...(platform ? { platform } : {}), }; } diff --git a/packages/core/src/build-artifacts/data-update-sources.ts b/packages/core/src/build-artifacts/data-update-sources.ts index eacf54955..aecabc6f5 100644 --- a/packages/core/src/build-artifacts/data-update-sources.ts +++ b/packages/core/src/build-artifacts/data-update-sources.ts @@ -11,15 +11,13 @@ import { } from "node:fs"; import { join, relative, resolve } from "node:path"; import { parseEnv } from "node:util"; +import { readAppliedRelease } from "../services/configuration-generation"; import { serviceSelectionPath } from "../services/deployment"; -import { releaseStatePaths } from "../services/release-state"; export function dataUpdateSourcesDigest(infraDir: string): string { const hash = createHash("sha256"); - for (const name of [ - ".env", - relative(infraDir, serviceSelectionPath(infraDir)), - relative(infraDir, releaseStatePaths(infraDir).overlay), - ]) { + // The release the applied generation pins decides the images a data update runs. + hash.update(`release\0${readAppliedRelease(infraDir)?.manifestDigest ?? "absent"}\0`); + for (const name of [".env", relative(infraDir, serviceSelectionPath(infraDir))]) { hash.update(name); let fd: number | undefined; try { diff --git a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts index 1b0c172be..e56dc6d56 100644 --- a/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts +++ b/packages/core/src/build-artifacts/fixed-configuration-maintenance.test.ts @@ -283,8 +283,6 @@ it("freezes the checked release helper graph and plans a complete 13-service sta helperContainerName: `openmapx-configuration-handoff-${handoff.handoffId}`, preparedAt: Date.now(), release, - previousReleaseOverlay: null, - previousReleaseState: null, }; expect(configurationHandoffIntentRecordSchema.parse(intent).release).toEqual(release); expect( diff --git a/packages/core/src/build-artifacts/target-lease.test.ts b/packages/core/src/build-artifacts/target-lease.test.ts index 468771d42..aa8d55b74 100644 --- a/packages/core/src/build-artifacts/target-lease.test.ts +++ b/packages/core/src/build-artifacts/target-lease.test.ts @@ -310,11 +310,10 @@ it("keeps a bare terminal maintenance summary fenced without the fixed owner sou a.closeArtifactTargetLeaseHandle(lease); c.closeControlInstallation(agent); }); -it("holds a target lease, the configuration writer and the release-store lock together", async () => { +it("holds a target lease and the configuration writer together", async () => { const { acquireConfigurationOwnership, configurationWriterLockPath } = await import( "../services/configuration-ownership" ); - const { acquireReleaseStoreLock } = await import("../services/release-store-lock"); const a = await api(); const c = await import("./control-authority"); const authority = owner(); @@ -324,9 +323,7 @@ it("holds a target lease, the configuration writer and the release-store lock to }); expect(heldLeases()).toHaveLength(1); const releaseWriter = await acquireConfigurationOwnership(root); - const store = await acquireReleaseStoreLock(root, {}, { failIfBusy: true }); expect(existsSync(configurationWriterLockPath(root))).toBe(true); - store.release(); releaseWriter(); c.retainControlEffectReceipt(authority, { outcome: "failed_without_effect", diff --git a/packages/core/src/repo-paths.ts b/packages/core/src/repo-paths.ts index 156cb6b1d..57b0e1736 100644 --- a/packages/core/src/repo-paths.ts +++ b/packages/core/src/repo-paths.ts @@ -57,7 +57,6 @@ export interface RepoPaths { infraDir: string; /** The applied configuration generation's compose file. */ composePath: string; - composeReleasePath: string; } export function repoPaths(start?: string): RepoPaths { @@ -77,7 +76,7 @@ export function repoPaths(start?: string): RepoPaths { "custom-integrations", ), infraDir: join(root, "infra", "docker"), - // The operations agent's state: the applied generation and the release overlay. + // The operations agent's state: the applied generation. composePath: join( root, "infra", @@ -88,14 +87,5 @@ export function repoPaths(start?: string): RepoPaths { "current", "docker-compose.generated.yml", ), - composeReleasePath: join( - root, - "infra", - "docker", - "state", - "ops-agent", - "releases", - "docker-compose.release.yml", - ), }; } diff --git a/packages/core/src/services/__fixtures__/compose-path-host.ts b/packages/core/src/services/__fixtures__/compose-path-host.ts index b51fd2546..8a3e17f08 100644 --- a/packages/core/src/services/__fixtures__/compose-path-host.ts +++ b/packages/core/src/services/__fixtures__/compose-path-host.ts @@ -1,5 +1,4 @@ import { readGenerationServiceFiles, serviceFileTreePath } from "../service-file-store"; -import { ensureReleaseStateDirectories } from "../release-state"; import { spawnSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { @@ -69,17 +68,10 @@ export async function composePathHost() { mode: 0o600, }); writeFileSync(join(infraDir, "public-data.txt"), "public infrastructure fixture\n"); - writeFileSync(join(infraDir, "overlay-public.txt"), "public overlay fixture\n"); const stack = { infraDir, composePath: currentConfigurationFile(infraDir, GENERATED_COMPOSE_FILE), - composeReleasePath: ensureReleaseStateDirectories(infraDir).overlay, }; - writeFileSync( - stack.composeReleasePath, - "services:\n alpha:\n image: example/alpha:release\n volumes:\n - ../../../../overlay-public.txt:/overlay-public:ro\n", - { mode: 0o600 }, - ); const dockerConfig = join(root, "docker-config"); mkdirSync(join(dockerConfig, "cli-plugins"), { recursive: true }); if (installedCompose) @@ -156,9 +148,7 @@ export async function composePathHost() { ...(noEnvResolution ? ["--no-env-resolution"] : []), ]); const pathChecks = (model: PathConfig, compose: string) => { - const checks: Record = { - releaseImage: model.services.alpha?.image === "example/alpha:release", - }; + const checks: Record = {}; for (const service of loaded) { const id = service.manifest.id, entry = model.services[id]; @@ -182,10 +172,6 @@ export async function composePathHost() { checks[`${id}.envFileRead`] = entry?.environment?.PUBLIC_FIXTURE_SETTING === "synthetic-public"; } - checks.overlayBind = sameFile( - model.services.alpha?.volumes?.find((v) => v.target === "/overlay-public")?.source ?? "", - join(infraDir, "overlay-public.txt"), - ); record({ pathChecks: checks, generationFile: compose, diff --git a/packages/core/src/services/__tests__/checked-release.test.ts b/packages/core/src/services/__tests__/checked-release.test.ts deleted file mode 100644 index 7761d1fe1..000000000 --- a/packages/core/src/services/__tests__/checked-release.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { createHash } from "node:crypto"; -import { - chmodSync, - existsSync, - mkdtempSync, - readdirSync, - readFileSync, - renameSync, - rmSync, - statSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; -import { releaseProvenance } from "../__fixtures__/release-provenance"; -import * as services from "../index"; - -const roots: string[] = []; -const locks = new Set(); -const sha = (value: string) => createHash("sha256").update(value).digest("hex"); -function manifest(release = "checked-release"): services.ReleaseManifest { - return { - schemaVersion: 1, - release, - images: Object.fromEntries( - services.RELEASE_IMAGE_NAMES.map((name) => [ - name, - `ghcr.io/openmapx/${name}@sha256:${"a".repeat(64)}`, - ]), - ) as services.ReleaseManifest["images"], - ...releaseProvenance(), - }; -} -async function fixture() { - const infraDir = mkdtempSync(join(tmpdir(), "openmapx-checked-release-")); - roots.push(infraDir); - const paths = services.ensureReleaseStateDirectories(infraDir); - const lock = await services.acquireReleaseStoreLock(paths.store, {}, { failIfBusy: true }); - locks.add(lock); - const input = { - infraDir, - manifest: manifest(), - ociDigest: `sha256:${"c".repeat(64)}`, - configurationIdentity: "d".repeat(64), - lock, - }; - return { - input, - paths, - release() { - locks.delete(lock); - lock.release(); - }, - }; -} -afterEach(() => { - for (const lock of locks) lock.release(); - locks.clear(); - for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); -}); - -describe("checked release cache publication", () => { - it("publishes canonical checked bytes and receipts without selecting or applying a release", async () => { - const f = await fixture(); - services.writeReleaseComposeArtifacts(manifest("running-release"), f.input.infraDir); - const before = readFileSync(f.paths.overlay, "utf8"); - writeFileSync(join(f.paths.store, "current.json"), "running-state", { mode: 0o600 }); - const result = services.cacheCheckedReleaseManifest(f.input); - const raw = readFileSync(join(f.paths.store, "checked-release.json"), "utf8"); - expect(services.parseReleaseManifest(raw)).toEqual(f.input.manifest); - expect(raw).toBe(services.canonicalReleaseManifest(f.input.manifest)); - expect(result).toEqual({ releaseId: "checked-release", manifestDigest: sha(raw) }); - expect(JSON.parse(readFileSync(join(f.paths.store, "selections.json"), "utf8"))).toEqual({ - "checked-release": { - configuration: "d".repeat(64), - ociDigest: `sha256:${"c".repeat(64)}`, - digest: sha(raw), - }, - }); - expect(JSON.parse(readFileSync(join(f.paths.store, "latest.json"), "utf8"))).toEqual({ - releaseId: "checked-release", - digest: sha(raw), - }); - expect(readFileSync(f.paths.overlay, "utf8")).toBe(before); - expect(readFileSync(join(f.paths.store, "current.json"), "utf8")).toBe("running-state"); - }); - - it("keeps the original immutable manifest inode on an exact repeated check", async () => { - const f = await fixture(); - services.cacheCheckedReleaseManifest(f.input); - const path = join(f.paths.store, "checked-release.json"); - const inode = statSync(path).ino; - services.cacheCheckedReleaseManifest(f.input); - expect(statSync(path).ino).toBe(inode); - expect(readdirSync(f.paths.store).filter((name) => name.includes(".tmp"))).toEqual([]); - }); - - it("refuses different bytes under an already checked release ID", async () => { - const f = await fixture(); - services.cacheCheckedReleaseManifest(f.input); - const before = readFileSync(join(f.paths.store, "checked-release.json"), "utf8"); - const changed = { - ...f.input.manifest, - images: { ...f.input.manifest.images, web: `ghcr.io/openmapx/web@sha256:${"0".repeat(64)}` }, - }; - expect(() => services.cacheCheckedReleaseManifest({ ...f.input, manifest: changed })).toThrow( - /authority|immutable|changed/i, - ); - expect(readFileSync(join(f.paths.store, "checked-release.json"), "utf8")).toBe(before); - expect(readdirSync(f.paths.store).filter((name) => name.includes(".tmp"))).toEqual([]); - }); - - it.each(["forged", "released"])("requires an actual held release lock (%s)", async (kind) => { - const f = await fixture(); - const lock = kind === "forged" ? { release() {} } : f.input.lock; - if (kind === "released") f.release(); - expect(() => services.cacheCheckedReleaseManifest({ ...f.input, lock })).toThrow( - /lock|ownership/i, - ); - expect(existsSync(join(f.paths.store, "checked-release.json"))).toBe(false); - }); - - it("refuses the sixty-fifth distinct release but permits the original immutable entry", async () => { - const f = await fixture(); - for (let index = 0; index < 64; index++) { - const value = manifest(`release-${index}`); - writeFileSync( - join(f.paths.store, `${value.release}.json`), - services.canonicalReleaseManifest(value), - { mode: 0o600 }, - ); - } - expect(() => services.cacheCheckedReleaseManifest(f.input)).toThrow(/limit/i); - expect(existsSync(join(f.paths.store, "checked-release.json"))).toBe(false); - expect( - services.cacheCheckedReleaseManifest({ ...f.input, manifest: manifest("release-0") }) - .releaseId, - ).toBe("release-0"); - }); - - it.each(["unknown-entry", "empty-entry", "oversized-entry", "writable-entry", "foreign-owner"])( - "refuses unsafe existing state before publication (%s)", - async (kind) => { - const f = await fixture(); - if (kind === "unknown-entry") - writeFileSync(join(f.paths.store, "unknown.txt"), "retain", { mode: 0o600 }); - if (kind === "empty-entry") - writeFileSync(join(f.paths.store, "other.json"), "", { mode: 0o600 }); - if (kind === "oversized-entry") - writeFileSync(join(f.paths.store, "other.json"), Buffer.alloc(32 * 1024 + 1), { - mode: 0o600, - }); - if (kind === "writable-entry") { - writeFileSync(join(f.paths.store, "other.json"), "{}", { mode: 0o600 }); - chmodSync(join(f.paths.store, "other.json"), 0o620); - } - const uid = kind === "foreign-owner" ? (process.geteuid?.() ?? 0) + 1 : undefined; - expect(() => services.cacheCheckedReleaseManifest({ ...f.input, uid })).toThrow( - /unsafe|owned|owner|uid|limit/i, - ); - expect(existsSync(join(f.paths.store, "checked-release.json"))).toBe(false); - expect(existsSync(join(f.paths.store, "latest.json"))).toBe(false); - }, - ); - - it.each(["current", "latest", "selections"])( - "never publishes a manifest over reserved state (%s)", - async (release) => { - const f = await fixture(); - expect(() => - services.cacheCheckedReleaseManifest({ ...f.input, manifest: manifest(release) }), - ).toThrow(/reserved|identity/i); - expect(existsSync(join(f.paths.store, `${release}.json`))).toBe(false); - }, - ); - - it("does not release another owner after the original guard was replaced", async () => { - const f = await fixture(); - const path = join(f.paths.store, services.RELEASE_STORE_LOCK_NAME); - renameSync(path, `${path}-original`); - const successor = await services.acquireReleaseStoreLock( - f.paths.store, - {}, - { failIfBusy: true }, - ); - locks.add(successor); - const owner = readFileSync(join(path, "owner.json"), "utf8"); - expect(() => f.release()).toThrow(/ownership.*changed/i); - expect(readFileSync(join(path, "owner.json"), "utf8")).toBe(owner); - }); -}); diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index 21d09740d..02f9c0879 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -13,12 +13,17 @@ import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { parse as parseYaml } from "yaml"; import { composePathHost, installedCompose } from "../__fixtures__/compose-path-host"; +import { releaseProvenance } from "../__fixtures__/release-provenance"; import { type ConfigurationInput, commitConfigurationGeneration, initializeConfigurationGenerations, readAppliedConfiguration, + readAppliedRelease, readAppliedServiceIds, + readGenerationConfiguration, + readGenerationRelease, + readPreviousConfigurationGeneration, renderConfiguration, withConfigurationGenerationWriter, } from "../configuration-generation"; @@ -31,6 +36,13 @@ import { stackComposeArgs, writeServiceSelectionFile, } from "../deployment"; +import { + canonicalReleaseManifest, + RELEASE_IMAGE_NAMES, + type ReleaseManifest, + releaseManifestDigest, + TRANSITOUS_TOOLS_IMAGE_ENV, +} from "../release-manifest"; import type { LoadedService } from "../types"; const roots: string[] = []; @@ -343,7 +355,6 @@ describe("stack compose arguments", () => { const paths = { infraDir: "/stack", composePath: `/stack/${CURRENT_CONFIGURATION}/docker-compose.generated.yml`, - composeReleasePath: "/stack/docker-compose.release.yml", }; expect(stackComposeArgs(paths, () => true)).toEqual([ "compose", @@ -351,8 +362,6 @@ describe("stack compose arguments", () => { "/stack/.env", "-f", `/stack/${CURRENT_CONFIGURATION}/docker-compose.generated.yml`, - "-f", - "/stack/docker-compose.release.yml", ]); expect(stackComposeArgs(paths, () => false)).toEqual([ "compose", @@ -437,7 +446,7 @@ it("applied selector and retention follow the current generation and explicit re ).toBe(45); }); -it("reads only frozen generated service IDs even when a release overlay declares disabled services", async () => { +it("reads only frozen generated service IDs", async () => { const { infraDir } = checkout(), generation = await commitConfigurationGeneration({ infraDir, @@ -445,13 +454,99 @@ it("reads only frozen generated service IDs even when a release overlay declares input: input({ selectedRoots: ["alpha"] }), }); const { readAppliedComposeServiceIds } = await import("../configuration-generation"); - writeFileSync( - join(infraDir, "docker-compose.release.yml"), - "services:\n app-api:\n image: example/api\n data-manager:\n image: example/dm\n", - { mode: 0o644 }, - ); expect(readAppliedComposeServiceIds(infraDir, generation.revisionId)).toEqual(["alpha"]); expect(() => readAppliedComposeServiceIds(infraDir, `cfg1_${"x".repeat(43)}`)).toThrow( /revision/, ); }); + +describe("release pins in the generation", () => { + const pinned = [service("app-api"), service("data-manager"), service("alpha")]; + const manifest = (release: string, digit: string): ReleaseManifest => ({ + schemaVersion: 1, + release, + images: Object.fromEntries( + RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${digit.repeat(64)}`, + ]), + ) as ReleaseManifest["images"], + ...releaseProvenance(), + databaseMigrationHead: { tag: "0064_example", when: 1794754360788 }, + }); + const compose = (infraDir: string) => + parseYaml( + readFileSync(join(infraDir, CURRENT_CONFIGURATION, "docker-compose.generated.yml"), "utf8"), + ) as { services: Record }> }; + const commit = (infraDir: string, release?: ReleaseManifest | null) => + commitConfigurationGeneration({ + infraDir, + services: pinned, + input: input({ selectedRoots: ["app-api", "data-manager", "alpha"] }), + ...(release !== undefined ? { release } : {}), + }); + + it("renders the release's images into the compose file and keeps its manifest", async () => { + const { infraDir } = checkout(); + const first = manifest("release-1", "1"); + await commit(infraDir, first); + const rendered = compose(infraDir); + expect(rendered.services["app-api"]?.image).toBe(first.images.api); + expect(rendered.services["data-manager"]?.image).toBe(first.images["data-manager"]); + expect(rendered.services["data-manager"]?.environment?.[TRANSITOUS_TOOLS_IMAGE_ENV]).toBe( + first.images["transitous-tools"], + ); + // A service the release does not cover keeps its manifest's image. + expect(rendered.services.alpha?.image).toBe("example/alpha:1"); + expect(readAppliedRelease(infraDir)).toEqual({ + manifest: first, + manifestDigest: releaseManifestDigest(first), + }); + expect( + readFileSync(join(infraDir, CURRENT_CONFIGURATION, "release-manifest.json"), "utf8"), + ).toBe(canonicalReleaseManifest(first)); + }); + + it("keeps the applied pins when a render names no release, and drops them on null", async () => { + const { infraDir } = checkout(); + const first = manifest("release-1", "1"); + await commit(infraDir, first); + await commit(infraDir); + expect(readAppliedRelease(infraDir)?.manifest.release).toBe("release-1"); + expect(compose(infraDir).services["app-api"]?.image).toBe(first.images.api); + await commit(infraDir, null); + expect(readAppliedRelease(infraDir)).toBeNull(); + expect(compose(infraDir).services["app-api"]?.image).toBe("example/app-api:1"); + expect(compose(infraDir).services["data-manager"]?.environment).toBeUndefined(); + }); + + it("records the generation it replaced and what it was rendered from", async () => { + const { infraDir } = checkout(); + const first = await commit(infraDir, manifest("release-1", "1")); + expect(readPreviousConfigurationGeneration(infraDir, first.revisionId)).toBeNull(); + const second = await commit(infraDir, manifest("release-2", "2")); + expect(readPreviousConfigurationGeneration(infraDir, second.revisionId)).toBe(first.revisionId); + expect(readGenerationConfiguration(infraDir, first.revisionId)).toMatchObject({ + domain: "maps.example.test", + selectedRoots: ["app-api", "data-manager", "alpha"], + release: manifest("release-1", "1"), + }); + expect(readGenerationRelease(infraDir, first.revisionId)?.manifest.release).toBe("release-1"); + }); + + it("keeps the recorded predecessor when an applied generation is committed again", async () => { + const { infraDir } = checkout(); + const first = await commit(infraDir); + const options = { + infraDir, + services: pinned, + input: input({ selectedRoots: ["app-api", "data-manager", "alpha"] }), + revisionId: `cfg1_${"r".repeat(43)}`, + }; + await commitConfigurationGeneration(options); + await commitConfigurationGeneration(options); + expect(readPreviousConfigurationGeneration(infraDir, options.revisionId)).toBe( + first.revisionId, + ); + }); +}); diff --git a/packages/core/src/services/__tests__/deployment-user.test.ts b/packages/core/src/services/__tests__/deployment-user.test.ts index dc150f09a..03dea3b1a 100644 --- a/packages/core/src/services/__tests__/deployment-user.test.ts +++ b/packages/core/src/services/__tests__/deployment-user.test.ts @@ -173,7 +173,6 @@ describe("the deployment's user", () => { const paths = { infraDir, composePath: join(infraDir, "docker-compose.generated.yml"), - composeReleasePath: join(infraDir, "docker-compose.release.yml"), }; const inherited = { UID: process.env.UID, GID: process.env.GID }; try { diff --git a/packages/core/src/services/__tests__/infra-state.test.ts b/packages/core/src/services/__tests__/infra-state.test.ts index 5a1d5b8e9..3b79215a5 100644 --- a/packages/core/src/services/__tests__/infra-state.test.ts +++ b/packages/core/src/services/__tests__/infra-state.test.ts @@ -188,38 +188,12 @@ describe("the writer lock", () => { }); }); -describe("the release overlay stack commands use", () => { - it("is used only as this user's own file", () => { - const { infraDir } = checkout(); - const overlay = join(infraDir, "state", "ops-agent", "releases", "docker-compose.release.yml"); - const paths = { - infraDir, - composePath: join(infraDir, CURRENT_CONFIGURATION, "docker-compose.generated.yml"), - composeReleasePath: overlay, - }; - expect(stackComposeArgs(paths)).not.toContain(overlay); - mkdirSync(join(infraDir, "state", "ops-agent", "releases"), { recursive: true, mode: 0o700 }); - writeFileSync(overlay, "services: {}\n", { mode: 0o600 }); - expect(stackComposeArgs(paths)).toContain(overlay); - chmodSync(overlay, 0o664); - expect(() => stackComposeArgs(paths)).toThrow("no group or other write"); - chmodSync(overlay, 0o600); - linkSync(overlay, join(infraDir, "overlay-alias.yml")); - expect(() => stackComposeArgs(paths)).toThrow("one link"); - rmSync(join(infraDir, "overlay-alias.yml")); - renameSync(overlay, join(infraDir, "overlay.yml")); - symlinkSync(join(infraDir, "overlay.yml"), overlay); - expect(() => stackComposeArgs(paths)).toThrow("regular file"); - }); -}); - describe("the deployment's .env", () => { it("is used only as the operator's own file", () => { const { infraDir } = checkout(); const paths = { infraDir, composePath: join(infraDir, CURRENT_CONFIGURATION, "docker-compose.generated.yml"), - composeReleasePath: join(infraDir, "state", "ops-agent", "releases", "release.yml"), }; const env = join(infraDir, ".env"); expect(stackComposeArgs(paths)).not.toContain("--env-file"); diff --git a/packages/core/src/services/__tests__/release-cache.test.ts b/packages/core/src/services/__tests__/release-cache.test.ts new file mode 100644 index 000000000..4abb7de60 --- /dev/null +++ b/packages/core/src/services/__tests__/release-cache.test.ts @@ -0,0 +1,117 @@ +import { chmodSync, mkdtempSync, readdirSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { releaseProvenance } from "../__fixtures__/release-provenance"; +import { + cachedReleases, + cacheReleaseManifest, + readCachedReleaseManifest, + readCheckedConfigurationRelease, + releaseCacheDirectory, +} from "../release-cache"; +import { + canonicalReleaseManifest, + RELEASE_IMAGE_NAMES, + type ReleaseManifest, + releaseChannel, + releaseManifestDigest, +} from "../release-manifest"; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); +function infra(): string { + const root = mkdtempSync(join(tmpdir(), "openmapx-release-cache-")); + roots.push(root); + return root; +} +function manifest(release: string, digit = "a"): ReleaseManifest { + return { + schemaVersion: 1, + release, + images: Object.fromEntries( + RELEASE_IMAGE_NAMES.map((name) => [ + name, + `ghcr.io/openmapx/${name}@sha256:${digit.repeat(64)}`, + ]), + ) as ReleaseManifest["images"], + ...releaseProvenance(), + }; +} +const channel = releaseChannel("main", "ghcr.io/openmapx/release-manifest"); + +it("caches a checked manifest under its digest and reads it back by that digest", () => { + const root = infra(); + const first = manifest("release-1"); + const cached = cacheReleaseManifest(root, first); + expect(cached).toEqual({ releaseId: "release-1", manifestDigest: releaseManifestDigest(first) }); + expect(readdirSync(releaseCacheDirectory(root))).toEqual([`${cached.manifestDigest}.json`]); + expect(readCachedReleaseManifest(root, cached.manifestDigest, "ghcr.io/openmapx")).toEqual(first); + // Caching the same release again is a no-op that marks it the newest. + expect(cacheReleaseManifest(root, first)).toEqual(cached); + expect(readCheckedConfigurationRelease(root, cached, channel)).toMatchObject({ + releaseId: "release-1", + serviceImages: { "app-api": first.images.api }, + }); +}); + +it("refuses a cached file whose bytes are not the manifest its name says", () => { + const root = infra(); + const cached = cacheReleaseManifest(root, manifest("release-1")); + const path = join(releaseCacheDirectory(root), `${cached.manifestDigest}.json`); + chmodSync(path, 0o600); + writeFileSync(path, canonicalReleaseManifest(manifest("release-2"))); + expect(() => readCachedReleaseManifest(root, cached.manifestDigest, "ghcr.io/openmapx")).toThrow( + /changed/, + ); + expect(() => cacheReleaseManifest(root, manifest("release-1"))).toThrow(/another manifest/); + expect(() => + readCheckedConfigurationRelease(root, { ...cached, releaseId: "release-2" }, channel), + ).toThrow(); +}); + +it("lists the releases of the configured repository, the one checked last first", () => { + const root = infra(); + const older = cacheReleaseManifest(root, manifest("release-1", "1")); + const newer = cacheReleaseManifest(root, manifest("release-2", "2")); + const past = new Date(Date.now() - 60_000); + utimesSync(join(releaseCacheDirectory(root), `${older.manifestDigest}.json`), past, past); + expect(cachedReleases(root, "ghcr.io/openmapx").map((entry) => entry.manifestDigest)).toEqual([ + newer.manifestDigest, + older.manifestDigest, + ]); + // Checking the older release again makes it the newest. + cacheReleaseManifest(root, manifest("release-1", "1")); + expect(cachedReleases(root, "ghcr.io/openmapx")[0]?.manifest.release).toBe("release-1"); + expect(cachedReleases(root, "registry.example.org/fork")).toEqual([]); +}); + +it("keeps at most 64 manifests, dropping the oldest", () => { + const root = infra(); + for (let index = 0; index < 66; index += 1) { + const cached = cacheReleaseManifest( + root, + manifest(`release-${index}`, (index % 10).toString()), + ); + const at = new Date(Date.now() - (66 - index) * 1000); + utimesSync(join(releaseCacheDirectory(root), `${cached.manifestDigest}.json`), at, at); + } + const releases = cachedReleases(root, "ghcr.io/openmapx"); + expect(releases).toHaveLength(64); + expect(releases.map((entry) => entry.manifest.release)).not.toContain("release-0"); + expect(releases[0]?.manifest.release).toBe("release-65"); +}); + +it("refuses a cache directory others can reach and a disabled channel", () => { + const root = infra(); + const cached = cacheReleaseManifest(root, manifest("release-1")); + expect(() => readCheckedConfigurationRelease(root, cached, releaseChannel("local"))).toThrow( + /disabled/, + ); + chmodSync(releaseCacheDirectory(root), 0o755); + expect(() => readCachedReleaseManifest(root, cached.manifestDigest, "ghcr.io/openmapx")).toThrow( + /Unsafe release cache/, + ); +}); diff --git a/packages/core/src/services/__tests__/release-manifest.test.ts b/packages/core/src/services/__tests__/release-manifest.test.ts index 912b41646..0bd596c8b 100644 --- a/packages/core/src/services/__tests__/release-manifest.test.ts +++ b/packages/core/src/services/__tests__/release-manifest.test.ts @@ -1,28 +1,12 @@ -import { - closeSync, - fstatSync, - lstatSync, - mkdtempSync, - openSync, - readFileSync, - rmSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { load } from "js-yaml"; import { describe, expect, it } from "vitest"; import { releaseProvenance } from "../__fixtures__/release-provenance"; import { canonicalReleaseManifest, - parseReleaseComposeSelection, parseReleaseManifest, type ReleaseManifest, releaseChannel, - renderReleaseCompose, - transitousToolsImageFromReleaseCompose, - writeReleaseComposeArtifacts, + releaseServiceImages, } from "../release-manifest"; -import { releaseStatePaths } from "../release-state"; const digest = (c: string) => `sha256:${c.repeat(64)}`; const manifest: ReleaseManifest = { @@ -40,7 +24,7 @@ const manifest: ReleaseManifest = { }; describe("shared release manifest", () => { - it("accepts a run-qualified mixed release and renders its exact reused digests", () => { + it("accepts a run-qualified mixed release and keeps its exact reused digests", () => { const mixed = { ...manifest, release: `${"a".repeat(40)}-123-2`, @@ -58,9 +42,24 @@ describe("shared release manifest", () => { }; const parsed = parseReleaseManifest(JSON.stringify(mixed)); expect(parsed).toEqual(mixed); - const overlay = renderReleaseCompose(parsed); - expect(overlay).toContain(`image: ${manifest.images.api}`); - expect(overlay).toContain(`image: ${manifest.images["ops-agent"]}`); + expect(releaseServiceImages(parsed)["app-api"]).toBe(manifest.images.api); + expect(releaseServiceImages(parsed)["ops-agent"]).toBe(manifest.images["ops-agent"]); + }); + + it("carries the release's database migration head through parsing and canonical bytes", () => { + const head = { tag: "0064_configuration_fence_drop_certificate", when: 1794754360788 }; + const withHead = { ...manifest, databaseMigrationHead: head }; + expect(parseReleaseManifest(JSON.stringify(withHead)).databaseMigrationHead).toEqual(head); + expect(JSON.parse(canonicalReleaseManifest(withHead)).databaseMigrationHead).toEqual(head); + expect(canonicalReleaseManifest(manifest)).not.toContain("databaseMigrationHead"); + for (const invalid of [ + { tag: "latest", when: 1 }, + { tag: head.tag, when: -1 }, + { tag: head.tag }, + ]) + expect(() => + parseReleaseManifest(JSON.stringify({ ...manifest, databaseMigrationHead: invalid })), + ).toThrow(/databaseMigrationHead/); }); it("parses an approved manifest and rejects tag references", () => { @@ -77,53 +76,9 @@ describe("shared release manifest", () => { it("pins only the six release images and no privacy collector or validation evidence", () => { expect(Object.keys(manifest.images)).toHaveLength(6); - const rendered = renderReleaseCompose(manifest); - const overlay = load(rendered) as { - services: Record; configs?: unknown[] }>; - configs?: unknown; - }; - expect(overlay.configs).toBeUndefined(); - expect(overlay.services["app-api"]).toEqual({ image: manifest.images.api }); - expect(overlay.services["ops-agent"]).toEqual({ image: manifest.images["ops-agent"] }); - expect(rendered).not.toMatch(/privacy/i); + expect(Object.keys(releaseServiceImages(manifest))).toHaveLength(5); expect(canonicalReleaseManifest(manifest)).not.toMatch(/privacy/i); }); - - it("writes a private release overlay", () => { - const root = mkdtempSync(join(tmpdir(), "openmapx-release-artifacts-")); - try { - const overlayPath = join( - root, - "state", - "ops-agent", - "releases", - "docker-compose.release.yml", - ); - expect(writeReleaseComposeArtifacts(manifest, root)).toEqual({ overlayPath }); - expect(lstatSync(releaseStatePaths(root).directory).mode & 0o777).toBe(0o700); - const overlay = openSync(overlayPath, "r"); - try { - expect(fstatSync(overlay).mode & 0o777).toBe(0o600); - expect(readFileSync(overlay, "utf8")).toBe(renderReleaseCompose(manifest)); - } finally { - closeSync(overlay); - } - } finally { - rmSync(root, { recursive: true, force: true }); - } - }); - - it("replaces the overlay atomically when another release is selected", () => { - const root = mkdtempSync(join(tmpdir(), "openmapx-release-artifacts-")); - try { - const { overlayPath } = writeReleaseComposeArtifacts(manifest, root); - const next = { ...manifest, release: "def456" }; - writeReleaseComposeArtifacts(next, root); - expect(readFileSync(overlayPath, "utf8")).toBe(renderReleaseCompose(next)); - } finally { - rmSync(root, { recursive: true, force: true }); - } - }); }); describe("release channel", () => { @@ -173,38 +128,4 @@ describe("release lockfile image contract and selection", () => { expect(canonicalReleaseManifest(input)).toBe(JSON.stringify(manifest)); }, ); - - it("round trips selected identity and six images, including quoted release IDs", () => { - const selected = { ...manifest, release: "release-test" }; - const images = manifest.images; - const rendered = renderReleaseCompose(selected); - expect(parseReleaseComposeSelection(rendered)).toEqual({ release: selected.release, images }); - expect(transitousToolsImageFromReleaseCompose(rendered)).toBe(images["transitous-tools"]); - expect(transitousToolsImageFromReleaseCompose("services: {}\n")).toBeNull(); - expect(rendered).not.toContain("ghcr.io/openmapx/docs"); - }); - - it("reads legacy service pins without inventing a release identity", () => { - expect( - parseReleaseComposeSelection(`services: - app-api: - image: ${manifest.images.api} -`), - ).toEqual({ - release: null, - images: { api: manifest.images.api }, - }); - expect(() => parseReleaseComposeSelection("services: [")).toThrow(); - expect(() => parseReleaseComposeSelection("hello")).toThrow(); - }); -}); - -it("does not attribute a modified service pin to stale release metadata", () => { - const rendered = renderReleaseCompose(manifest).replace( - `image: ${manifest.images.api}`, - `image: ghcr.io/openmapx/api@${digest("0")}`, - ); - const selected = parseReleaseComposeSelection(rendered); - expect(selected.release).toBeNull(); - expect(selected.images.api).toBe(`ghcr.io/openmapx/api@${digest("0")}`); }); diff --git a/packages/core/src/services/__tests__/release-store-lock-acquisition.test.ts b/packages/core/src/services/__tests__/release-store-lock-acquisition.test.ts deleted file mode 100644 index da6d76ec1..000000000 --- a/packages/core/src/services/__tests__/release-store-lock-acquisition.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { - existsSync, - mkdirSync, - mkdtempSync, - readFileSync, - renameSync, - rmSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, expect, it, vi } from "vitest"; -import { acquireReleaseStoreLock, RELEASE_STORE_LOCK_NAME } from "../release-store-lock"; - -const fault = vi.hoisted(() => ({ path: "", beforeFault: undefined as (() => void) | undefined })); -vi.mock("node:fs", async (original) => { - const fs = await original(); - return { - ...fs, - lstatSync: (...args: Parameters) => { - if (String(args[0]) === fault.path) { - fault.path = ""; - fault.beforeFault?.(); - throw Object.assign(Error("Synthetic observation failure"), { code: "EIO" }); - } - return fs.lstatSync(...args); - }, - }; -}); -let root = ""; -afterEach(() => { - fault.path = ""; - fault.beforeFault = undefined; - if (root) rmSync(root, { recursive: true, force: true }); -}); -it("releases a physical lock when post-acquisition observation fails before exposing a holder", async () => { - root = mkdtempSync(join(tmpdir(), "openmapx-lock-acquisition-")); - fault.path = join(root, RELEASE_STORE_LOCK_NAME); - await expect(acquireReleaseStoreLock(root)).rejects.toThrow("Synthetic observation failure"); - expect(existsSync(join(root, RELEASE_STORE_LOCK_NAME))).toBe(false); - const next = await acquireReleaseStoreLock(root, {}, { failIfBusy: true }); - next.release(); -}); - -it("does not delete a successor while cleaning a failed acquisition observation", async () => { - root = mkdtempSync(join(tmpdir(), "openmapx-lock-acquisition-")); - const path = join(root, RELEASE_STORE_LOCK_NAME); - fault.path = path; - const successor = JSON.stringify({ - pid: process.pid, - nonce: "successor", - acquiredAtMs: Date.now(), - leaseMs: 900000, - }); - fault.beforeFault = () => { - renameSync(path, `${path}-original`); - mkdirSync(path); - writeFileSync(join(path, "owner.json"), successor, { mode: 0o600 }); - }; - await expect(acquireReleaseStoreLock(root)).rejects.toThrow("Synthetic observation failure"); - expect(readFileSync(join(path, "owner.json"), "utf8")).toBe(successor); -}); diff --git a/packages/core/src/services/__tests__/release-store-lock.test.ts b/packages/core/src/services/__tests__/release-store-lock.test.ts deleted file mode 100644 index 846e2bea9..000000000 --- a/packages/core/src/services/__tests__/release-store-lock.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { - linkSync, - mkdirSync, - mkdtempSync, - readdirSync, - readFileSync, - rmSync, - symlinkSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { acquireStoreLock } from "../release-store-lock"; - -let root: string; -const lockName = ".test-release-lock"; -beforeEach(() => { - root = mkdtempSync(join(tmpdir(), "openmapx-release-lock-")); -}); -afterEach(() => { - rmSync(root, { recursive: true, force: true }); -}); - -describe("release-store lock filesystem policy", () => { - it("does not reclaim a live backup operation before its fixed 35-minute lease", async () => { - const first = await acquireStoreLock(root, lockName, {}, () => 0, false, 35 * 60000); - let reads = 0; - const later = () => 20 * 60000 + (++reads > 2 ? 61000 : 0); - try { - await expect(acquireStoreLock(root, lockName, {}, later, false, 35 * 60000)).rejects.toThrow( - /busy/, - ); - } finally { - first.release(); - } - }); - it("publishes its owner, rejects a competing holder, and allows reuse after release", async () => { - const first = await acquireStoreLock(root, lockName); - const owner = JSON.parse(readFileSync(join(root, lockName, "owner.json"), "utf8")); - expect(owner.pid).toBe(process.pid); - expect(typeof owner.nonce).toBe("string"); - expect(Number.isFinite(owner.acquiredAtMs)).toBe(true); - await expect(acquireStoreLock(root, lockName, {}, Date.now, true)).rejects.toThrow(/busy/); - first.release(); - expect(readdirSync(root)).toEqual([]); - const second = await acquireStoreLock(root, lockName); - try { - expect(readdirSync(join(root, lockName))).toEqual(["owner.json"]); - } finally { - second.release(); - } - }); - - it("reads and reclaims an expired valid owner without changing lease policy", async () => { - const directory = join(root, lockName); - mkdirSync(directory); - writeFileSync(join(directory, "owner.json"), JSON.stringify({ acquiredAtMs: 0 })); - const lock = await acquireStoreLock(root, lockName, {}, () => 1_000_000); - try { - expect(JSON.parse(readFileSync(join(directory, "owner.json"), "utf8")).acquiredAtMs).toBe( - 1_000_000, - ); - expect(readdirSync(directory)).toEqual(["owner.json"]); - } finally { - lock.release(); - } - }); - - it("reads a live valid owner and reports contention at its acquisition deadline", async () => { - const directory = join(root, lockName); - mkdirSync(directory); - const bytes = JSON.stringify({ acquiredAtMs: 0 }); - writeFileSync(join(directory, "owner.json"), bytes); - let calls = 0; - await expect( - acquireStoreLock(root, lockName, {}, () => (calls++ === 0 ? 0 : 60_000)), - ).rejects.toThrow(/busy/); - expect(readFileSync(join(directory, "owner.json"), "utf8")).toBe(bytes); - }); - - it.each(["symlink", "hardlink", "directory", "oversize", "invalid UTF-8", "invalid JSON"])( - "rejects an unsafe %s owner instead of reclaiming it", - async (kind) => { - const directory = join(root, lockName); - mkdirSync(directory); - const owner = join(directory, "owner.json"); - const target = join(root, "target"); - const contents = JSON.stringify({ acquiredAtMs: 0 }); - if (kind === "directory") mkdirSync(owner); - else if (kind === "symlink" || kind === "hardlink") { - writeFileSync(target, contents); - if (kind === "symlink") symlinkSync(target, owner); - else linkSync(target, owner); - } else - writeFileSync( - owner, - kind === "oversize" - ? "x".repeat(4097) - : kind === "invalid UTF-8" - ? Buffer.from([0xff]) - : "{", - ); - await expect(acquireStoreLock(root, lockName, {}, () => 1_000_000)).rejects.toThrow( - /unreadable/, - ); - expect(readdirSync(directory)).toEqual(["owner.json"]); - if (kind === "symlink" || kind === "hardlink") - expect(readFileSync(target, "utf8")).toBe(contents); - }, - ); -}); diff --git a/packages/core/src/services/checked-configuration-release.ts b/packages/core/src/services/checked-configuration-release.ts deleted file mode 100644 index a99c416f5..000000000 --- a/packages/core/src/services/checked-configuration-release.ts +++ /dev/null @@ -1,79 +0,0 @@ -/** Checked release data only. Reading this does not acquire targets or grant effects. */ - -import { createHash } from "node:crypto"; -import { relative, sep } from "node:path"; -import { z } from "zod"; -import { AnchoredDirectory } from "../build-artifacts/filesystem"; -import { - canonicalReleaseManifest, - parseReleaseManifest, - ReleaseChannelDisabledError, -} from "./release-manifest"; -import type { ReleaseSelection } from "./release-selection"; -import { releaseStatePaths } from "./release-state"; - -const selectionSchema = z.strictObject({ - releaseId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/), - manifestDigest: z.string().regex(/^[a-f0-9]{64}$/), -}); -function immutable(value: T): T { - if (value && typeof value === "object") { - for (const child of Object.values(value)) immutable(child); - Object.freeze(value); - } - return value; -} -/** Snapshot the existing private checked store, under its existing32KiB limit. - * Effect producers must revalidate this same identity under their real locks; - * neither the parsed data nor a matching digest is mutation authority. */ -export function readCheckedConfigurationRelease( - infraDir: string, - input: { releaseId: string; manifestDigest: string }, - channel: ReleaseSelection, -) { - const selected = selectionSchema.parse(input); - if (channel.kind === "local") throw new ReleaseChannelDisabledError(); - const infra = new AnchoredDirectory(infraDir); - try { - const parents: AnchoredDirectory[] = []; - let directory = infra; - try { - for (const part of relative(infraDir, releaseStatePaths(infraDir).store).split(sep)) { - directory = directory.directory(part, false, true); - parents.push(directory); - } - } catch (error) { - for (const parent of parents.reverse()) parent.close(); - throw error; - } - try { - const raw = directory.read(`${selected.releaseId}.json`, 32 * 1024).toString("utf8"), - manifest = parseReleaseManifest(raw, channel.imagePrefix), - canonical = canonicalReleaseManifest(manifest), - digest = createHash("sha256").update(canonical).digest("hex"); - if ( - manifest.release !== selected.releaseId || - canonical !== raw || - digest !== selected.manifestDigest - ) - throw Error("Checked configuration release changed"); - return immutable({ - releaseId: selected.releaseId, - manifestDigest: digest, - manifest, - serviceImages: { - "app-api": manifest.images.api, - "app-web": manifest.images.web, - "data-manager": manifest.images["data-manager"], - "ops-agent": manifest.images["ops-agent"], - "transitous-runner": manifest.images["transitous-runner"], - }, - }); - } finally { - for (const parent of parents.reverse()) parent.close(); - } - } finally { - infra.close(); - } -} -export type CheckedConfigurationRelease = ReturnType; diff --git a/packages/core/src/services/checked-release.ts b/packages/core/src/services/checked-release.ts deleted file mode 100644 index 03a023141..000000000 --- a/packages/core/src/services/checked-release.ts +++ /dev/null @@ -1,189 +0,0 @@ -/** Checked metadata only; publication here selects no images and grants no effects. */ -import { createHash, randomUUID } from "node:crypto"; -import { - closeSync, - constants, - fsyncSync, - linkSync, - lstatSync, - openSync, - readdirSync, - unlinkSync, - writeFileSync, -} from "node:fs"; -import { dirname, join } from "node:path"; -import { atomicWrite, fsyncDirectory } from "../release-files"; -import { canonicalReleaseManifest, type ReleaseManifest } from "./release-manifest"; -import type { ReleaseSelection } from "./release-selection"; -import { - assertReleaseStateDirectory, - parseReleaseSelectionReceipts, - RELEASE_ID_PATTERN, - RELEASE_MANIFEST_FILE_PATTERN, - RELEASE_STORE_LIMITS, - RELEASE_STORE_POINTER_FILES, - type ReleaseStateOptions, - readReleaseStateFile, - releaseStateFileProblem, - releaseStatePaths, -} from "./release-state"; -import { - assertReleaseStoreLock, - RELEASE_STORE_LOCK_NAME, - type ReleaseStoreLock, -} from "./release-store-lock"; - -const digest = (raw: string) => createHash("sha256").update(raw).digest("hex"); - -export function releaseConfigurationIdentity(selection: ReleaseSelection): string { - return digest(JSON.stringify(selection)); -} - -/** Same bounded enumeration used by legacy resolution and inspection. */ -export function listReleaseStoreManifests( - directory: string, - options: ReleaseStateOptions = {}, -): string[] { - assertReleaseStateDirectory(directory, options); - const entries = readdirSync(directory); - if (entries.length > RELEASE_STORE_LIMITS.entries + 5) - throw Error("Release store limit exceeded"); - const names: string[] = []; - let bytes = 0; - for (const name of entries) { - const path = join(directory, name); - const stat = lstatSync(path); - if (name === RELEASE_STORE_LOCK_NAME) { - const uid = options.uid ?? process.geteuid?.(); - if ( - !stat.isDirectory() || - stat.isSymbolicLink() || - (uid !== undefined && stat.uid !== uid) || - (stat.mode & 0o022) !== 0 - ) - throw Error("Release store contains an unsafe entry"); - continue; - } - if ((RELEASE_STORE_POINTER_FILES as readonly string[]).includes(name)) { - const maximum = - name === "selections.json" - ? RELEASE_STORE_LIMITS.selectionsBytes - : RELEASE_STORE_LIMITS.stateBytes; - if (stat.size < 1 || releaseStateFileProblem(stat, maximum, options)) - throw Error("Release store contains an unsafe entry"); - continue; - } - const releaseId = RELEASE_MANIFEST_FILE_PATTERN.exec(name)?.[1]; - if ( - !releaseId || - stat.size < 1 || - releaseStateFileProblem(stat, RELEASE_STORE_LIMITS.manifestBytes, options) - ) - throw Error("Release store contains an unsafe entry"); - bytes += stat.size; - if ( - bytes > RELEASE_STORE_LIMITS.entries * RELEASE_STORE_LIMITS.manifestBytes || - names.length >= RELEASE_STORE_LIMITS.entries - ) - throw Error("Release store limit exceeded"); - names.push(releaseId); - } - return names; -} - -function immutableWrite(path: string, raw: string, options: ReleaseStateOptions) { - const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`; - const fd = openSync( - temporary, - constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, - 0o600, - ); - try { - writeFileSync(fd, raw, "utf8"); - fsyncSync(fd); - } finally { - closeSync(fd); - } - let failure: unknown; - try { - linkSync(temporary, path); - fsyncDirectory(dirname(path)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") failure = error; - else { - try { - if (readReleaseStateFile(path, RELEASE_STORE_LIMITS.manifestBytes, options) !== raw) - throw Error("Release authority rejected"); - } catch (readError) { - failure = readError; - } - } - } - try { - unlinkSync(temporary); - fsyncDirectory(dirname(path)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT" && failure === undefined) - failure = error; - } - if (failure !== undefined) throw failure; -} - -/** Caller already owns the actual release-store lock. Transaction policy is its own. */ -export function cacheCheckedReleaseManifest(input: { - infraDir: string; - manifest: ReleaseManifest; - ociDigest: string; - configurationIdentity: string; - lock: ReleaseStoreLock; - uid?: number; -}): { releaseId: string; manifestDigest: string } { - const paths = releaseStatePaths(input.infraDir); - const options = { uid: input.uid }; - assertReleaseStateDirectory(paths.directory, options); - assertReleaseStateDirectory(paths.store, options); - assertReleaseStoreLock(input.lock, paths.store); - if ( - !RELEASE_ID_PATTERN.test(input.manifest.release) || - (RELEASE_STORE_POINTER_FILES as readonly string[]).includes(`${input.manifest.release}.json`) || - !/^sha256:[a-f0-9]{64}$/.test(input.ociDigest) || - !/^[a-f0-9]{64}$/.test(input.configurationIdentity) - ) - throw Error("Invalid or reserved checked release identity"); - const canonical = canonicalReleaseManifest(input.manifest); - if (Buffer.byteLength(canonical) > RELEASE_STORE_LIMITS.manifestBytes) - throw Error("Release manifest limit exceeded"); - const names = listReleaseStoreManifests(paths.store, options); - if (!names.includes(input.manifest.release) && names.length >= RELEASE_STORE_LIMITS.entries) - throw Error("Release store limit exceeded"); - let receipts = {}; - try { - receipts = parseReleaseSelectionReceipts( - readReleaseStateFile( - join(paths.store, "selections.json"), - RELEASE_STORE_LIMITS.selectionsBytes, - options, - ), - ); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - } - const manifestDigest = digest(canonical); - const updated = parseReleaseSelectionReceipts( - JSON.stringify({ - ...receipts, - [input.manifest.release]: { - configuration: input.configurationIdentity, - ociDigest: input.ociDigest, - digest: manifestDigest, - }, - }), - ); - immutableWrite(join(paths.store, `${input.manifest.release}.json`), canonical, options); - atomicWrite(join(paths.store, "selections.json"), JSON.stringify(updated)); - atomicWrite( - join(paths.store, "latest.json"), - JSON.stringify({ releaseId: input.manifest.release, digest: manifestDigest }), - ); - return { releaseId: input.manifest.release, manifestDigest }; -} diff --git a/packages/core/src/services/compose-renderer.ts b/packages/core/src/services/compose-renderer.ts index 4b17b22bb..9078fcc86 100644 --- a/packages/core/src/services/compose-renderer.ts +++ b/packages/core/src/services/compose-renderer.ts @@ -11,6 +11,7 @@ import { PROXY_ENTRYPOINT_LABEL_VALUE, readExternalProxyNetwork, } from "./external-proxy"; +import { TRANSITOUS_TOOLS_IMAGE_ENV } from "./release-manifest"; import { detectConsumesCycle } from "./resolver"; import { isRuntimeResourceKey, @@ -203,6 +204,14 @@ function hardlinkTargetDir(consumerId: string, c: ServiceConsumes): string { } export interface RenderContext { + /** + * The release a generation pins: the image of each pinned built-in service, + * and the Transitous tools image data-manager runs its jobs with. + */ + releasePins?: { + images: Readonly>; + transitousTools: string; + }; platform?: import("./platform-configuration").PlatformConfiguration; platformValues?: import("./platform-configuration").PlatformValues; domain?: string; @@ -591,8 +600,9 @@ export function renderServiceSnippet( const m = service.manifest; const c = m.container; + const pinned = service.isBuiltIn ? ctx.releasePins?.images[m.id] : undefined; const snippet: ComposeServiceSnippet = { - image: serviceContainerImageReference(c), + image: pinned ?? serviceContainerImageReference(c), }; // Pin the container name when the manifest opts in — see ServiceContainer. @@ -730,6 +740,11 @@ export function renderServiceSnippet( ); snippet.environment = env; } + if (service.isBuiltIn && m.id === "data-manager" && ctx.releasePins) + snippet.environment = { + ...snippet.environment, + [TRANSITOUS_TOOLS_IMAGE_ENV]: ctx.releasePins.transitousTools, + }; // Vault secrets (the container track): mount each as a Docker secret at // `/run/secrets/` and point `_FILE` at it. The value never enters // the environment (which is exposed via `docker inspect`/`/proc/environ`); diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 20ae42872..a26d9d38e 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -61,6 +61,13 @@ import { platformConfigurationSchema, resolvePlatformConfiguration, } from "./platform-configuration"; +import { + canonicalReleaseManifest, + parseReleaseManifest, + type ReleaseManifest, + releaseManifestDigest, + releaseServiceImages, +} from "./release-manifest"; import { normalizeReleaseSelection } from "./release-selection"; import { type ReleaseServiceFiles, @@ -90,6 +97,10 @@ import { seedMissingWritableCopies, seedWritableCopies } from "./writable-copies const MAX_GENERATIONS = 16; const MAX_GENERATION_BYTES = 2 * 1024 * 1024; const RESOLVED_CONFIG_FILE = "resolved-config.generated.json"; +/** The release whose images the generation pins, as its canonical manifest. */ +const RELEASE_MANIFEST_FILE = "release-manifest.json"; +/** Which generation the pointer last moved from, and to which. */ +const POINTER_MOVE_FILE = `${CONFIGURATION_DIRECTORY}/.last-pointer-move.json`; const REVISION = /^cfg1_[A-Za-z0-9_-]{43}$/; /** @@ -131,6 +142,11 @@ export interface CommitConfigurationOptions { * bundled, otherwise the checkout's). */ releaseServiceFiles?: ReleaseServiceFiles; + /** + * The release whose images the generation pins. Omitted, the applied + * generation's release is kept; null renders the manifests' own image tags. + */ + release?: ReleaseManifest | null; /** * The bind mounts of every container, running or stopped: a service-file * tree one of them mounts is kept. Omitted (Docker can't tell), no tree is @@ -762,7 +778,7 @@ export function readAppliedServiceIds( return new Set(expanded.enabledIds); } -/** Exact service closure of the frozen generated base, excluding release-overlay stubs. */ +/** Exact service closure of the frozen generated base. */ export function readAppliedComposeServiceIds( infraDir: string, expectedRevisionId: string, @@ -800,9 +816,14 @@ function readOwnGenerationFile(path: string, modes: readonly bigint[]): string { export function readAppliedConfiguration( infraDir: string, ): Omit { - const empty = { serviceConfigs: [], integrationConfigs: [], serviceSecrets: [] }; const generation = appliedGenerationDirectory(infraDir); - if (!generation) return empty; + if (!generation) return { serviceConfigs: [], integrationConfigs: [], serviceSecrets: [] }; + return readGenerationInputs(generation); +} + +function readGenerationInputs( + generation: string, +): Omit { // Each file is opened without following links and must be this user's // own, as the generation's tree was checked. const resolved = JSON.parse( @@ -885,6 +906,107 @@ export function readAppliedReleaseSelection( ); } +export interface GenerationRelease { + manifest: ReleaseManifest; + manifestDigest: string; +} + +/** The release a retained generation pins, or null when it runs the manifests' own tags. */ +export function readGenerationRelease( + infraDir: string, + revisionId: string, +): GenerationRelease | null { + const directory = dirname(configurationGenerationComposeFile(infraDir, revisionId)); + let raw: string; + try { + raw = readOwnGenerationFile(join(directory, RELEASE_MANIFEST_FILE), [0o400n, 0o600n]); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + // The pins were checked against the repository configured at commit; they + // must still share one image prefix. + const api = (JSON.parse(raw) as { images?: { api?: unknown } })?.images?.api; + const manifest = parseReleaseManifest( + raw, + typeof api === "string" ? api.slice(0, Math.max(0, api.lastIndexOf("/api@"))) : "", + ); + if (canonicalReleaseManifest(manifest) !== raw) throw new Error(UNSAFE); + return { manifest, manifestDigest: releaseManifestDigest(manifest) }; +} + +/** The release the applied generation pins, or null. */ +export function readAppliedRelease(infraDir: string): GenerationRelease | null { + const active = currentConfigurationGeneration(infraDir); + return active ? readGenerationRelease(infraDir, active) : null; +} + +/** + * The generation that was applied before `revisionId`, or null when that is + * unknown: the first generation, or a pointer move whose record was not + * written. Only the latest move is recorded, so only the applied generation + * has a predecessor. + */ +export function readPreviousConfigurationGeneration( + infraDir: string, + revisionId: string, +): string | null { + const path = join(infraDir, POINTER_MOVE_FILE); + let move: { from?: unknown; to?: unknown }; + try { + move = JSON.parse(readOwnGenerationFile(path, [0o600n])); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + if (move.to !== revisionId) return null; + if (typeof move.from !== "string" || !REVISION.test(move.from)) throw new Error(UNSAFE); + return move.from; +} + +async function recordPointerMove(infraDir: string, from: string | null, to: string): Promise { + const path = join(infraDir, POINTER_MOVE_FILE); + if (!from) { + rmSync(path, { force: true }); + return; + } + const temporary = `${path}.tmp`; + rmSync(temporary, { force: true }); + try { + await writeDurable(temporary, `${JSON.stringify({ from, to })}\n`, 0o600); + renameSync(temporary, path); + await syncDirectory(dirname(path)); + } finally { + rmSync(temporary, { force: true }); + } +} + +/** + * What a retained generation was rendered from: its domain, configuration, + * secrets, selection and release, to render it again. + */ +export function readGenerationConfiguration( + infraDir: string, + revisionId: string, +): ConfigurationInput & { release: ReleaseManifest | null } { + const directory = dirname(configurationGenerationComposeFile(infraDir, revisionId)); + const { domain } = JSON.parse( + readOwnGenerationFile(join(directory, RESOLVED_CONFIG_FILE), [0o400n, 0o600n]), + ) as { domain?: unknown }; + if (typeof domain !== "string") throw new Error(UNSAFE); + const selection = JSON.parse( + readOwnGenerationFile(join(directory, SERVICE_SELECTION_FILE), [0o400n, 0o600n]), + ) as { selected?: unknown }; + if (!Array.isArray(selection.selected) || selection.selected.some((id) => typeof id !== "string")) + throw new Error(UNSAFE); + return { + ...readGenerationInputs(directory), + domain, + selectedRoots: selection.selected as string[], + release: readGenerationRelease(infraDir, revisionId)?.manifest ?? null, + }; +} + /** Call under writer ownership so the pointer and its projections are one observation. * Only the resolved integration file is read: service secrets never enter this result. */ @@ -1264,6 +1386,7 @@ export type RenderConfigurationOptions = Pick< | "input" | "allowMissingSelected" | "releaseServiceFiles" + | "release" >; /** @@ -1279,10 +1402,22 @@ export function renderConfiguration(options: RenderConfigurationOptions) { input, options.allowMissingSelected ?? false, ); - const release = options.releaseServiceFiles ?? releaseServiceFiles(options.services); - const overrides = readServiceOverrides(infraDir, release); - const serviceFiles = resolveServiceFiles(options.services, release, overrideOverlay(overrides)); + const files = options.releaseServiceFiles ?? releaseServiceFiles(options.services); + const overrides = readServiceOverrides(infraDir, files); + const serviceFiles = resolveServiceFiles(options.services, files, overrideOverlay(overrides)); + const release = + options.release === undefined + ? (readAppliedRelease(infraDir)?.manifest ?? null) + : options.release; const context: RenderContext = { + ...(release + ? { + releasePins: { + images: releaseServiceImages(release), + transitousTools: release.images["transitous-tools"], + }, + } + : {}), domain: input.domain, ...(input.platform ? { @@ -1316,6 +1451,7 @@ export function renderConfiguration(options: RenderConfigurationOptions) { rendered: renderCompose(resolved.enabled, context), serviceFiles, overrides, + release, }; } @@ -1379,7 +1515,7 @@ async function commitOwnedConfigurationFiles( release: ReleaseServiceFiles, ): Promise { const { infraDir, input } = options; - const { resolved, context, rendered, serviceFiles, overrides } = prepared; + const { resolved, context, rendered, serviceFiles, overrides, release: pinned } = prepared; const generations = join(infraDir, CONFIGURATION_GENERATIONS); await retainConfigurationGenerations( infraDir, @@ -1410,6 +1546,7 @@ async function commitOwnedConfigurationFiles( await writeDurable( join(temporary, RESOLVED_CONFIG_FILE), `${JSON.stringify({ + domain: input.domain, serviceConfigs: input.serviceConfigs, integrationConfigs: input.integrationConfigs, ...(input.transitCredentialsRevision @@ -1419,6 +1556,12 @@ async function commitOwnedConfigurationFiles( })}\n`, 0o600, ); + if (pinned) + await writeDurable( + join(temporary, RELEASE_MANIFEST_FILE), + canonicalReleaseManifest(pinned), + 0o600, + ); // The service files its containers mount, in the store beside the // generations; the generation records which trees it uses. writeServiceFileTrees(infraDir, serviceFiles.trees.values()); @@ -1518,6 +1661,7 @@ async function commitOwnedConfigurationFiles( serviceFileReport: [], }; await syncDirectory(dirname(join(infraDir, CURRENT_CONFIGURATION))); + await recordPointerMove(infraDir, currentRevisionId, revisionId); } else { committed = { revisionId, diff --git a/packages/core/src/services/deployment.ts b/packages/core/src/services/deployment.ts index 32e33d518..0489d2346 100644 --- a/packages/core/src/services/deployment.ts +++ b/packages/core/src/services/deployment.ts @@ -70,7 +70,6 @@ export interface StackPaths { infraDir: string; /** The applied generation's compose file. */ composePath: string; - composeReleasePath: string; } export function currentConfigurationFile(infraDir: string, name: string): string { @@ -93,7 +92,6 @@ export function stackComposeArgs( const inherited = deploymentUserEnvironmentProblem(paths.infraDir); if (inherited) throw new Error(inherited); } - const hasOverlay = (fileExists ?? ownedStateFilePresent)(paths.composeReleasePath); // Compose records this path in container provenance. Pin the generation, // rather than the movable current symlink, so deselection retains cleanup authority. let composePath = paths.composePath; @@ -123,7 +121,6 @@ export function stackComposeArgs( ...(hasEnvFile ? ["--env-file", envFile] : []), "-f", composePath, - ...(hasOverlay ? ["-f", paths.composeReleasePath] : []), ]; } @@ -181,33 +178,6 @@ export function deploymentUserEnvironmentProblem( return null; } -/** - * Whether a file of the agent's state is there: false when it is missing, and - * true only for a regular file of this user with one link and no group or - * other write. Anything else in its place is refused, never passed on. - */ -export function ownedStateFilePresent(path: string): boolean { - let stats: ReturnType; - try { - stats = lstatSync(path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; - throw error; - } - const owner = process.geteuid?.(); - if ( - !stats.isFile() || - stats.nlink !== 1 || - (owner !== undefined && stats.uid !== owner) || - (stats.mode & 0o022) !== 0 - ) { - throw new Error( - `${path} must be a regular file owned by uid ${owner}, with one link and no group or other write`, - ); - } - return true; -} - /** * `infra/docker/.env` holds the deployment's credentials and settings, and * Compose interpolates it into every service. Like a file under `secrets/`, it diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 73b67d31d..f19ee054a 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -28,12 +28,6 @@ export { WELL_KNOWN_CAPABILITIES, WELL_KNOWN_DATA_TYPES, } from "./capabilities"; -export * from "./checked-configuration-release"; -export { - cacheCheckedReleaseManifest, - listReleaseStoreManifests, - releaseConfigurationIdentity, -} from "./checked-release"; export { type CheckoutDifference, checkoutDifferencesFromRelease, @@ -88,6 +82,7 @@ export { configurationGenerationComposeFile, configurationGenerationWriterCommit, currentConfigurationGeneration, + type GenerationRelease, initializeConfigurationGenerations, inspectAppliedConfiguration, type RenderConfigurationOptions, @@ -95,9 +90,13 @@ export { readAppliedConfiguration, readAppliedPlatformConfiguration, readAppliedPlatformValues, + readAppliedRelease, readAppliedReleaseSelection, readAppliedServiceIds, readAppliedTransitCredentialsRevision, + readGenerationConfiguration, + readGenerationRelease, + readPreviousConfigurationGeneration, renderConfiguration, withConfigurationGenerationWriter, } from "./configuration-generation"; @@ -160,7 +159,6 @@ export { GENERATED_COMPOSE_FILE, GENERATED_HARDLINK_PLAN_FILE, loadIntegrationSchemas, - ownedStateFilePresent, readDesiredSelection, readOwnedStateFile, readServiceSelectionFile, @@ -259,17 +257,23 @@ export { readBuiltInServices, validateReleaseServiceAuthority, } from "./release-authority-inventory"; +export { + type CheckedConfigurationRelease, + cachedReleases, + cacheReleaseManifest, + readCachedReleaseManifest, + readCheckedConfigurationRelease, + releaseCacheDirectory, +} from "./release-cache"; export { configurationImageReference, readServiceReleaseImageSelection, - validateExistingReleaseSelection, } from "./release-image-selection"; export { canonicalReleaseManifest, DEFAULT_RELEASE_MANIFEST_REPOSITORY, normalizeReleaseSelection, type ProductReleaseMetadata, - parseReleaseComposeSelection, parseReleaseManifest, RELEASE_IMAGE_NAMES, RELEASE_MANIFEST_CONTAINER_PATH, @@ -278,21 +282,18 @@ export { RELEASE_SELECTOR_ENV, type ReleaseChannel, ReleaseChannelDisabledError, - type ReleaseComposeSelection, type ReleaseManifest, type ReleaseProducer, type ReleaseSelection, releaseChannel, releaseDigestFromPull, + releaseManifestDigest, releaseManifestImage, releaseSelection, - renderReleaseCompose, + releaseServiceImages, TRANSITOUS_TOOLS_IMAGE_ENV, - transitousToolsImageFromReleaseCompose, validateResolvedSelection, - writeReleaseComposeArtifacts, } from "./release-manifest"; -export { clearReleaseComposeSelection } from "./release-selection-clear"; export { parseReleaseServiceFiles, type ReleaseServiceFile, @@ -304,36 +305,6 @@ export { serializeReleaseServiceFiles, serviceFileReferences, } from "./release-service-files"; -export { - assertReleaseStateDirectory, - ensureReleaseStateDirectories, - ensureReleaseStateDirectory, - parseReleasePointer, - parseReleaseSelectionReceipts, - RELEASE_ID_PATTERN, - RELEASE_MANIFEST_FILE_PATTERN, - RELEASE_OVERLAY_FILE, - RELEASE_STATE_DIRECTORY, - RELEASE_STORE_LIMITS, - RELEASE_STORE_POINTER_FILES, - type ReleasePointer, - type ReleaseSelectionReceipts, - type ReleaseStateOptions, - type ReleaseStatePaths, - readReleaseOverlay, - readReleaseOverlayFile, - readReleaseStateBytes, - readReleaseStateFile, - releaseStateFileProblem, - releaseStatePaths, -} from "./release-state"; -export { - acquireReleaseStoreLock, - assertReleaseStoreLock, - RELEASE_STORE_LOCK_NAME, - type ReleaseStoreLock, - type ReleaseStoreLockHooks, -} from "./release-store-lock"; export { detectConsumesCycle, findByCapability, @@ -366,6 +337,7 @@ export { parseServiceIdList, SERVICE_SELECTION_ENV, } from "./selection"; +export { selfUpdateServiceOrder } from "./self-update-order"; export { describeServiceFiles, type ServiceFileReportLine } from "./service-file-report"; export { serviceFileStore } from "./service-file-store"; export * from "./service-lifecycle-receipt"; diff --git a/packages/core/src/services/release-cache.ts b/packages/core/src/services/release-cache.ts new file mode 100644 index 000000000..6fb955a7e --- /dev/null +++ b/packages/core/src/services/release-cache.ts @@ -0,0 +1,244 @@ +import { randomUUID } from "node:crypto"; +import { + closeSync, + constants, + fstatSync, + fsyncSync, + linkSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + readSync, + unlinkSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { fsyncDirectory } from "../release-files"; +import { + canonicalReleaseManifest, + parseReleaseManifest, + type ReleaseChannel, + ReleaseChannelDisabledError, + type ReleaseManifest, + releaseManifestDigest, + releaseServiceImages, +} from "./release-manifest"; +import type { ReleaseSelection } from "./release-selection"; + +/** + * The operations agent's cache of release manifests it has checked, named by + * their digest: `infra/docker/state/ops-agent/releases/.json`. An + * update names the checked release it applies by that digest; the release a + * generation runs is kept in the generation itself, not here. The newest file + * is the release last checked. Only the agent's user may own the directory + * and its files; anything else is refused, never repaired. + */ +const RELEASE_CACHE_DIRECTORY = "state/ops-agent/releases"; +const RELEASE_CACHE_ENTRIES = 64; +const RELEASE_MANIFEST_BYTES = 32 * 1024; +const CACHED_MANIFEST = /^([a-f0-9]{64})\.json$/; +const RELEASE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +export interface ReleaseCacheOptions { + /** The uid the cache must belong to: the effective uid unless a test injects one. */ + uid?: number; +} + +export function releaseCacheDirectory(infraDir: string): string { + return join(infraDir, RELEASE_CACHE_DIRECTORY); +} + +function owner(options: ReleaseCacheOptions): number | undefined { + return options.uid ?? process.geteuid?.(); +} + +/** Creates the directory owner-only when missing; an existing one is checked, never changed. */ +function cacheDirectory(infraDir: string, options: ReleaseCacheOptions, create: boolean): string { + const path = releaseCacheDirectory(infraDir); + if (create) mkdirSync(path, { recursive: true, mode: 0o700 }); + const stats = lstatSync(path); + const uid = owner(options); + if ( + !stats.isDirectory() || + stats.isSymbolicLink() || + (uid !== undefined && stats.uid !== uid) || + (stats.mode & 0o077) !== 0 + ) + throw new Error( + `Unsafe release cache: ${path} must be a directory owned by uid ${uid} that its group and others cannot reach`, + ); + return path; +} + +/** One cached file, checked on its opened descriptor so it cannot be swapped meanwhile. */ +function readCachedFile(path: string, options: ReleaseCacheOptions): string { + const descriptor = openSync( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + try { + const opened = fstatSync(descriptor); + const uid = owner(options); + if ( + !opened.isFile() || + opened.nlink !== 1 || + (uid !== undefined && opened.uid !== uid) || + (opened.mode & 0o022) !== 0 || + opened.size < 1 || + opened.size > RELEASE_MANIFEST_BYTES + ) + throw new Error(`${path} is not a release manifest of this cache`); + const buffer = Buffer.alloc(opened.size); + let offset = 0; + while (offset < buffer.length) { + const count = readSync(descriptor, buffer, offset, buffer.length - offset, offset); + if (count === 0) throw new Error(`${path} changed while it was read`); + offset += count; + } + return new TextDecoder("utf-8", { fatal: true }).decode(buffer); + } finally { + closeSync(descriptor); + } +} + +function cachedEntries(directory: string): { digest: string; mtimeMs: number }[] { + const entries: { digest: string; mtimeMs: number }[] = []; + for (const name of readdirSync(directory)) { + const digest = CACHED_MANIFEST.exec(name)?.[1]; + if (!digest) continue; + const stats = lstatSync(join(directory, name)); + if (stats.isFile()) entries.push({ digest, mtimeMs: stats.mtimeMs }); + } + return entries.sort((a, b) => b.mtimeMs - a.mtimeMs || a.digest.localeCompare(b.digest)); +} + +/** + * Caches a checked manifest under its digest and marks it the newest. An + * existing entry must hold the same bytes. The oldest entries beyond the + * cache's bound are removed. + */ +export function cacheReleaseManifest( + infraDir: string, + manifest: ReleaseManifest, + options: ReleaseCacheOptions = {}, +): { releaseId: string; manifestDigest: string } { + const directory = cacheDirectory(infraDir, options, true); + const canonical = canonicalReleaseManifest(manifest); + if (Buffer.byteLength(canonical) > RELEASE_MANIFEST_BYTES) + throw new Error("Release manifest limit exceeded"); + const manifestDigest = releaseManifestDigest(manifest); + const path = join(directory, `${manifestDigest}.json`); + const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`; + const descriptor = openSync( + temporary, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, + 0o600, + ); + try { + writeFileSync(descriptor, canonical, "utf8"); + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } + try { + linkSync(temporary, path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + if (readCachedFile(path, options) !== canonical) + throw new Error(`${path} holds another manifest`); + const now = new Date(); + utimesSync(path, now, now); + } finally { + unlinkSync(temporary); + } + fsyncDirectory(directory); + for (const stale of cachedEntries(directory).slice(RELEASE_CACHE_ENTRIES)) + unlinkSync(join(directory, `${stale.digest}.json`)); + return { releaseId: manifest.release, manifestDigest }; +} + +/** The cached manifest with this digest, checked against the configured image repository. */ +export function readCachedReleaseManifest( + infraDir: string, + manifestDigest: string, + imagePrefix: string, + options: ReleaseCacheOptions = {}, +): ReleaseManifest { + if (!/^[a-f0-9]{64}$/.test(manifestDigest)) throw new Error("Invalid release manifest digest"); + const directory = cacheDirectory(infraDir, options, false); + const raw = readCachedFile(join(directory, `${manifestDigest}.json`), options); + const manifest = parseReleaseManifest(raw, imagePrefix); + if ( + canonicalReleaseManifest(manifest) !== raw || + releaseManifestDigest(manifest) !== manifestDigest + ) + throw new Error("Cached release manifest changed"); + return manifest; +} + +/** The cached releases of this image repository, the one checked last first. */ +export function cachedReleases( + infraDir: string, + imagePrefix: string, + options: ReleaseCacheOptions = {}, +): { manifest: ReleaseManifest; manifestDigest: string }[] { + let directory: string; + try { + directory = cacheDirectory(infraDir, options, false); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + const releases: { manifest: ReleaseManifest; manifestDigest: string }[] = []; + for (const entry of cachedEntries(directory)) { + try { + releases.push({ + manifest: readCachedReleaseManifest(infraDir, entry.digest, imagePrefix, options), + manifestDigest: entry.digest, + }); + } catch { + // A manifest of another configured repository is not this channel's release. + } + } + return releases; +} + +function immutable(value: T): T { + if (value && typeof value === "object") { + for (const child of Object.values(value)) immutable(child); + Object.freeze(value); + } + return value; +} + +/** + * A checked release an update names by id and digest. Reading it grants no + * effect; the effect revalidates the same identity under its own locks. + */ +export function readCheckedConfigurationRelease( + infraDir: string, + input: { releaseId: string; manifestDigest: string }, + channel: ReleaseSelection | ReleaseChannel, + options: ReleaseCacheOptions = {}, +) { + if (channel.kind === "local" || channel.kind === "disabled") + throw new ReleaseChannelDisabledError(); + if (!RELEASE_ID.test(input.releaseId)) throw new Error("Invalid release id"); + const manifest = readCachedReleaseManifest( + infraDir, + input.manifestDigest, + channel.imagePrefix, + options, + ); + if (manifest.release !== input.releaseId) + throw new Error("Checked configuration release changed"); + return immutable({ + releaseId: input.releaseId, + manifestDigest: input.manifestDigest, + manifest, + serviceImages: { ...releaseServiceImages(manifest) }, + }); +} +export type CheckedConfigurationRelease = ReturnType; diff --git a/packages/core/src/services/release-image-selection.ts b/packages/core/src/services/release-image-selection.ts index 52f7a6ec8..3fa4ba709 100644 --- a/packages/core/src/services/release-image-selection.ts +++ b/packages/core/src/services/release-image-selection.ts @@ -1,23 +1,13 @@ -/** Read-only eligibility shared by CLI preparation and the fixed normal pull. - * This module never resolves, writes or repairs an overlay. */ -import { createHash } from "node:crypto"; -import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync } from "node:fs"; +/** Read-only: the image the applied generation pins for a service. */ import { serviceImageReferenceSchema } from "@openmapx/extension-contracts/service"; import { JSON_SCHEMA, load } from "js-yaml"; -import { canonicalJson } from "../ops/canonical-json"; +import { readAppliedRelease } from "./configuration-generation"; import { - parseReleaseComposeSelection, - RELEASE_IMAGE_NAMES, RELEASE_PINNED_SERVICE_IDS, - type ReleaseChannel, - type ReleaseComposeSelection, releaseChannel, + releaseServiceImages, } from "./release-manifest"; -import { - assertReleaseStateDirectory, - releaseStateFileProblem, - releaseStatePaths, -} from "./release-state"; + export function configurationImageReference(composeYaml: string, serviceId: string) { const document = load(composeYaml, { schema: JSON_SCHEMA }) as { services?: Record; @@ -25,83 +15,25 @@ export function configurationImageReference(composeYaml: string, serviceId: stri return serviceImageReferenceSchema.parse(document?.services?.[serviceId]?.image); } -export function validateExistingReleaseSelection( - raw: string, - channel: ReleaseChannel = releaseChannel(), -): ReleaseComposeSelection { - const selected = parseReleaseComposeSelection(raw), - prefix = - channel.kind === "enabled" ? channel.imagePrefix : selected.images.api?.split("/api@")[0]; - if ( - !selected.release || - !prefix || - !RELEASE_IMAGE_NAMES.every( - (name) => - selected.images[name]?.startsWith(`${prefix}/${name}@`) && - serviceImageReferenceSchema.safeParse(selected.images[name]).success, - ) - ) - throw Error( - "Existing release overlay is malformed or belongs to another configured repository; explicitly reselect or clear it", - ); - return selected; -} - +/** + * The pinned image of a release service in the applied generation, and the + * digest of the release manifest that pins it (`releaseSelectionDigest`). A + * release service without a pin is refused unless release resolution is + * disabled; other services have neither. + */ export function readServiceReleaseImageSelection(infraDir: string, serviceId: string) { - const managed = (RELEASE_PINNED_SERVICE_IDS as readonly string[]).includes(serviceId), - channel = releaseChannel(), - path = releaseStatePaths(infraDir).overlay; - let fd: number | undefined; - try { - fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); - assertReleaseStateDirectory(releaseStatePaths(infraDir).directory); - const before = fstatSync(fd); - if (releaseStateFileProblem(before, 128 * 1024) !== null) - throw Error("Release selection unavailable"); - const raw = readFileSync(fd), - after = fstatSync(fd), - current = lstatSync(path); - if ( - before.dev !== current.dev || - before.ino !== current.ino || - before.size !== after.size || - before.mtimeMs !== after.mtimeMs || - before.ctimeMs !== after.ctimeMs || - current.size !== after.size || - current.mtimeMs !== after.mtimeMs || - current.ctimeMs !== after.ctimeMs - ) - throw Error("Release selection changed"); - const selected = validateExistingReleaseSelection( - new TextDecoder("utf-8", { fatal: true }).decode(raw), - channel, - ); - return { - imageReference: managed - ? selected.images[ - serviceId === "app-api" - ? "api" - : serviceId === "app-web" - ? "web" - : (serviceId as keyof typeof selected.images) - ]! - : null, - releaseSelectionDigest: managed - ? createHash("sha256") - .update("openmapx-service-release-selection-v1\0") - .update(canonicalJson(selected)) - .digest("hex") - : null, - }; - } catch (error) { - if ( - fd !== undefined || - (error as NodeJS.ErrnoException).code !== "ENOENT" || - (managed && channel.kind !== "disabled") - ) - throw error; + if (!(RELEASE_PINNED_SERVICE_IDS as readonly string[]).includes(serviceId)) return { imageReference: null, releaseSelectionDigest: null }; - } finally { - if (fd !== undefined) closeSync(fd); + const release = readAppliedRelease(infraDir); + if (!release) { + if (releaseChannel().kind === "disabled") + return { imageReference: null, releaseSelectionDigest: null }; + throw new Error("The applied generation pins no release; select one with compose release"); } + return { + imageReference: releaseServiceImages(release.manifest)[ + serviceId as (typeof RELEASE_PINNED_SERVICE_IDS)[number] + ], + releaseSelectionDigest: release.manifestDigest, + }; } diff --git a/packages/core/src/services/release-manifest.ts b/packages/core/src/services/release-manifest.ts index 49ff0a10b..50141d6d6 100644 --- a/packages/core/src/services/release-manifest.ts +++ b/packages/core/src/services/release-manifest.ts @@ -20,31 +20,15 @@ export { /** * The aggregate OpenMapX release manifest published by * `.github/workflows/docker.yml` as `//release-manifest:main`. - * Parsing and overlay rendering are shared by the admin system updater - * (apps/api) and the CLI so both produce byte-identical - * `docker-compose.release.yml` overlays. + * A configuration generation renders its image pins into its compose file and + * keeps the manifest beside it, so the generation names its release. */ -import { createHash, randomBytes } from "node:crypto"; -import { - closeSync, - constants, - fsyncSync, - openSync, - renameSync, - rmSync, - writeFileSync, -} from "node:fs"; -import { JSON_SCHEMA, load } from "js-yaml"; +import { createHash } from "node:crypto"; import z from "zod/v4"; -import { - ensureReleaseStateDirectory, - type ReleaseStateOptions, - releaseStatePaths, -} from "./release-state"; export const RELEASE_MANIFEST_CONTAINER_PATH = "/release-manifest.json"; -/** Runtime services whose images the release overlay pins as one release. */ +/** Runtime services whose images a generation pins as one release. */ export const RELEASE_PINNED_SERVICE_IDS = [ "app-api", "app-web", @@ -161,13 +145,23 @@ export interface ProductReleaseMetadata { upgradeOrigins: string[]; limitations: string[]; } +/** The newest database migration the release's API ships, from its migration journal. */ +export interface ReleaseDatabaseMigrationHead { + tag: string; + /** The journal entry's `when`, which the migrator records as `created_at`. */ + when: number; +} export interface ReleaseManifest { schemaVersion: 1; release: string; images: Record<(typeof RELEASE_IMAGE_NAMES)[number], string>; + /** Display metadata: who built the release. Nothing is authorised from it. */ producer: ReleaseProducer; + /** Display metadata: how each image was built. Nothing is authorised from it. */ buildMetadata: ReleaseBuildMetadata; product?: ProductReleaseMetadata; + /** Absent in manifests published before CI recorded it; rollback then refuses. */ + databaseMigrationHead?: ReleaseDatabaseMigrationHead; } const producerSchema = z.strictObject({ @@ -203,6 +197,10 @@ const productSchema = z.strictObject({ upgradeOrigins: z.array(z.string().min(1)).min(1), limitations: z.array(z.string().min(1)), }); +const databaseMigrationHeadSchema = z.strictObject({ + tag: z.string().regex(/^\d{4}_[a-z0-9_]{1,120}$/), + when: z.number().int().positive().safe(), +}); function stableObject(value: unknown): unknown { if (Array.isArray(value)) return value.map(stableObject); if (value && typeof value === "object") @@ -243,6 +241,7 @@ export function parseReleaseManifest( producer?: unknown; buildMetadata?: unknown; product?: unknown; + databaseMigrationHead?: unknown; }; if (candidate.schemaVersion !== 1) throw new Error("Unsupported release lockfile schemaVersion"); if (typeof candidate.release !== "string" || !RELEASE_ID_PATTERN.test(candidate.release)) { @@ -270,6 +269,11 @@ export function parseReleaseManifest( } if (candidate.product !== undefined && !productSchema.safeParse(candidate.product).success) throw new Error("Release lockfile product is invalid"); + if ( + candidate.databaseMigrationHead !== undefined && + !databaseMigrationHeadSchema.safeParse(candidate.databaseMigrationHead).success + ) + throw new Error("Release lockfile databaseMigrationHead is invalid"); const product = candidate.product as ProductReleaseMetadata | undefined; if ( product && @@ -285,7 +289,7 @@ export function parseReleaseManifest( } as ReleaseManifest; } -/** Stable bytes used by the ops-agent release store and transaction digest. */ +/** Stable bytes: what the manifest cache and a generation store, and what its digest covers. */ export function canonicalReleaseManifest(manifest: ReleaseManifest): string { return JSON.stringify({ schemaVersion: 1, @@ -301,153 +305,31 @@ export function canonicalReleaseManifest(manifest: ReleaseManifest): string { producer: stableObject(manifest.producer), buildMetadata: stableObject(manifest.buildMetadata), ...(manifest.product ? { product: stableObject(manifest.product) } : {}), + ...(manifest.databaseMigrationHead + ? { + databaseMigrationHead: { + tag: manifest.databaseMigrationHead.tag, + when: manifest.databaseMigrationHead.when, + }, + } + : {}), }); } -/** What an overlay is rendered from: its release's identity and pins. */ -interface ReleaseOverlaySource { - /** The `x-openmapx-release` metadata, as the overlay's first line carries it. */ - metadata: string; - images: ReleaseManifest["images"]; -} - -function overlaySource(manifest: ReleaseManifest): ReleaseOverlaySource { - return { - metadata: JSON.stringify({ - release: manifest.release, - images: Object.fromEntries(RELEASE_IMAGE_NAMES.map((name) => [name, manifest.images[name]])), - ...(manifest.product ? { product: manifest.product } : {}), - digest: createHash("sha256").update(canonicalReleaseManifest(manifest)).digest("hex"), - }), - images: manifest.images, - }; -} - -function renderOverlay(source: ReleaseOverlaySource): string { - const { images } = source; - return [ - `x-openmapx-release: ${source.metadata}`, - "services:", - " app-api:", - ` image: ${images.api}`, - " app-web:", - ` image: ${images.web}`, - " data-manager:", - ` image: ${images["data-manager"]}`, - " environment:", - ` ${TRANSITOUS_TOOLS_IMAGE_ENV}: ${images["transitous-tools"]}`, - " ops-agent:", - ` image: ${images["ops-agent"]}`, - " transitous-runner:", - ` image: ${images["transitous-runner"]}`, - "", - ].join("\n"); -} - -/** The overlay of `manifest` as it is written in the release state. */ -export function renderReleaseCompose(manifest: ReleaseManifest): string { - return renderOverlay(overlaySource(manifest)); -} - -function fsyncDirectory(path: string): void { - const descriptor = openSync(path, constants.O_RDONLY | constants.O_DIRECTORY); - try { - fsyncSync(descriptor); - } finally { - closeSync(descriptor); - } +/** The SHA-256 of the canonical manifest: the release's identity in generations and the cache. */ +export function releaseManifestDigest(manifest: ReleaseManifest): string { + return createHash("sha256").update(canonicalReleaseManifest(manifest)).digest("hex"); } -/** Atomically switch the Compose overlay in the release state of `infraDir`. */ -export function writeReleaseComposeArtifacts( +/** The image each pinned service runs in `manifest`. */ +export function releaseServiceImages( manifest: ReleaseManifest, - infraDir: string, - options: ReleaseStateOptions = {}, -): { overlayPath: string } { - const { directory, overlay: overlayPath } = releaseStatePaths(infraDir); - ensureReleaseStateDirectory(directory, options); - fsyncDirectory(directory); - const temporaryOverlay = `${overlayPath}.${randomBytes(12).toString("hex")}.partial`; - let overlayDescriptor: number | undefined; - try { - overlayDescriptor = openSync( - temporaryOverlay, - constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, - 0o600, - ); - writeFileSync(overlayDescriptor, renderReleaseCompose(manifest), { encoding: "utf8" }); - fsyncSync(overlayDescriptor); - closeSync(overlayDescriptor); - overlayDescriptor = undefined; - renameSync(temporaryOverlay, overlayPath); - fsyncDirectory(directory); - } finally { - if (overlayDescriptor !== undefined) closeSync(overlayDescriptor); - rmSync(temporaryOverlay, { force: true }); - } - return { overlayPath }; -} - -/** - * Read the Transitous helper image pinned by an existing release overlay. The - * overlay is our own rendered output, so a targeted line match is sufficient - * and avoids a YAML dependency in callers. - */ -export function transitousToolsImageFromReleaseCompose(overlayYaml: string): string | null { - const match = new RegExp(`^\\s*${TRANSITOUS_TOOLS_IMAGE_ENV}:\\s*(\\S+)\\s*$`, "m").exec( - overlayYaml, - ); - const image = match?.[1]; - return image && new RegExp(`^[a-z0-9][a-z0-9._/-]*/transitous-tools@${DIGEST}$`).test(image) - ? image - : null; -} - -export interface ReleaseComposeSelection { - release: string | null; - images: Partial>; -} - -/** Read local selection, never infer running container versions from the overlay. */ -export function parseReleaseComposeSelection(raw: string): ReleaseComposeSelection { - const object = (value: unknown): Record => - value !== null && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : {}; - const document = object(load(raw, { schema: JSON_SCHEMA })); - if ( - !document.services || - typeof document.services !== "object" || - Array.isArray(document.services) - ) { - throw new Error("Release overlay must contain a services mapping"); - } - const services = object(document.services); - const images: ReleaseComposeSelection["images"] = {}; - const pins: Record = { - api: object(services["app-api"]).image, - web: object(services["app-web"]).image, - "data-manager": object(services["data-manager"]).image, - "ops-agent": object(services["ops-agent"]).image, - "transitous-runner": object(services["transitous-runner"]).image, - "transitous-tools": object(object(services["data-manager"]).environment)[ - TRANSITOUS_TOOLS_IMAGE_ENV - ], - }; - for (const name of RELEASE_IMAGE_NAMES) { - const pin = pins[name]; - if (typeof pin === "string" && new RegExp(`^\\S+@${DIGEST}$`).test(pin)) images[name] = pin; - } - const metadata = object(document["x-openmapx-release"]); - const metadataImages = object(metadata.images); - const matches = RELEASE_IMAGE_NAMES.every( - (name) => images[name] && metadataImages[name] === images[name], - ); +): Readonly> { return { - release: - matches && typeof metadata.release === "string" && metadata.release.trim() - ? metadata.release - : null, - images, + "app-api": manifest.images.api, + "app-web": manifest.images.web, + "data-manager": manifest.images["data-manager"], + "ops-agent": manifest.images["ops-agent"], + "transitous-runner": manifest.images["transitous-runner"], }; } diff --git a/packages/core/src/services/release-selection-clear.ts b/packages/core/src/services/release-selection-clear.ts deleted file mode 100644 index 033f53835..000000000 --- a/packages/core/src/services/release-selection-clear.ts +++ /dev/null @@ -1,111 +0,0 @@ -/** Selection metadata only. Does not modify images, running state or evidence. */ -import { createHash } from "node:crypto"; -import { - closeSync, - constants, - fstatSync, - fsyncSync, - lstatSync, - openSync, - readSync, - type Stats, - unlinkSync, -} from "node:fs"; -import { dirname, resolve } from "node:path"; -import { parseReleaseComposeSelection } from "./release-manifest"; -import { - ensureReleaseStateDirectories, - releaseStateFileProblem, - releaseStatePaths, -} from "./release-state"; -import { acquireReleaseStoreLock } from "./release-store-lock"; -export async function clearReleaseComposeSelection(opts: { - path: string; - expectedReleaseId?: string; - expectedOverlayDigest?: string | null; - infraDir?: string; - uid?: number; -}): Promise<{ path: string; cleared: boolean }> { - if (opts.expectedOverlayDigest != null && !/^[a-f0-9]{64}$/.test(opts.expectedOverlayDigest)) - throw Error("Invalid release selection digest"); - const path = resolve(opts.path); - const parent = dirname(path); - const infraDir = opts.infraDir ?? resolve(parent, "../../.."); - const stateOptions = { uid: opts.uid }; - const paths = releaseStatePaths(infraDir); - if (resolve(paths.overlay) !== path) throw Error("Release selection path is unsafe"); - ensureReleaseStateDirectories(infraDir, stateOptions); - const store = paths.store; - const lock = await acquireReleaseStoreLock(store, {}, { failIfBusy: true }); - try { - let selected: Stats; - try { - selected = lstatSync(path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - if (opts.expectedReleaseId !== undefined || typeof opts.expectedOverlayDigest === "string") - throw Error("Release selection changed"); - return { path, cleared: false }; - } - if (opts.expectedOverlayDigest === null) throw Error("Release selection changed"); - if ( - releaseStateFileProblem(selected, 128 * 1024, stateOptions) !== null || - !selected.isFile() || - selected.isSymbolicLink() || - selected.nlink !== 1 || - (opts.expectedReleaseId !== undefined && selected.size > 64 * 1024) - ) - throw Error("Release selection overlay is unsafe"); - const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const opened = fstatSync(fd); - if ( - opened.dev !== selected.dev || - opened.ino !== selected.ino || - opened.size !== selected.size - ) - throw Error("Release selection changed"); - if (opts.expectedReleaseId !== undefined || opts.expectedOverlayDigest !== undefined) { - const bytes = Buffer.alloc(opened.size); - let offset = 0; - while (offset < bytes.length) { - const count = readSync(fd, bytes, offset, bytes.length - offset, offset); - if (!count) throw Error("Release selection changed"); - offset += count; - } - if ( - (opts.expectedReleaseId !== undefined && - parseReleaseComposeSelection(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) - .release !== opts.expectedReleaseId) || - (opts.expectedOverlayDigest !== undefined && - createHash("sha256").update(bytes).digest("hex") !== opts.expectedOverlayDigest) - ) - throw Error("Release selection changed"); - } - const after = lstatSync(path); - if ( - after.dev !== opened.dev || - after.ino !== opened.ino || - after.size !== opened.size || - after.mtimeMs !== opened.mtimeMs || - after.nlink !== 1 - ) - throw Error("Release selection changed"); - unlinkSync(path); - } finally { - closeSync(fd); - } - const descriptor = openSync( - parent, - constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, - ); - try { - fsyncSync(descriptor); - } finally { - closeSync(descriptor); - } - return { path, cleared: true }; - } finally { - lock.release(); - } -} diff --git a/packages/core/src/services/release-state.ts b/packages/core/src/services/release-state.ts deleted file mode 100644 index 2df5b5ce1..000000000 --- a/packages/core/src/services/release-state.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readSync } from "node:fs"; -import { dirname, join } from "node:path"; - -/** - * The operations agent's releases, in its own state directory: - * `infra/docker/state/ops-agent/releases`. It holds - * - * - `store/`, the release manifests the agent resolved and its pointers to - * them (`current.json`, `latest.json`, `selections.json`), with the store - * lock; - * - `docker-compose.release.yml`, the overlay pinning the selected release. - * - * Only the agent's user can reach it, and the host CLI runs as that user. So - * every directory and file here must be that user's own and unwritable by - * anyone else; anything else is refused, never repaired. - */ -export const RELEASE_STATE_DIRECTORY = "state/ops-agent/releases"; -export const RELEASE_OVERLAY_FILE = "docker-compose.release.yml"; - -/** The pointer files beside the manifests in the release store. */ -export const RELEASE_STORE_POINTER_FILES = [ - "current.json", - "latest.json", - "selections.json", -] as const; -export const RELEASE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; -/** A stored manifest is named after its release: `.json`. */ -export const RELEASE_MANIFEST_FILE_PATTERN = /^([A-Za-z0-9][A-Za-z0-9._-]{0,127})\.json$/; -export const RELEASE_STORE_LIMITS = { - entries: 64, - manifestBytes: 32 * 1024, - stateBytes: 4 * 1024, - selectionsBytes: 32 * 1024, -} as const; - -export interface ReleaseStatePaths { - directory: string; - store: string; - overlay: string; -} - -export function releaseStatePaths(infraDir: string): ReleaseStatePaths { - const directory = join(infraDir, RELEASE_STATE_DIRECTORY); - return { - directory, - store: join(directory, "store"), - overlay: join(directory, RELEASE_OVERLAY_FILE), - }; -} - -export interface ReleaseStateOptions { - /** The uid the release state must belong to: the effective uid unless a test injects one. */ - uid?: number; -} - -function stateOwner(options: ReleaseStateOptions): number | undefined { - return options.uid ?? process.geteuid?.(); -} - -function mode(stats: { mode: number }): string { - return `0${(stats.mode & 0o777).toString(8)}`; -} - -/** - * Why `stats` cannot be one of the agent's release-state files, or null: a - * regular file of this user with one link and no group or other write, of at - * most `maximum` bytes. - */ -export function releaseStateFileProblem( - stats: { - isFile(): boolean; - isSymbolicLink(): boolean; - nlink: number; - uid: number; - mode: number; - size: number; - }, - maximum: number, - options: ReleaseStateOptions = {}, -): string | null { - const owner = stateOwner(options); - if (!stats.isFile() || stats.isSymbolicLink()) return "is not a regular file"; - if (stats.nlink !== 1) return "has more than one link"; - if (owner !== undefined && stats.uid !== owner) - return `is owned by uid ${stats.uid}, not ${owner}`; - if ((stats.mode & 0o022) !== 0) return `is writable by its group or others (mode ${mode(stats)})`; - if (stats.size > maximum) return `is larger than ${maximum} bytes`; - return null; -} - -/** A real directory of this user that its group and others cannot reach. */ -export function assertReleaseStateDirectory(path: string, options: ReleaseStateOptions = {}): void { - const stats = lstatSync(path); - const owner = stateOwner(options); - if ( - !stats.isDirectory() || - stats.isSymbolicLink() || - (owner !== undefined && stats.uid !== owner) || - (stats.mode & 0o077) !== 0 - ) { - throw new Error( - `Unsafe release state directory: ${path} must be a directory owned by uid ${owner} that its group and others cannot reach`, - ); - } -} - -/** Creates a missing directory owner-only; an existing one is checked, never changed. */ -export function ensureReleaseStateDirectory(path: string, options: ReleaseStateOptions = {}): void { - try { - lstatSync(path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - try { - mkdirSync(path, { recursive: true, mode: 0o700 }); - } catch (created) { - // Another writer created it first; it is checked like any existing one. - if ((created as NodeJS.ErrnoException).code !== "EEXIST") throw created; - } - } - assertReleaseStateDirectory(path, options); -} - -/** `releases/` and `store/`, created owner-only when missing. */ -export function ensureReleaseStateDirectories( - infraDir: string, - options: ReleaseStateOptions = {}, -): ReleaseStatePaths { - const paths = releaseStatePaths(infraDir); - ensureReleaseStateDirectory(paths.directory, options); - ensureReleaseStateDirectory(paths.store, options); - return paths; -} - -/** - * Reads one of the agent's release-state files. Every check is made on the - * opened descriptor, so the file cannot be swapped between check and read. A - * missing file keeps its ENOENT code; a symbolic link fails to open. - */ -export function readReleaseStateFile( - path: string, - maximum: number, - options: ReleaseStateOptions = {}, -): string { - return new TextDecoder("utf-8", { fatal: true }).decode( - readReleaseStateBytes(path, maximum, options), - ); -} - -/** Bounded owned bytes, including malformed UTF-8 during conditional recovery. */ -export function readReleaseStateBytes( - path: string, - maximum: number, - options: ReleaseStateOptions = {}, -): Buffer { - let descriptor: number; - try { - descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") throw error; - throw new Error(`${path} must be a regular file, not a symbolic link`); - } - try { - const opened = fstatSync(descriptor); - const problem = releaseStateFileProblem(opened, maximum, options); - if (problem) throw new Error(`${path} ${problem}`); - const buffer = Buffer.alloc(opened.size); - let offset = 0; - while (offset < buffer.length) { - const count = readSync(descriptor, buffer, offset, buffer.length - offset, offset); - if (count === 0) throw new Error(`${path} changed while it was read`); - offset += count; - } - const after = fstatSync(descriptor); - if (after.dev !== opened.dev || after.ino !== opened.ino || after.size !== opened.size) { - throw new Error(`${path} changed while it was read`); - } - return buffer; - } finally { - closeSync(descriptor); - } -} - -/** - * The release overlay at `path`, or null when there is none. Its directory and - * the file itself must be the agent user's own; anything else is refused. - */ -export function readReleaseOverlayFile( - path: string, - options: ReleaseStateOptions = {}, -): string | null { - try { - lstatSync(path); - assertReleaseStateDirectory(dirname(path), options); - return readReleaseStateFile(path, RELEASE_STORE_LIMITS.manifestBytes, options); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; - throw error; - } -} - -export function readReleaseOverlay( - infraDir: string, - options: ReleaseStateOptions = {}, -): string | null { - return readReleaseOverlayFile(releaseStatePaths(infraDir).overlay, options); -} - -export interface ReleasePointer { - releaseId: string; - digest?: string; - updateJobId?: string; -} - -/** `current.json` or `latest.json`; `latest.json` must name its digest. */ -export function parseReleasePointer(raw: string, requireDigest: boolean): ReleasePointer { - const candidate = JSON.parse(raw) as unknown; - if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) { - throw new Error("Release authority rejected"); - } - const value = candidate as Record; - if ( - typeof value.releaseId !== "string" || - !RELEASE_ID_PATTERN.test(value.releaseId) || - (value.digest !== undefined && - (typeof value.digest !== "string" || !/^[a-f0-9]{64}$/.test(value.digest))) || - (requireDigest && value.digest === undefined) || - (value.updateJobId !== undefined && - (typeof value.updateJobId !== "string" || !RELEASE_ID_PATTERN.test(value.updateJobId))) - ) { - throw new Error("Release authority rejected"); - } - return { - releaseId: value.releaseId, - ...(typeof value.digest === "string" ? { digest: value.digest } : {}), - ...(typeof value.updateJobId === "string" ? { updateJobId: value.updateJobId } : {}), - }; -} - -export type ReleaseSelectionReceipts = Record< - string, - { configuration: string; ociDigest: string; digest: string } ->; - -/** `selections.json`: which configuration checked each stored release, at which digests. */ -export function parseReleaseSelectionReceipts(raw: string): ReleaseSelectionReceipts { - const value = JSON.parse(raw); - if ( - !value || - typeof value !== "object" || - Array.isArray(value) || - Object.keys(value).length > RELEASE_STORE_LIMITS.entries - ) - throw new Error("Invalid release selection receipts"); - for (const [id, receipt] of Object.entries(value) as [string, Record][]) - if ( - !RELEASE_ID_PATTERN.test(id) || - !receipt || - Object.keys(receipt).sort().join() !== "configuration,digest,ociDigest" || - !/^[a-f0-9]{64}$/.test(String(receipt.configuration)) || - !/^[a-f0-9]{64}$/.test(String(receipt.digest)) || - !/^sha256:[a-f0-9]{64}$/.test(String(receipt.ociDigest)) - ) - throw new Error("Invalid release selection receipt"); - return value; -} diff --git a/packages/core/src/services/release-store-lock.ts b/packages/core/src/services/release-store-lock.ts deleted file mode 100644 index 9661cc50a..000000000 --- a/packages/core/src/services/release-store-lock.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { randomUUID } from "node:crypto"; -import { - closeSync, - constants, - fstatSync, - lstatSync, - mkdirSync, - openSync, - readSync, - rmSync, -} from "node:fs"; -import { join, resolve } from "node:path"; -import { atomicWrite, durableUnlink, fsyncDirectory } from "../release-files"; - -export const RELEASE_STORE_LOCK_NAME = ".release-store.lock"; -const RELEASE_STORE_LOCK_TTL_MS = 15 * 60_000; -const RELEASE_STORE_LOCK_ACQUIRE_TIMEOUT_MS = 60_000; -const RELEASE_STORE_LOCK_RETRY_MS = 50; -const MAX_RELEASE_STATE_BYTES = 4 * 1024; - -function strictReadFile(path: string, maximum: number): string { - const before = lstatSync(path); - if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.size > maximum) { - throw new Error("Release authority rejected"); - } - const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const opened = fstatSync(descriptor); - if (opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) { - throw new Error("Release authority rejected"); - } - const buffer = Buffer.alloc(opened.size); - let offset = 0; - while (offset < buffer.length) { - const count = readSync(descriptor, buffer, offset, buffer.length - offset, offset); - if (count === 0) throw new Error("Release authority rejected"); - offset += count; - } - const after = fstatSync(descriptor); - if (after.dev !== opened.dev || after.ino !== opened.ino || after.size !== opened.size) { - throw new Error("Release authority rejected"); - } - return new TextDecoder("utf-8", { fatal: true }).decode(buffer); - } finally { - closeSync(descriptor); - } -} - -/** - * A no-replace, cross-process lock over the release store. - * - * `mkdir` is atomic and fails with `EEXIST` when the directory already exists, - * so it serializes ops-agent processes without a shared runtime. The owner - * record inside it carries a lease: a holder that died without releasing is - * reclaimed only after the lease expires, and reclamation itself races through - * the same `mkdir`, so two reclaimers cannot both win. - */ -export interface ReleaseStoreLock { - release(): void; -} -const physicalLockOwners = new WeakMap(); -const liveReleaseLocks = new WeakMap< - ReleaseStoreLock, - { directory: string; dev: number; ino: number; owner: string } ->(); - -/** Object identity and the original physical owner, never a caller boolean. */ -export function assertReleaseStoreLock(lock: ReleaseStoreLock, directory: string): void { - const held = liveReleaseLocks.get(lock); - if (!held || held.directory !== resolve(directory)) throw Error("Release store lock is not held"); - const path = join(directory, RELEASE_STORE_LOCK_NAME); - const stat = lstatSync(path); - if ( - !stat.isDirectory() || - stat.isSymbolicLink() || - stat.dev !== held.dev || - stat.ino !== held.ino || - strictReadFile(join(path, "owner.json"), MAX_RELEASE_STATE_BYTES) !== held.owner - ) - throw Error("Release store lock ownership changed"); -} - -export interface ReleaseStoreLockHooks { - afterLockDirectoryCreate?: () => void; - beforeOwnerRecordWrite?: () => void; -} - -export async function acquireStoreLock( - directory: string, - lockName: string, - hooks: ReleaseStoreLockHooks = {}, - nowMs: () => number = Date.now, - failIfBusy = false, - leaseMs = RELEASE_STORE_LOCK_TTL_MS, -): Promise { - if ( - !Number.isSafeInteger(leaseMs) || - leaseMs < RELEASE_STORE_LOCK_TTL_MS || - leaseMs > 24 * 60 * 60 * 1000 - ) - throw Error("Invalid store lock lease"); - const lockPath = join(directory, lockName); - const ownerPath = join(lockPath, "owner.json"); - const deadline = nowMs() + RELEASE_STORE_LOCK_ACQUIRE_TIMEOUT_MS; - const owner = { pid: process.pid, nonce: randomUUID() }; - while (true) { - try { - mkdirSync(lockPath, { mode: 0o700 }); - hooks.afterLockDirectoryCreate?.(); - hooks.beforeOwnerRecordWrite?.(); - const ownerRecord = JSON.stringify({ ...owner, acquiredAtMs: nowMs(), leaseMs }); - atomicWrite(ownerPath, ownerRecord); - fsyncDirectory(lockPath); - const lock: ReleaseStoreLock = { - release() { - durableUnlink(ownerPath); - try { - rmSync(lockPath, { recursive: true, force: true }); - fsyncDirectory(directory); - } catch { - // A already-removed lock directory is not an error: the caller's - // critical section is over either way. - } - }, - }; - physicalLockOwners.set(lock, ownerRecord); - return lock; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - } - if (failIfBusy) throw new Error("Release store is busy"); - // Held by someone else. Reclaim only an expired lease, and only by - // removing the exact directory we observed as expired. - let expired = false; - try { - const raw = strictReadFile(ownerPath, MAX_RELEASE_STATE_BYTES); - const record = JSON.parse(raw) as { acquiredAtMs?: unknown; leaseMs?: unknown }; - const retainedLease = record.leaseMs ?? leaseMs; - if ( - typeof retainedLease !== "number" || - !Number.isSafeInteger(retainedLease) || - retainedLease < RELEASE_STORE_LOCK_TTL_MS || - retainedLease > 24 * 60 * 60 * 1000 - ) - throw Error("Store lock lease is unreadable"); - expired = - typeof record.acquiredAtMs !== "number" || - !Number.isFinite(record.acquiredAtMs) || - nowMs() - record.acquiredAtMs > retainedLease; - } catch (error) { - // A lock directory without a readable owner record is either mid- - // acquisition or abandoned before its record landed. Treat it as - // expired only once it is older than the lease. - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - let createdAtMs = Number.POSITIVE_INFINITY; - try { - createdAtMs = lstatSync(lockPath).mtimeMs; - } catch { - continue; - } - expired = nowMs() - createdAtMs > RELEASE_STORE_LOCK_TTL_MS; - } else { - throw new Error("Release store lock is unreadable"); - } - } - if (expired) { - try { - rmSync(lockPath, { recursive: true, force: true }); - fsyncDirectory(directory); - } catch { - // Another process reclaimed it first; retry through mkdir. - } - continue; - } - if (nowMs() >= deadline) throw new Error("Release store is busy"); - await new Promise((resolve) => setTimeout(resolve, RELEASE_STORE_LOCK_RETRY_MS)); - } -} - -export async function acquireReleaseStoreLock( - directory: string, - hooks: ReleaseStoreLockHooks = {}, - options: { failIfBusy?: boolean } = {}, -): Promise { - const physical = await acquireStoreLock( - directory, - RELEASE_STORE_LOCK_NAME, - hooks, - Date.now, - options.failIfBusy, - ); - try { - const path = join(directory, RELEASE_STORE_LOCK_NAME); - const stat = lstatSync(path); - const owner = strictReadFile(join(path, "owner.json"), MAX_RELEASE_STATE_BYTES); - if (owner !== physicalLockOwners.get(physical)) - throw Error("Release store lock ownership changed"); - const lock: ReleaseStoreLock = { - release() { - assertReleaseStoreLock(lock, directory); - try { - physical.release(); - } finally { - liveReleaseLocks.delete(lock); - } - }, - }; - liveReleaseLocks.set(lock, { - directory: resolve(directory), - dev: stat.dev, - ino: stat.ino, - owner, - }); - return lock; - } catch (error) { - // Never clean up a replacement owner. If observation remains unavailable, - // leave the guard fail-closed rather than guessing which process owns it. - let ours = false; - try { - ours = - strictReadFile( - join(directory, RELEASE_STORE_LOCK_NAME, "owner.json"), - MAX_RELEASE_STATE_BYTES, - ) === physicalLockOwners.get(physical); - } catch { - /* Ownership cannot be established. */ - } - if (ours) physical.release(); - throw error; - } -} diff --git a/packages/core/src/services/self-update-order.ts b/packages/core/src/services/self-update-order.ts new file mode 100644 index 000000000..d065af37d --- /dev/null +++ b/packages/core/src/services/self-update-order.ts @@ -0,0 +1,20 @@ +import { z } from "zod"; + +/** The services a self-update replaces last, in this order; the agent always comes last. */ +const PROTECTED_ORDER = ["postgis", "redis", "app-web", "app-api", "traefik", "ops-agent"] as const; +const serviceId = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/); + +/** + * The order a self-update recreates services in: every other service first, + * by id, then the protected ones, so the database comes before its clients + * and the operations agent is replaced last. Rollback uses the same order. + */ +export function selfUpdateServiceOrder(serviceIds: readonly string[]): string[] { + const values = z.array(serviceId).min(1).max(64).parse(serviceIds); + if (new Set(values).size !== values.length) throw Error("Duplicate frozen handoff target"); + const protectedIds = new Set(PROTECTED_ORDER); + return [ + ...values.filter((value) => !protectedIds.has(value)).sort(), + ...PROTECTED_ORDER.filter((value) => values.includes(value)), + ]; +} diff --git a/scripts/docker-release-contract.test.ts b/scripts/docker-release-contract.test.ts index 2bb9466ff..2f69afb0e 100644 --- a/scripts/docker-release-contract.test.ts +++ b/scripts/docker-release-contract.test.ts @@ -231,7 +231,7 @@ describe("Docker release trust gate", () => { const build = load(release).jobs.build.steps.find((step) => step.uses?.startsWith("docker/build-push-action@"), ); - const expression = (value: string) => "$" + `{{ ${value} }}`; + const expression = (value: string) => `\${{ ${value} }}`; const cacheRef = `type=registry,ref=${expression("env.IMAGE_PREFIX")}/${expression("matrix.app")}:buildcache`; expect(build?.with?.["cache-from"]).toBe(cacheRef); const cacheTo = String(build?.with?.["cache-to"]); @@ -321,6 +321,16 @@ describe("Docker release trust gate", () => { ]), ), }; + mkdirSync(join(temp, "source/apps/api/src/db/migrations/meta"), { recursive: true }); + writeFileSync( + join(temp, "source/apps/api/src/db/migrations/meta/_journal.json"), + JSON.stringify({ + entries: [ + { idx: 0, tag: "0000_initial", when: 1 }, + { idx: 1, tag: "0001_newest_change", when: 1794754360788 }, + ], + }), + ); mkdirSync(join(temp, "release-plan")); writeFileSync( join(temp, "release-plan/release-plan.json"), @@ -365,6 +375,7 @@ describe("Docker release trust gate", () => { schemaVersion: 1, release: `${"a".repeat(40)}-1-2`, buildMetadata, + databaseMigrationHead: { tag: "0001_newest_change", when: 1794754360788 }, producer: { repository: "OpenMapX/openmapx", workflowPath: ".github/workflows/ci.yml", diff --git a/scripts/product-release.test.ts b/scripts/product-release.test.ts index 486e847d0..efc9a24ef 100644 --- a/scripts/product-release.test.ts +++ b/scripts/product-release.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { parseReleaseComposeSelection } from "../packages/core/src/services/release-manifest"; +import { parse } from "yaml"; import { fixture } from "./product-release.fixtures"; import { inspectPublication, @@ -281,9 +281,15 @@ it.each([ it("provides an auditable usable overlay and exact evidence links before publishing approval", async () => { const t = setup(); const review = await t.stage(); - expect( - parseReleaseComposeSelection(t.state.draft.assets["docker-compose.release.yml"]), - ).toMatchObject({ release: t.candidate.version, images: t.candidate.manifest.images }); + const summary = parse(t.state.draft.assets["docker-compose.release.yml"]) as { + "x-openmapx-release": { release: string; images: Record }; + services: Record; + }; + expect(summary["x-openmapx-release"]).toMatchObject({ + release: t.candidate.version, + images: t.candidate.manifest.images, + }); + expect(summary.services["app-api"]?.image).toBe(t.candidate.manifest.images.api); expect(review.readiness.checks.cleanInstall.evidence).toBe( "https://example.invalid/synthetic/cleanInstall", ); From 4a51fcfc6c131f520025066fd5787074a4b55a20 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 23:07:43 +0100 Subject: [PATCH 15/42] docs(control): describe the lease store, recoverable locks, release pins and control rollback The control-plane reference moves the lease store, the recoverable locks, the release pins in the generation and the operator exit from the target design into the current sections. The upgrading guide replaces the services recover-update procedure with openmapx control and the old note that rollback is not promised with the rollback command, its database refusal and the configuration it reverts. Services administration replaces the manual rmdir recovery with openmapx control and says which actions refuse while their services are held; the CLI reference gains the control commands; and the remaining mentions of the release overlay now describe the pins in the applied configuration. --- .../administration/platform-configuration.md | 26 +-- .../administration/services-administration.md | 66 ++++--- docs/docs/developer/architecture.md | 3 +- .../docs/developer/build-artifacts-control.md | 184 ++++++++++++------ docs/docs/developer/cli-reference.md | 36 ++-- docs/docs/developer/product-releases.md | 4 +- docs/docs/guides/transit-engines.md | 4 +- docs/docs/install/configuration.md | 4 +- docs/docs/install/getting-started.md | 18 +- docs/docs/install/managing-services.md | 13 +- docs/docs/install/requirements.md | 2 +- docs/docs/install/upgrading.md | 169 ++++++++-------- .../core/src/build-artifacts/lease-store.ts | 6 +- release-notes/update-admission-recovery.md | 28 --- 14 files changed, 327 insertions(+), 236 deletions(-) delete mode 100644 release-notes/update-admission-recovery.md diff --git a/docs/docs/administration/platform-configuration.md b/docs/docs/administration/platform-configuration.md index a73e5f519..8a2d3eba4 100644 --- a/docs/docs/administration/platform-configuration.md +++ b/docs/docs/administration/platform-configuration.md @@ -54,22 +54,26 @@ checked release update. Images must provide the supported configuration helper. A cancel request does not undo an already committed update. If work has committed, the original verified completion wins. A failed pull, interrupted helper, missing mount or uncertain publication can leave the operation fenced as requiring recovery. -The system does not claim a rollback from an error message or an old overlay alone. +The system does not claim a rollback from an error message or old image pins alone. It releases ownership only after proving the original operation settled. -Inspect the original job and host ops-agent journal. Do not delete locks, fabricate -completion, or submit repeated fresh updates while an outcome is unknown. Restore -connectivity and observe the original operation; where exact recovery is unavailable, -use host access to diagnose and recover the deployment. Initial Docker installation, +Inspect the original job and the host's `pnpm openmapx control status`. Do not delete +locks, fabricate completion, or submit repeated fresh updates while an outcome is +unknown. Restore connectivity and observe the original operation; where exact recovery +is unavailable, resolve the operation from the host with `openmapx control resolve` +(see [Upgrading](../install/upgrading.md#operations-whose-outcome-is-unknown)). Initial Docker installation, bootstrap credentials, offline agent recovery and destructive full-stack shutdown remain host operations. The UI does not expose arbitrary environment variables, mount paths, executable options or Docker socket access. ## Clear a selected release pin -In **System → Updates**, **Selected pin** is separate from the running release. -**Clear selected release pin** requires the exact `CLEAR RELEASE PIN` confirmation -and queues an Activity job bound to that pin. A changed pin is refused. Clearing -leaves images, running containers and the configured channel -unchanged. It does not select local images: a later configuration action may -resolve the configured channel again. Finish any pending data-manager update first. +In **System → Updates**, **Selected pin** names the release the applied +configuration generation pins. **Clear selected release pin** requires the exact +`CLEAR RELEASE PIN` confirmation and queues an Activity job bound to that pin. A +changed pin is refused. Clearing renders the applied configuration again as a new +generation without pins; running containers keep their images until they are next +recreated, and the configured channel is unchanged. It does not select local +images: a later start may pin the configured channel again. Finish any pending +data-manager update first. To return to the previous generation with its pins, use +`openmapx control rollback` from the host (see [Upgrading](../install/upgrading.md#rolling-back)). diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index 11a859297..440a0fc41 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -100,8 +100,8 @@ new targets to the request. Updates offer **without dependency recreation**, cor Builds take an optional region, a "continue on errors" toggle, and MOTIS-only import (`--import`) for a selected MOTIS build or Build All with enabled, buildable MOTIS. Other actions cannot carry these options. Whole-stack starts explicitly name the -frozen generated service closure so release-overlay entries cannot create disabled -services. Each +frozen generated service closure, so they never create a service the generation +does not enable. Each action is queued as a background job — the toast names the job id, and the **View Jobs** link opens the activity view where the job's log streams. @@ -115,7 +115,8 @@ check the returned job before deliberately requesting another run. **Stage selected images** downloads immutable image content without rendering, retagging, starting, or recreating services. It resolves the configured image for -the host's platform; release-managed services use the exact selected release pin. +the host's platform; release-managed services use the exact release pin of the +applied configuration. Configured tags and running-container ownership stay unchanged. In local release mode, floating references are resolved to manifest digests. An unavailable or ambiguous platform manifest fails before downloading. Follow the retained job in @@ -235,7 +236,14 @@ closure. It preserves existing Data Manager and operations-agent containers with `--no-recreate`, then applies ordinary services with `--no-deps`. Apply or update Data Manager from its service page. Use **System → Deployment** or **System → Updates** to replace the agent through the guarded helper. Up does not silently replace either -protected container or start services mentioned only by a release overlay. +protected container. + +Stop, restart, start, update, removal, Compose Up and Down, and backup create and +restore each claim their services first, in the operations agent's lease store, +from the panel and from the host CLI alike. While another operation holds one of +the services, the action is refused at once with a busy message that names the +holding operation's key and points to `openmapx control status`. Pulling, staging +images, builds and probes take no lease. Full shutdown stops the panel and its agent, so the page shows a copyable host command instead of a nonfunctional stop button: @@ -482,29 +490,33 @@ The agent handles SIGTERM and SIGINT by closing its listener and awaiting close hooks, including any running prune pass. Its shutdown deadline is 30 seconds; the shipped Compose manifest allows 45 seconds before Docker forces termination. Re-render and recreate the agent to adopt that Compose setting. A forced stop, -process crash or power failure can still leave an operation or writer guard -requiring supervised recovery. - -If a service reports a busy or unrecoverable writer after a crash: - -1. Preserve the logs and an ownership/mode-preserving copy of the control stores. - Identify the affected role and its actual mounts using the deployment's - rendered Compose configuration. -2. Stop app-api, Data Manager and the operations agent, and establish that no host - CLI, fixed CLI or helper container can still write those same stores. The age - of a directory or absence of one PID is insufficient evidence. -3. Only after establishing that exclusion, an interrupted **empty writer-guard - directory** may be removed with `rmdir` at its verified exact path. Do not - recursively remove a control store, target lease/lock, execution, receipt or - index record. Do not delete signed records to make capacity available. -4. Restart the services and inspect their recovery logs and Activity. Original - signature and completion checks still apply. Removing a stale - writer guard proves neither cancellation nor containment: an uncertain - operation must remain fenced until its original recovery requirements hold. - -Mismatched signed files or missing completion evidence need separate supervised -recovery. Do not repair those by editing signatures, copying records or -repeatedly deleting guards. +process crash or power failure can still leave an operation whose outcome is +unknown, or a lock whose holder cannot be proved gone. + +If a service reports a busy target, writer or lock after a crash, recover from the +host: + +1. Run `pnpm openmapx control status`. It lists the held leases with their + operation keys and children, the agent operations without a final outcome, and + the holders of the configuration writer, lease store and backup store locks. +2. For an operation whose outcome is unknown, check the containers (the command + prints what Docker reports for the held services), then record the outcome + with `pnpm openmapx control resolve --op --as committed|failed`. This + releases the lease; for a configuration maintenance it also settles Data + Manager's configuration fence. +3. For a lease whose holder is gone, `pnpm openmapx control clear --op ` + drops it. A cleared configuration maintenance leaves the fence prepared until + you resolve the same operation. +4. A lock is reclaimed automatically when its holder is provably gone. When it is + not (on macOS, or a host CLI holder seen from a container), first check that + the named process has exited, then run + `pnpm openmapx control clear --lock writer|leases|backups`. + +Every clear and resolve is appended to +`infra/docker/state/ops-agent/control-audit.jsonl` and printed. Do not delete the +lease store, lock files or signed records by hand: a missing record proves +neither cancellation nor containment, and an uncertain operation must stay held +until it is resolved. ## Platform configuration topics diff --git a/docs/docs/developer/architecture.md b/docs/docs/developer/architecture.md index 4608feb38..71ca581e6 100644 --- a/docs/docs/developer/architecture.md +++ b/docs/docs/developer/architecture.md @@ -47,7 +47,8 @@ openmapx/ └── docs/ this documentation site ``` -The CLI writes generated Compose files, release overlays, and hardlink plans +The CLI writes configuration generations (Compose files with release image pins, +and hardlink plans) under `infra/docker/`. That directory also contains operator input and durable state, including `.env`, service selection, secrets, and data; it is not a throwaway build directory. Do not hand-edit generated Compose output. diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index 91c2587ee..ae1ec0e1f 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -34,36 +34,36 @@ pass once the replacement removes the defect. "Not yet pinned" names a scenario the suite cannot express yet; the guarantee has no crash-point evidence until it is. -| Mechanism group | Failure prevented | How credible | Verdict | Crash-point scenario (status) | -| ------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Typed agent operations, per-role agent tokens, an API data-manager token held to the API's routes, resource policy, owned-service resolver | A compromised API or data-manager runs arbitrary Docker commands or widens the targets it may touch | The real boundary against a compromised service | Keep | Same key with a different body is a conflict (pinned); the API's data-manager token is refused on the agent's routes (pinned) | -| Separate Postgres roles for the API and data-manager | A compromised data-manager forges API rows: accounts, administrative jobs or desired configuration | Real against data-manager: PostgreSQL privileges, a command allowlist and restores that recreate each role's objects as that role. A compromised API can still make data-manager, OpenConditions' roles or the tile owner run its functions through `public`, never the superuser; an operator's superuser session can run a role's code (see Known weaknesses) | Keep | Restore of the pre-update backup (not yet pinned). Until it is, the PostGIS entrypoint test (`scripts/postgis-entrypoint.test.ts`), the database role test (`apps/api/src/db/__tests__/database-roles-postgres.test.ts`) and the owners-and-grants restore test (`packages/cli/__tests__/backup-restore-postgres.integration.test.ts`) cover the roles | -| Agent job journal and idempotent operation keys | A retry after a lost reply applies an operation twice | Observed need: an earlier missing journal caused an eight-hour outage | Keep | Lost reply on configuration prepare (expected failure today); lost reply on data fence prepare and settle (pinned) | -| Drain before self-replacement, ordered recreate, agent last | A self-update leaves services half-replaced, or the agent replaces itself while work is in flight | Observed need | Keep, simplify the helper | Helper killed mid-recreate (pinned); agent restarted while the helper runs (pinned); CLI command while the helper runs (expected failure today) | -| Configuration generations, one atomic pointer, retention against live containers | A crash leaves a half-written configuration, or pruning removes files a running container still uses | Observed need | Keep | API killed between saving desired rows and applying the generation (expected failure today) | -| Pre-pull, release manifest digests, post-recreate verification | An unavailable registry or an unhealthy image is discovered after the old containers are gone | Real failures, about the right size | Keep | Registry outage before maintenance (pinned); new image unhealthy after recreate (pinned) | -| Confined hardlinks, archive validation, descriptor-anchored paths on shared data | A peer plants a symlink on `/data` and redirects an agent write | Real: data-manager can write `/data` | Keep, narrowed | Unit tests only | -| Backup workflow and administrative job runner with one job owner | Two API processes run the same job, or a double-submitted update creates two jobs | Real, about the right size | Keep | Concurrent admissions (pinned); API killed after the job is claimed (expected failure today) | -| "Unknown never releases", effect-started record, cancellation fence | A crash after an effect began is mistaken for "nothing happened" and the targets are released or the effect is repeated | The most valuable rule in the layer | Keep the rule, simplify the form to one journal row | Agent killed while a Compose effect is in flight (expected failure today); agent killed after accepting, before the effect (pinned); cancelled during the fence settle (pinned); cancelled mid-recreate (expected failure today); deadline variants (not yet pinned) | -| Target leases and the child inventory | Two writers (agent, host CLI, data job, helper) change one service at once, or a surviving delegated writer loses its claim | Real guarantee; a kernel lock cannot cover a surviving helper | Simplify to durable rows in one agent-private store | Concurrent admissions (pinned); a surviving data-manager child after an agent restart (not yet pinned) | -| Data-job fence | Configuration maintenance runs while a data job writes the same data | Postgres is the right tool | Done: one SQL row under the existing advisory lock | Data-manager restarted while the fence is held (pinned); an update requested while a data job runs (pinned); data-manager killed during a data job while the agent is down (not yet pinned) | -| Maintenance state machine and candidate lifecycle | A crash between saving and applying configuration leaves rows and files disagreeing | Real guarantee, very large form | Simplify to one record and three phases | API killed between saving desired rows and applying the generation (expected failure today) | -| Configuration writer lock and artifact-store lock | Two writers render or publish at once | Real; today a crash inside the lock needs a manual `rmdir` | Simplify to a recoverable lock never held across a Docker effect | Concurrent admissions (pinned); CLI command while the helper runs (expected failure today) | -| Release overlay and selection receipts | Images and configuration drift apart | Two pointers where one suffices | Simplify to image pins inside the generation | Registry outage before maintenance (pinned) | -| Two data-manager admission models | A crashed legacy data job blocks all data work | The second model caused that failure | Merge | Data-manager killed during a data job while the agent is down (not yet pinned) | -| Role keys, certificates, per-lease and child keys, signed assertions, witnesses | A role forges another role's lease records on the shared ledger | Exists only because lease state sits on shared `/data` | Cut once lease state moves to the agent store | Covered by the authority gate (agent state mounted nowhere else) and the pinned scenarios above | -| Sequence numbers, retention clock, quotas, retention families, retirement commits, byte budgets, `mkdir` guards | Unbounded growth of signed records, replay after deletion | Self-inflicted by the signed records; a time-based prune replaces it | Done: one prune 72 hours after the root key expires; a free-space check at admission | Agent state filesystem full right after an effect (pinned) | -| Settlement cells and slots, the no-guard closure, recovery proofs | A lost settle reply leaves the fence held forever | Third-order machinery | Cut for the fence: settle is idempotent on the fence row. Recovery proofs stay until the operator exit | Lost reply on data fence settle (pinned) | -| Sealed configuration snapshot queue and its lock | The agent applies a configuration the API did not approve | File transport created the need | Cut, the payload travels in the authenticated operation | API killed between saving desired rows and applying the generation (expected failure today) | +| Mechanism group | Failure prevented | How credible | Verdict | Crash-point scenario (status) | +| ------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Typed agent operations, per-role agent tokens, an API data-manager token held to the API's routes, resource policy, owned-service resolver | A compromised API or data-manager runs arbitrary Docker commands or widens the targets it may touch | The real boundary against a compromised service | Keep | Same key with a different body is a conflict (pinned); the API's data-manager token is refused on the agent's routes (pinned) | +| Separate Postgres roles for the API and data-manager | A compromised data-manager forges API rows: accounts, administrative jobs or desired configuration | Real against data-manager: PostgreSQL privileges, a command allowlist and restores that recreate each role's objects as that role. A compromised API can still make data-manager, OpenConditions' roles or the tile owner run its functions through `public`, never the superuser; an operator's superuser session can run a role's code (see Known weaknesses) | Keep | Restore of the pre-update backup (not yet pinned). Until it is, the PostGIS entrypoint test (`scripts/postgis-entrypoint.test.ts`), the database role test (`apps/api/src/db/__tests__/database-roles-postgres.test.ts`) and the owners-and-grants restore test (`packages/cli/__tests__/backup-restore-postgres.integration.test.ts`) cover the roles | +| Agent job journal and idempotent operation keys | A retry after a lost reply applies an operation twice | Observed need: an earlier missing journal caused an eight-hour outage | Keep | Lost reply on configuration prepare (expected failure today); lost reply on data fence prepare and settle (pinned) | +| Drain before self-replacement, ordered recreate, agent last | A self-update leaves services half-replaced, or the agent replaces itself while work is in flight | Observed need | Keep, simplify the helper | Helper killed mid-recreate (pinned); agent restarted while the helper runs (pinned); CLI command while the helper runs (expected failure today) | +| Configuration generations, one atomic pointer, retention against live containers | A crash leaves a half-written configuration, or pruning removes files a running container still uses | Observed need | Keep | API killed between saving desired rows and applying the generation (expected failure today) | +| Pre-pull, release manifest digests, post-recreate verification | An unavailable registry or an unhealthy image is discovered after the old containers are gone | Real failures, about the right size | Keep | Registry outage before maintenance (pinned); new image unhealthy after recreate (pinned) | +| Confined hardlinks, archive validation, descriptor-anchored paths on shared data | A peer plants a symlink on `/data` and redirects an agent write | Real: data-manager can write `/data` | Keep, narrowed | Unit tests only | +| Backup workflow and administrative job runner with one job owner | Two API processes run the same job, or a double-submitted update creates two jobs | Real, about the right size | Keep | Concurrent admissions (pinned); API killed after the job is claimed (expected failure today) | +| "Unknown never releases", effect-started record, cancellation fence | A crash after an effect began is mistaken for "nothing happened" and the targets are released or the effect is repeated | The most valuable rule in the layer | Keep the rule, simplify the form to one journal row | Agent killed while a Compose effect is in flight (expected failure today); agent killed after accepting, before the effect (pinned); cancelled during the fence settle (pinned); cancelled mid-recreate (expected failure today); deadline variants (not yet pinned) | +| Target leases and the child inventory | Two writers (agent, host CLI, data job, helper) change one service at once, or a surviving delegated writer loses its claim | Real guarantee; a kernel lock cannot cover a surviving helper | Done: rows in one agent-private store, `leases.json`, written under a recoverable lock | Concurrent admissions (pinned); a surviving data-manager child after an agent restart (not yet pinned) | +| Data-job fence | Configuration maintenance runs while a data job writes the same data | Postgres is the right tool | Done: one SQL row under the existing advisory lock | Data-manager restarted while the fence is held (pinned); an update requested while a data job runs (pinned); data-manager killed during a data job while the agent is down (not yet pinned) | +| Maintenance state machine and candidate lifecycle | A crash between saving and applying configuration leaves rows and files disagreeing | Real guarantee, very large form | Simplify to one record and three phases | API killed between saving desired rows and applying the generation (expected failure today) | +| Configuration writer lock and artifact-store lock | Two writers render or publish at once | Real; a crash inside the lock no longer needs a manual `rmdir` | Done for the writer, lease, backup-store and update receipt locks: a lock file reclaimed when its holder is provably gone | Concurrent admissions (pinned); CLI command while the helper runs (expected failure today) | +| Release overlay and selection receipts | Images and configuration drift apart | Two pointers where one suffices | Done: image pins and the manifest inside the generation; `openmapx control rollback` | Registry outage before maintenance (pinned) | +| Two data-manager admission models | A crashed legacy data job blocks all data work | The second model caused that failure | Merge | Data-manager killed during a data job while the agent is down (not yet pinned) | +| Role keys, certificates, per-lease and child keys, signed assertions, witnesses | A role forges another role's lease records on the shared ledger | Exists only because lease state sits on shared `/data` | Cut once lease state moves to the agent store | Covered by the authority gate (agent state mounted nowhere else) and the pinned scenarios above | +| Sequence numbers, retention clock, quotas, retention families, retirement commits, byte budgets, `mkdir` guards | Unbounded growth of signed records, replay after deletion | Self-inflicted by the signed records; a time-based prune replaces it | Done: one prune 72 hours after the root key expires; a free-space check at admission | Agent state filesystem full right after an effect (pinned) | +| Settlement cells and slots, the no-guard closure, recovery proofs | A lost settle reply leaves the fence held forever | Third-order machinery | Cut for the fence: settle is idempotent on the fence row. Recovery proofs stay until the operator exit | Lost reply on data fence settle (pinned) | +| Sealed configuration snapshot queue and its lock | The agent applies a configuration the API did not approve | File transport created the need | Cut, the payload travels in the authenticated operation | API killed between saving desired rows and applying the generation (expected failure today) | ## Glossary - **Target.** A service ID, such as `motis` or `app-api`, whose containers a writer may change. Writers claim targets before any Docker effect. - **Lease.** A durable claim on a set of targets by one execution. While it is - held, no other writer may begin an effect on those targets. Today a lease is a - signed owner certificate plus target lock files in the shared ledger under - `infra/docker/data/.admin-target-control/`. + held, no other writer may begin an effect on those targets. Leases are rows in + the agent's store, `infra/docker/state/ops-agent/leases.json`; see + [Lease store](#lease-store). - **Child.** A delegated unit of work recorded under a lease before it starts: a fixed CLI child, a data-manager job or the configuration helper. A lease cannot be released until every attempted child is settled. @@ -78,7 +78,9 @@ it is. writer from a record. A crash inside a `mkdir` guard leaves the directory behind until an operator removes it. - **Generation.** An immutable rendered configuration directory under - `infra/docker/state/ops-agent/configuration/generations/`, named `cfg1_…`. + `infra/docker/state/ops-agent/configuration/generations/`, named `cfg1_…`. It + holds the compose file with the release's image pins and the release manifest + those pins come from. - **Pointer.** The symbolic link `infra/docker/state/ops-agent/configuration/current` that names the live generation. Switching generations is one atomic rename of the pointer. @@ -275,8 +277,8 @@ second root. Acquisition has one five-second budget. The limits are 256 registered targets, 64 child intents, 128 KiB per lease ledger and 256 unsettled leases. Settled receipts remain until the control prune below removes them. -Host ordering is target lease, nonblocking transit reservation, configuration -writer, then release store. A busy reservation must settle an unused owner without +Host ordering is target lease, nonblocking transit reservation, then +configuration writer. A busy reservation must settle an unused owner without effects; a borrower settles its own node and never releases the parent. No writer is held while awaiting a child that needs that same writer. @@ -307,6 +309,101 @@ Target borrowers also check every intermediate ancestor. Canceling an intermedia borrower fences its retained descendants and returns their original attempted identities without deleting or settling any descendant evidence. +### Lease store + +Every lease lives in one file in the agent's state, `infra/docker/state/ops-agent/leases.json`, +replaced atomically under the recoverable lock `leases.lock` (see [Locks](#locks)). No +other service mounts that directory. The writers are: + +- the agent, for its own operations; +- data-manager, through the typed agent operations `lease.acquire`, `lease.read`, + `lease.update` and `lease.settleChild` over its own agent token; +- the host CLI, the agent's fixed CLI children, the configuration helper and the + catalog bootstrap, which write the file themselves under the same lock. + +A lease row names its owner execution, the targets (any registered service ID, +community services included) and the children recorded under it before they start. +A row is deleted when the owner and every attempted child have a known outcome; the +owner keeps its own release time in its private records. An unknown outcome keeps +the row until an operator resolves it. + +| From | Event | To | +| ------ | ---------------------------------------------------- | -------------------------------------------------------- | +| (none) | Owner acquires free targets | `held` | +| `held` | Owner and every attempted child have a known outcome | `settled`, then the row is deleted | +| `held` | Owner or a child has an unknown outcome | stays `held` until `openmapx control resolve` or `clear` | + +Every Docker effect that changes containers takes its lease through one admission +path: stop, restart, start, recreate, update, remove, `stack.stop`, repository +removal, backup create and restore. Pull, build, exec and probes take none. A held +target refuses the operation before anything is recorded, with a busy error that +names the holder's executor, operation key, kind and state and points to +`openmapx control status`. On the host, `services stop|restart`, `compose up|down` +and `backup create|restore` take the same leases. A Docker command that reports a +failure (a non-zero exit, or an error from Docker) records a contained, known +outcome and releases the lease; a crash, timeout, killed process or lost daemon +connection leaves the outcome unknown and the lease held. + +### Locks + +The configuration writer (`state/ops-agent/configuration/writer.lock`), the lease +store (`leases.lock`), the backup store (`backups/.backup-store.lock`) and the CLI's +system update receipt each use one lock file. It records the holder's PID, boot ID, +PID namespace, container and process start time, and another writer reclaims it +only when that holder is provably gone: the machine booted since, the holder shares +this PID namespace and its process ended, or its container is no longer running. +Reclaimers take turns through a short guard file beside the lock. A live holder is +waited for briefly and then named in a busy error. The writer and lease locks are +never held across a Docker effect; the backup-store lock is held for a whole backup +effect, and a second backup waits up to a minute. + +Some dead holders cannot be proved gone, and their lock stays until +`openmapx control clear --lock `: + +- a holder in another PID namespace without a container ID, such as the host CLI + as the agent or the helper sees it; +- any holder on macOS, which records no process start time, so a reused process + ID counts as alive. + +Check that the named process is gone first. The same command also removes a +reclaim guard that a dead reclaimer left beside the lock. + +### Release pins and rollback + +A generation renders the image of every release service (`app-api`, `app-web`, +`data-manager`, `ops-agent`, `transitous-runner`) and data-manager's Transitous +tools image from one release manifest, and keeps that canonical manifest as +`release-manifest.json`; its SHA-256 is the release's identity. A render that names +no release keeps the applied generation's pins. A release update's maintenance +commits the generation with the release it froze, then the helper verifies the +images and recreates the services. Clearing the selected pin renders the applied +configuration again without pins. The agent caches the manifests it checked under +`state/ops-agent/releases/.json`; the newest is the release an update can +apply. A manifest's `producer` and `buildMetadata` are display metadata. + +The configuration directory records the last pointer move, which names the +generation applied before the current one. `openmapx control rollback` renders +that generation's configuration and release again as a new generation and +recreates the services in the self-update order. It refuses when the database's +newest applied migration (`created_at` in `drizzle.__drizzle_migrations`) is newer +than the target release's `databaseMigrationHead`, which CI writes into the release +manifest from the API's migration journal; the migrator alone would run an older +image against a newer schema. Rolling back also reverts the configuration made +since: the release selection, the selected services, service configuration and +secrets. + +### Operator exit + +`openmapx control status` prints the held leases, the agent operations without a +final outcome and the lock holders. `control clear --op ` drops a lease whose +holder is gone, and `control resolve --op --as committed|failed` records the +outcome of an operation whose outcome is unknown and releases it. Both are +host-only, print what Docker reports for the held targets first, and are appended +to `state/ops-agent/control-audit.jsonl` and printed. Clearing a configuration +maintenance leaves the data-manager's configuration fence prepared; `control +resolve` of the same operation settles it through the CLI's data-manager token, +also after a clear. `control clear --lock ` is described under [Locks](#locks). + ### Authority keys and the control prune New root authority uses `opk1_t1_<14 lower-case hex expiry>_<22 canonical @@ -507,44 +604,11 @@ written by atomic rename with fsync. Startup never moves `attempted` or `effect_started` back to `accepted`. -### Lease record - -`LeaseRecord { leaseId, ownerOperationKey, targets, children, state }`, where each -child names a data-manager job or helper container. - -| From | Event | To | -| ------ | ---------------------------------------- | ----------------------------------------------- | -| (none) | Owner operation acquires free targets | `held` | -| `held` | Owner and every attempted child terminal | `released` | -| `held` | Owner or a child is `recovery_required` | stays `held` until an operator exit resolves it | - ### Other target rules -- **Lease store and writer lock.** The agent, its helper and the host CLI write the - lease store directly, each under the same lock file. A lock file records PID, - boot ID, PID namespace, container and process start time, and is reclaimed only - when that process is provably gone. Reclaimers take turns through a short guard - file beside the lock. The lock is never held across a Docker effect. - **Desired configuration.** The payload travels in the authenticated operation. The job row moves through three phases: candidate prepared; rows and decision saved in one transaction; generation applied by a keyed operation. Recovery runs the same forward code. -- **Generation.** Configuration and image pins sit behind one atomic pointer. The - previous pointer is the manual rollback target. -- **Operator exit.** `openmapx control status`, plus `control clear --op ` or - `control resolve --as committed|failed`: host-only, printing what is believed - live, and audited. Clearing a configuration maintenance leaves the data-manager's - configuration fence prepared; `control resolve` of the same operation settles it, - also after a clear. -- **Lock reclaim limits.** Some dead holders cannot be proved gone, and their lock - stays until `openmapx control clear --lock `: - - a holder in another PID namespace without a container ID, such as the host CLI - as the agent or the helper sees it; - - any holder on macOS, which records no process start time, so a reused process - ID counts as alive. - - Check that the named process is gone first. The same command also removes a - reclaim guard that a dead reclaimer left beside the lock. - - **Self-update order.** Pull first, drain, run the helper from the current image, recreate in order with verification after each service, agent last. diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index 68acadf28..ae0dafee5 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -75,8 +75,8 @@ in full under [Managing services](../install/managing-services.md). | `services build ` | Build prepared artifacts for the named services, then re-render and re-link. Flags: `--region `, `--continue-on-error`, `--import` (MOTIS only: also run `motis import`). | | `services build-all` | Build every service that declares a build, in order. Flags: `--region `, `--fail-fast`. | | `services start [ids...]` | Render, apply hardlinks, then `docker compose up -d` the named services. Flag: `--preset `. | -| `services stop [ids...]` | `docker compose stop` the named services. Flag: `--preset `. | -| `services restart [ids...]` | In-place reboot (`docker compose restart`); does **not** re-render. Flag: `--preset `. | +| `services stop [ids...]` | `docker compose stop` the named services. Refused while another operation holds one of them. Flag: `--preset `. | +| `services restart [ids...]` | In-place reboot (`docker compose restart`); does **not** re-render. Refused while another operation holds one of the services. Flag: `--preset `. | | `services update [ids...]` | Re-render, link, pull the configured images, then replace containers with `up -d --force-recreate`. When `app-api` is among them, it first stops `app-api` and applies the new image's database migrations in a one-off container. Flag: `--preset `. For core OpenMapX app releases, use the aggregate release-manifest procedure in [Upgrading](../install/upgrading.md). | | `services status [id]` | Container status table for one or all services (`docker compose ps`). | | `services logs ` | Stream service logs. Flags: `--tail ` (default `100`), `--follow`. | @@ -94,15 +94,15 @@ and mix freely with explicit ids — see the Render and operate the generated Docker Compose stack. OpenMapX keeps no hand-written compose file — the renderer derives it from the enabled manifests. -| Command | Description | -| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `compose render` | Apply the selection as a new configuration generation: compose file and hardlink plan from the manifests, with the settings and credentials the admin panel last applied. Generates any missing platform secret files in `infra/docker/secrets/`, among them the API's and data-manager's database passwords and the data-manager caller tokens. Flags: `--domain ` (default `$DOMAIN`), `--services `, `--preset `. | -| `compose up` | Render, apply hardlinks, resolve the release overlay if it is missing, then `docker compose up -d` the whole selection. Flags: `--domain `, `--services `, `--preset `. | -| `compose release` | Resolve `OPENMAPX_RELEASE` (main, preview, exact version/digest; default main) and write `infra/docker/state/ops-agent/releases/docker-compose.release.yml`, pinning the six product images: API, web, data-manager, ops-agent, Transitous runner, and Transitous tools. `--check` validates/caches metadata without changing pins or running services. Bare selection is bootstrap or exact-match recovery only when Data Manager exists. `--status` inspects local pins without registry access; `--clear` inspects Docker and refuses to remove a valid overlay used by its matching Data Manager. Lifecycle commands include the selected overlay. `OPENMAPX_RELEASE=local` explicitly disables release resolution. | -| `system update` | Apply an already checked release through the real administrator's public API. Requires `--release-id`, `--digest`, `--session-file`, and a new private `--receipt`. Always creates and verifies a backup. `--api-origin` defaults to `https://DOMAIN`; `--resume` reuses the original requests/jobs. See [Upgrading](../install/upgrading.md#4-check-and-apply-the-complete-release) for private session-file and recovery requirements. | -| `compose down` | Stop the stack (`docker compose down`). Flag: `--volumes` removes named volumes (**destructive**). | -| `compose pull [ids...]` | Pull the selected images (no args pulls all services); resolve or preserve a coherent overlay for product services. For core OpenMapX app releases, use the aggregate release-manifest procedure in [Upgrading](../install/upgrading.md). | -| `compose rotate-redis-password` | Replace the password and reconcile its ACL while Redis clients are stopped. Each file is replaced atomically; the pair is not one transaction. Flag: `--confirm-clients-stopped`. Follow the [rotation procedure](https://github.com/OpenMapX/openmapx/blob/main/infra/docker/secrets/README.md#redis-password-and-redis-aclconf-generated). | +| Command | Description | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `compose render` | Apply the selection as a new configuration generation: compose file and hardlink plan from the manifests, with the settings and credentials the admin panel last applied. Generates any missing platform secret files in `infra/docker/secrets/`, among them the API's and data-manager's database passwords and the data-manager caller tokens. Flags: `--domain ` (default `$DOMAIN`), `--services `, `--preset `. | +| `compose up` | Render, apply hardlinks, then `docker compose up -d` the whole selection. When the applied configuration pins no release, it first pins the release the configured selection resolves to; existing pins are kept. Flags: `--domain `, `--services `, `--preset `. | +| `compose release` | Resolve `OPENMAPX_RELEASE` (main, preview, exact version/digest; default main) and render a new configuration generation that pins the six product images: API, web, data-manager, ops-agent, Transitous runner, and Transitous tools. `--check` validates and caches metadata without changing pins or running services. Bare selection is bootstrap or exact-match recovery only when Data Manager exists. `--status` shows the applied configuration's pins without registry access; `--clear` renders the configuration without pins and refuses while Data Manager runs the pinned image. `OPENMAPX_RELEASE=local` explicitly disables release resolution. | +| `system update` | Apply an already checked release through the real administrator's public API. Requires `--release-id`, `--digest`, `--session-file`, and a new private `--receipt`. Always creates and verifies a backup. `--api-origin` defaults to `https://DOMAIN`; `--resume` reuses the original requests/jobs. See [Upgrading](../install/upgrading.md#4-check-and-apply-the-complete-release) for private session-file and recovery requirements. | +| `compose down` | Stop the stack (`docker compose down`). Flag: `--volumes` removes named volumes (**destructive**). | +| `compose pull [ids...]` | Pull the selected images (no args pulls all services); for product services, pin a coherent release first when the applied configuration pins none. For core OpenMapX app releases, use the aggregate release-manifest procedure in [Upgrading](../install/upgrading.md). | +| `compose rotate-redis-password` | Replace the password and reconcile its ACL while Redis clients are stopped. Each file is replaced atomically; the pair is not one transaction. Flag: `--confirm-clients-stopped`. Follow the [rotation procedure](https://github.com/OpenMapX/openmapx/blob/main/infra/docker/secrets/README.md#redis-password-and-redis-aclconf-generated). | For the difference between `services start` (a subset) and `compose up` (the whole stack), and how `--services` overrides the persisted selection for a single @@ -114,6 +114,20 @@ has links to make or prune. With nothing prepared it succeeds. When engines have built data, use `services start `, which applies the plan through a signed execution. +## `control` + +The operator exit for the update system. Host only: it reads and writes the +operations agent's lease store under `infra/docker/state/ops-agent`. Every clear, +resolve and rollback is appended to `control-audit.jsonl` there and printed. + +| Command | Description | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `control status` | Show the held leases with their owner, targets and children, the agent operations without a final outcome, and the holders of the writer, lease store and backup store locks. | +| `control resolve --op --as committed\|failed` | Record the outcome of an operation whose outcome is unknown and release its lease. Prints what Docker reports for the held services first. Resolving a configuration maintenance also settles Data Manager's configuration fence, also after a clear. | +| `control clear --op ` | Drop the lease of an operation whose holder is gone. A cleared configuration maintenance leaves the fence prepared until `control resolve`. | +| `control clear --lock ` | Remove a lock (`writer`, `leases` or `backups`, or the file name a busy message prints) whose holder cannot be proved gone, and a reclaim guard a dead reclaimer left beside it. Check that the holder has exited first: on macOS, and for a host CLI holder seen from a container, a dead holder is not reclaimed automatically. | +| `control rollback [--yes]` | Render the configuration generation applied before the current one again, with its release pins, and recreate the services in the self-update order. Refuses when the database has a migration newer than that release's recorded migration head. Rolling back also reverts the configuration made since: release selection, selected services, service configuration and secrets. Without `--yes` it only prints the plan. | + ## `data` Download source data, build prepared artifacts, and keep consumer directories diff --git a/docs/docs/developer/product-releases.md b/docs/docs/developer/product-releases.md index 246fab6e9..b1c16635a 100644 --- a/docs/docs/developer/product-releases.md +++ b/docs/docs/developer/product-releases.md @@ -68,11 +68,11 @@ GitHub authorization errors remain errors. They are never treated as a missing d | `OPENMAPX_RELEASE=preview` | Most recent deliberately published named prerelease. Missing preview fails. | | `OPENMAPX_RELEASE=v0.1.0-alpha.1` | Exact named prerelease; content must match the version. | | `OPENMAPX_RELEASE=sha256:<64hex>` | Exact metadata image digest, including an authorized staged candidate. | -| `OPENMAPX_RELEASE=local` | Explicit local workflow; existing overlays still apply until cleared. | +| `OPENMAPX_RELEASE=local` | Explicit local workflow; the applied release pins still apply until cleared. | The optional trusted repository setting is `OPENMAPX_RELEASE_MANIFEST_REPOSITORY`, default `ghcr.io/openmapx/release-manifest`. No stable/latest channel or main fallback exists. After the first real publication, explicitly choose preview for named prereleases; installation never guesses whether it exists. See [configuration migration](../install/configuration.md#changing-release-selection). -Use `pnpm openmapx compose release --status` to inspect deliberate pins. For existing installations, `compose release --check` caches a candidate without changing selected pins; System → Updates or authenticated `system update` applies the exact checked ID/digest through the coordinated producer. Bare `compose release` refuses to change a running Data Manager's pin and is limited to bootstrap or exact-match recovery. Normal service commands preserve an existing coherent overlay. A configuration change requires a new check. See [Upgrading](../install/upgrading.md). +Use `pnpm openmapx compose release --status` to inspect deliberate pins. For existing installations, `compose release --check` caches a candidate without changing selected pins; System → Updates or authenticated `system update` applies the exact checked ID/digest through the coordinated producer. Bare `compose release` refuses to change a running Data Manager's pin and is limited to bootstrap or exact-match recovery. Normal service commands keep the applied configuration's release pins. A configuration change requires a new check. See [Upgrading](../install/upgrading.md). ## Verification boundaries diff --git a/docs/docs/guides/transit-engines.md b/docs/docs/guides/transit-engines.md index 88034fadb..69dcddfa0 100644 --- a/docs/docs/guides/transit-engines.md +++ b/docs/docs/guides/transit-engines.md @@ -85,8 +85,8 @@ pnpm openmapx data generate-api-keys Keep this file private. Data-manager mounts its directory read-only and reads `/config/transitous/api-keys.json` at every run, so edits apply at the next one. See the [secrets directory's README](https://github.com/OpenMapX/openmapx/blob/main/infra/docker/secrets/README.md) -for its layout. The release overlay must be selected before a -host-side MOTIS build so its helper image matches the release. +for its layout. A release must be pinned before a host-side MOTIS build, so its +helper image matches the release. ### 3. Build the prepared data diff --git a/docs/docs/install/configuration.md b/docs/docs/install/configuration.md index b523a817a..95b152681 100644 --- a/docs/docs/install/configuration.md +++ b/docs/docs/install/configuration.md @@ -222,7 +222,7 @@ you intend to manage through that surface, then recreate ops-agent. Setting ### Changing release selection `main` selects the development channel. `preview` selects the latest deliberately -published named prerelease and fails if none exists. Changing configuration does not silently replace an existing overlay: inspect `pnpm openmapx compose release --status`, then use System → Updates or authenticated `system update` after `compose release --check`. Bare selection is for bootstrap or exact-match recovery; clearing refuses an overlay used by its matching Data Manager. +published named prerelease and fails if none exists. Changing configuration does not silently replace the applied release pins: inspect `pnpm openmapx compose release --status`, then use System → Updates or authenticated `system update` after `compose release --check`. Bare selection is for bootstrap or exact-match recovery; clearing refuses while the running Data Manager uses the pinned image. ### Generated secret files and recovery @@ -838,7 +838,7 @@ so the catalogue stays in sync with the live manifests. ### Changing release selection -The default remains `main` before the first named preview exists. For deliberately published prereleases, set `preview` after the first publication. Changing configuration does not silently replace an existing overlay: inspect `pnpm openmapx compose release --status`, then use System → Updates or authenticated `system update` after `compose release --check`. Bare selection is for bootstrap or exact-match recovery; clearing refuses an overlay used by its matching Data Manager. +The default remains `main` before the first named preview exists. For deliberately published prereleases, set `preview` after the first publication. Changing configuration does not silently replace the applied release pins: inspect `pnpm openmapx compose release --status`, then use System → Updates or authenticated `system update` after `compose release --check`. Bare selection is for bootstrap or exact-match recovery; clearing refuses while the running Data Manager uses the pinned image. ### Generating configuration from the admin panel diff --git a/docs/docs/install/getting-started.md b/docs/docs/install/getting-started.md index 200f7d8e3..6494796a0 100644 --- a/docs/docs/install/getting-started.md +++ b/docs/docs/install/getting-started.md @@ -105,20 +105,20 @@ the defaults, so do not copy a file containing only optional engines. ```bash pnpm openmapx compose release pnpm openmapx compose release --status -pnpm openmapx compose render ``` -Select the release first: render takes the services' files from the selected -release, not from the checkout, and refuses while the checkout's built-in -services differ from that release's. Render creates platform secret files and a configuration generation under +`compose release` resolves the release and renders the first configuration +generation pinning it. Render takes the services' files from the pinned release, +not from the checkout, and refuses while the checkout's built-in services differ +from that release's. Render creates platform secret files and a configuration generation under `infra/docker/state/ops-agent/configuration/generations/`. The `infra/docker/state/ops-agent/configuration/current` pointer selects the generated Compose file and hardlink plan. Keep these deployment files; do not edit generated YAML. -Release resolution writes `infra/docker/state/ops-agent/releases/docker-compose.release.yml`, pinning the -complete application image set. If the registry cannot be reached, restore -access before continuing. `--status` reports the local selection; it does not -inspect running containers. See [Upgrading](./upgrading.md) for channel and pin +The generation's compose file pins the complete application image set, and the +generation keeps the release manifest beside it. Later renders keep those pins. +If the registry cannot be reached, restore access before continuing. `--status` +reports the applied configuration's pins; it does not inspect running containers. See [Upgrading](./upgrading.md) for channel and pin changes. ## 5. Bring up the infrastructure first @@ -206,7 +206,7 @@ pnpm openmapx compose up pnpm openmapx services status ``` -This starts all services in the current selection, retaining the release overlay. +This starts all services in the current selection with the release pins of the applied configuration. With prepared data from steps 6 and 7, it stops on the `/link` refusal described in step 8; start the stack with `services start` and the service ids instead. If you added a first-import engine such as Nominatim or Valhalla, expect a long diff --git a/docs/docs/install/managing-services.md b/docs/docs/install/managing-services.md index fd74b3635..34f828cef 100644 --- a/docs/docs/install/managing-services.md +++ b/docs/docs/install/managing-services.md @@ -253,17 +253,24 @@ pnpm openmapx services status # container table (all se pnpm openmapx services status valhalla # one service pnpm openmapx compose pull [ids...] # pull images without starting containers pnpm openmapx compose release --check # check candidate without changing release pins +pnpm openmapx control status # held services and unfinished operations pnpm openmapx compose rotate-redis-password --confirm-clients-stopped # rotate Redis ACL/secret ``` The distinction between `restart` and `start`/`update` matters: +Stop, restart, start, update and `compose up`/`down` claim their services in the +operations agent's lease store first. While another operation holds one, the +command exits 1 and names the holding operation; `pnpm openmapx control status` +shows it, and [Upgrading](./upgrading.md#operations-whose-outcome-is-unknown) +describes how to resolve one whose outcome is unknown. + - **`restart`** reboots the container in place. It does not re-render, so it won't pick up a changed manifest, `.env`, or selection; use it to bounce a service whose image and config are unchanged. - **`start`** re-renders first, so it picks up compose-file changes. - **`update`** re-renders, pulls the selected images, and force-recreates the - target containers. An existing release overlay remains pinned. Select a newer + target containers. The applied release pins stay. Select a newer application release through System → Updates or authenticated `system update` after `compose release --check`; see [Upgrading](./upgrading.md). Use `--no-deps` to replace only the named containers when dependencies are already healthy. @@ -496,8 +503,8 @@ that image. Each such agent render says so in its output. Overrides apply to both. The agent's files are the checkout's only when its image is a local build. -`OPENMAPX_RELEASE=local` does not clear an existing release overlay, which -keeps pinning the release's images until you clear it with +`OPENMAPX_RELEASE=local` does not clear the release pins of the applied +configuration, which keep pinning the release's images until you clear them with `pnpm openmapx compose release --clear`. And an ops-agent image pulled by its tag, such as `ghcr.io/openmapx/ops-agent:main`, is a published build, not yours. In either case the agent renders that release's service files, not your diff --git a/docs/docs/install/requirements.md b/docs/docs/install/requirements.md index d4af8a18a..6755f0a14 100644 --- a/docs/docs/install/requirements.md +++ b/docs/docs/install/requirements.md @@ -51,7 +51,7 @@ read the registry credentials directory if private images are used. Only the operator's account may write the checkout: services mount configuration files from it, `services build` uses its build contexts, and -`infra/docker/.env` and the release overlay live in it. A private group per +`infra/docker/.env` lives in it. A private group per user, the Debian and Ubuntu default, meets this even with their umask of `002`. Where the account's group has other members, clone and pull with umask `022`. diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index 67245374b..41e382190 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -24,7 +24,8 @@ this choice with the operator and enables it by default. The host checkout supplies the CLI, service definitions, and configuration used to operate the stack. Application images are prebuilt and selected through a release manifest. A Git pull updates the checkout but leaves running containers -unchanged; a release selection writes image pins but does not replace containers. +unchanged; a release selection renders a new configuration that pins the release's +images but does not replace containers. Both operations are followed by an explicit service update. Application release metadata selects six digest-pinned images, including the @@ -175,9 +176,9 @@ pnpm openmapx compose release --check ``` The check pulls only the small metadata image, validates the six immutable -image pins, and caches the canonical manifest. It prints a -**Checked release** ID and a **Manifest digest**. It does not write the release -overlay, download application images or replace containers. Configuration changes +image pins, and caches the canonical manifest under its digest. It prints a +**Checked release** ID and a **Manifest digest**. It does not change the applied +configuration's pins, download application images or replace containers. Configuration changes require inspection before checking again. Create a private directory for the client receipt and an operator-provided @@ -256,12 +257,13 @@ remove it. Bare `pnpm openmapx compose release` is for a clean installation without an existing Data Manager, or restoring the exact immutable pin its container already -uses. It refuses a different pin, ambiguous/unavailable Docker observations, or -an active transaction. The container's recorded image reference and actual image -identity must both match. It never recreates containers. +uses. It renders a new configuration generation that pins the resolved release. +It refuses a different Data Manager pin and ambiguous or unavailable Docker +observations; the container's recorded image reference and actual image identity +must both match. It never recreates containers. Do not select a new pin and then run `services update data-manager`. Host service -lifecycle commands preserve an existing selection; a Data Manager update needs +lifecycle commands keep the applied pins; a Data Manager update needs the coordinated producer above. Updating other services individually does not complete an application release and API migrations move forward only. @@ -271,27 +273,20 @@ prerelease. Exact versions or metadata image digests pin a particular selection. A missing preview or pin fails without choosing main. See [Product releases](../developer/product-releases.md). -A legacy transaction or unconsumed handoff must finish or recover before a new -update. Preserve those records and inspect Activity and diagnostics; do not delete -writer guards or reinterpret a selected overlay as proof of completion. +An unfinished update or unconsumed handoff must finish or recover before a new +update. Preserve those records and inspect Activity and diagnostics; do not +reinterpret pinned images in the applied configuration as proof of completion. ### Recovering a refused host service update A standalone host `services start` or `services update` that fails releases its service locks as it exits, once its failure is recorded under -`infra/docker/data/ops-agent/lifecycle/.json`, so you can fix the +`infra/docker/state/ops-agent/lifecycle/.json`, so you can fix the cause and run the command again. Containers that already started are left running; the retry converges on them. The locks stay in place when the failure could not be recorded, or when a hardlink request had already been issued to Data Manager. A later command that finds a service locked names the operation holding it. -When the command failed at the hardlink step itself, `services recover-update` -below may release the locks. When the hardlink request had already completed and a -later step failed, such as a Compose start that hit a port conflict, -`recover-update` refuses with `Original failure before hardlink completion required` -and no automated release exists yet: those services stay locked. Keep the original -records and do not delete guards. - A read-only traffic inspection failure is terminal. Data Manager also settles an interrupted legacy data job by observation, at startup and then every minute. It looks up each host operation the job recorded by its operation key, and it never @@ -304,28 +299,33 @@ restarted mid-operation it reports the operation as failed with a recovery-required outcome, which does not prove the host effect ended, so the job stays interrupted until the operation's deadline plus 30 seconds has passed. -For a standalone host `services start` or `services update` that failed at -hardlinks after committing its configuration, retain its original operation key -from `infra/docker/data/ops-agent/lifecycle/.json`. Once the -original deadline has expired, the installation owner can run: +### Operations whose outcome is unknown + +Every command that changes containers (stop, restart, start, recreate, update, +remove, `compose up` and `compose down`, backup create and restore) first claims +its services in the operations agent's lease store. While one holds a service, +another refuses at once with a message that names the holder's operation key and +points to `openmapx control status`. A failure Docker reports releases the +services; a crash, timeout, killed command or lost connection to the Docker +daemon leaves the outcome unknown, and the services stay held until you resolve +it. On the host: ```bash -pnpm openmapx services recover-update --operation-key ORIGINAL_OPERATION_KEY +pnpm openmapx control status # held leases, unfinished operations, lock holders +pnpm openmapx control resolve --op OPERATION_KEY --as failed # or --as committed +pnpm openmapx control clear --op OPERATION_KEY # drop a lease whose holder is gone +pnpm openmapx control clear --lock writer # a lock whose holder cannot be proved gone ``` -This requires the updated CLI, Data Manager and ops-agent. It verifies the original signed -failure and asks Data Manager to attest, under its database admission lock, that -the original child has no SQL admission or private adoption record. Only then -can it revoke that unused grant and release the original service locks. A missing -file or HTTP 404 alone is insufficient. Adopted or uncertain work still refuses. -Keep all original records when it refuses; do not delete guards. - -Recovery preserves the failed receipt and committed configuration. It neither -replays the child nor changes containers, and does not make a backup. After -successful recovery, use **System → Updates** or the checked `system update` -workflow above for the next application release. An already blocked installation -may require a separately supervised upgrade before these fixes can run; changing -the checkout alone does not update its running Data Manager. +Check the containers first: each command prints what Docker reports for the held +services before it changes anything. Resolve as `committed` when the effect +completed and as `failed` when it did not. Resolving a configuration maintenance +also settles Data Manager's configuration fence, which otherwise keeps refusing +data jobs; clearing one leaves that fence prepared until you resolve it. Every +clear and resolve is appended to `infra/docker/state/ops-agent/control-audit.jsonl`. +A lock is reclaimed automatically when its holder is provably gone; on macOS, and +for a host CLI holder seen from a container, `control clear --lock` is the way out +once you have checked that the holder has exited. Rendering stops, with the reason, when `infra/docker/.env` is not the deployment user's own regular file, has more than one link, or its group or others can write @@ -337,41 +337,38 @@ inspect the original Activity job and preserve its recovery records. Do not infe a rollback from a disconnected API or an unchanged release-status field. System status includes all five application services. -API and CLI service lifecycle commands automatically include an -existing release overlay, so restarting or re-rendering the stack retains the -digest-pinned release. Keep the overlay with the deployment; -clear it with `pnpm openmapx compose release --clear` only when intentionally -removing the local selection. +The applied configuration generation holds the release's image pins, so +restarting or re-rendering the stack keeps the digest-pinned release. Clear the +pins with `pnpm openmapx compose release --clear` only when intentionally running +images without a release. Forks and mirrored registries select their own channel with `OPENMAPX_RELEASE` and `OPENMAPX_RELEASE_MANIFEST_REPOSITORY` (see [Configuration](./configuration.md)). -`services update` preserves an existing release selection and stops before -container recreation if pulling that pinned release fails. Use System → Updates or the authenticated `system update` workflow +`services update` keeps the applied release pins and stops before container +recreation if pulling that pinned release fails. Use System → Updates or the authenticated `system update` workflow when deliberately applying a newer release. -For local-image workflows, explicit `OPENMAPX_RELEASE=local` disables automatic release resolution. It does -not remove or bypass an existing release overlay. Clear the local selection -explicitly: +For local-image workflows, explicit `OPENMAPX_RELEASE=local` disables automatic +release resolution. It does not remove the pins of the applied configuration. +Clear them explicitly: ```bash pnpm openmapx compose release --clear ``` -This idempotent command removes only `infra/docker/state/ops-agent/releases/docker-compose.release.yml`. -It inspects existing Data Manager containers and refuses to clear a valid overlay -that a matching coordinator still uses. Clearing mismatched or malformed state is -only a recovery action; Docker ambiguity or observation failure refuses. It -preserves running containers and `state/ops-agent/releases/store` manifests and recorded -running state. It refuses to clear while the release store is locked or an update -transaction awaits completion or recovery. `--clear` cannot be combined with -`--status` or `--check`. +This renders the applied configuration again as a new generation without pins; +running containers keep their images until they are next recreated. It refuses +while the running Data Manager uses the pinned image, and on ambiguous or failed +Docker observations. `--clear` cannot be combined with `--status` or `--check`. Clearing does not change your environment. If automatic resolution remains -enabled, the next start/update selects a release again. To use local images, -separately set `OPENMAPX_RELEASE=local` in your deployment environment. -With no overlay and resolution disabled, service updates pull -the manifest tags and may continue after pull failures to allow local images. -This is not an offline mode; Compose can still pull application images. -`compose pull` resolves a coherent release when no overlay exists and product services are included; it preserves existing deliberate pins. Exact helper pull failures never build a replacement image. +enabled, the next `compose up` pins a release again. To use local images, +separately set `OPENMAPX_RELEASE=local` in your deployment environment. With no +pins and resolution disabled, service updates pull the manifest tags and may +continue after pull failures to allow local images. This is not an offline mode; +Compose can still pull application images. `compose up` and `compose pull` pin +the release the configured selection resolves to when the applied configuration +pins none and product services are included; they keep existing pins. Exact +helper pull failures never build a replacement image. To reconcile the **entire** enabled stack while retaining the selected app release, use the CLI: @@ -381,9 +378,8 @@ pnpm openmapx compose pull pnpm openmapx compose up ``` -`compose up` refreshes the generated base file without overwriting the existing -release selection in `docker-compose.release.yml`. Heavy backend engines that did -not change are left running untouched. +`compose up` renders a new generation that keeps the applied release pins. Heavy +backend engines that did not change are left running untouched. :::note[Pull vs. build] There is no "rebuild the app" step in a normal upgrade. The application images are @@ -418,8 +414,7 @@ containers or make a backup. Start a fresh checked update afterward; use `--no-backup` for CLI updates and their resumes when that is your selected policy, or disable the panel's backup -option. `services recover-update` does not recover an API-owned application -job. Do not delete locks, clear database flags or replay the preparation grant. +option. Do not delete locks, clear database flags or replay the preparation grant. ## 5. Database migrations apply automatically @@ -444,8 +439,9 @@ the error is logged there. If you created an optional :::caution[Migrations move forward only] Drizzle migrations are forward-only. There is no automatic down-migration. Once a -new `app-api` has migrated the database, rolling back to an older app image is not -supported unless you also restore the matching database backup. For upgrades +new `app-api` has migrated the database, `openmapx control rollback` refuses to +return to the older release (see [Rolling back](#rolling-back)); restore the +matching database backup instead. For upgrades that cross a schema change, consider taking the optional backup in step 1 when the deployment contains state you cannot readily recreate. ::: @@ -488,7 +484,7 @@ curl --fail --silent --show-error https://maps.example.com/api/status Replace `maps.example.com` with the actual domain. The CLI loads `.env` only into its own process, so `${DOMAIN}` is not automatically available in your shell. -`compose release --status` reports local pins, while `services status` reports +`compose release --status` reports the applied configuration's pins, while `services status` reports container/health state; neither alone proves matching running digests. Inspect the running image identity when verifying an exact release. @@ -506,13 +502,34 @@ and [OSM data queries](../guides/osm-data-queries.md). ## Rolling back -Rollback needs a recovery decision. Image selection does not reverse database -migrations or a committed configuration change. Preserve the previous coherent -manifest and matching database/configuration backups. If a rollback is compatible, -set `OPENMAPX_RELEASE` to its exact metadata digest, check it with `--check` and -review it through the same coordinated workflow. There is no positional -`compose release ` command. Bare selection is permitted only for bootstrap -or restoring the exact Data Manager pin already in use. +`openmapx control rollback` returns to the configuration generation that was +applied before the current one, with the release it pins: + +```bash +pnpm openmapx control rollback # print the plan; nothing changes +pnpm openmapx control rollback --yes # roll back +``` + +It renders that generation's configuration and release again as a new generation +and recreates the services in the self-update order: every other service first, +then PostGIS, Redis, the web app, the API, Traefik and the operations agent last. +A service that fails to recreate stops the rollback; the services after it are +left as they are. + +**Rolling back also reverts every configuration change made since that +generation:** the release selection, the selected services, service +configuration and secrets all return to the previous generation's. Re-apply any +change you still want afterwards. + +The command refuses when the release would change and the database has a newer +migration than the previous release ships. The migrator only applies migrations +newer than the newest applied one, so an older API would otherwise run against a +newer schema without complaint. CI records each release's newest migration in its +release manifest; a release published before that, or a previous configuration +without pins, cannot be checked and is refused too. In those cases, restore the +backup taken before the update. PostGIS must be running so that the command can +read the database's migrations. Only the latest switch between generations is +recorded, so a second rollback returns to the configuration you rolled back from. ## Reclaiming disk after an upgrade diff --git a/packages/core/src/build-artifacts/lease-store.ts b/packages/core/src/build-artifacts/lease-store.ts index 3b972d5fc..adebf6e46 100644 --- a/packages/core/src/build-artifacts/lease-store.ts +++ b/packages/core/src/build-artifacts/lease-store.ts @@ -18,9 +18,9 @@ import { type ContainerRunning, withRecoverableLock } from "../services/recovera /** * Target leases, in the operations agent's own state directory, which no other * service mounts. Every writer that changes containers claims its targets here - * first: the agent for its own operations and on behalf of data-manager and the - * host CLI, the configuration helper while the agent is replaced, the agent's - * fixed CLI children, and the host CLI directly only while the agent is down. + * first: the agent for its own operations and on behalf of data-manager, the + * configuration helper while the agent is replaced, the agent's fixed CLI + * children, and the host CLI, which writes the file itself. * * The file is replaced atomically under a recoverable lock, held only for the * read and write, never across a Docker effect. A lease names its owner and diff --git a/release-notes/update-admission-recovery.md b/release-notes/update-admission-recovery.md deleted file mode 100644 index c3abdee9b..000000000 --- a/release-notes/update-admission-recovery.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -impacts: [operator-action] ---- - -## What changes - -Read-only traffic inspection failures no longer leave an unfinished maintenance -job blocking application updates. Data Manager startup repairs only the exact -legacy traffic failure with no attempted agent operations. - -The host CLI adds `services recover-update --operation-key ORIGINAL_OPERATION_KEY` -for an expired standalone service lifecycle that failed at hardlinks. It requires -Data Manager's signed database and private-record non-admission proof before -revoking the unused grant and releasing the original locks. - -## Who is affected - -Installations whose CLI or admin update is blocked by one of these retained -pre-dispatch failures. Adopted or uncertain operations stay fenced. - -## Required action - -Upgrade the CLI, running Data Manager and ops-agent before using recovery. Preserve the -original failed receipt and operation key. Follow the recovery section in the -upgrade guide; a blocked installation may need a separately supervised upgrade. -Recovery does not replay work, change containers or create a backup. Continue -with the normal checked CLI or admin update after recovery; selected pins and -source changes alone do not establish a running deployment. From d70a45ae7ee17b7d08adb1d37dab51fac7036261 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 23:07:57 +0100 Subject: [PATCH 16/42] docs(release-notes): note the lease store, control commands, release pins and rollback One fragment covers the agent-owned lease store, the recoverable locks and openmapx control, superseding the services recover-update note; another covers the release pins in the generation and openmapx control rollback. The pending release channel note no longer speaks of hand-written overlays. --- release-notes/control-leases.md | 53 +++++++++++++++++++++ release-notes/product-release-channels.md | 6 +-- release-notes/release-pins-in-generation.md | 38 +++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 release-notes/control-leases.md create mode 100644 release-notes/release-pins-in-generation.md diff --git a/release-notes/control-leases.md b/release-notes/control-leases.md new file mode 100644 index 000000000..76c1feb7b --- /dev/null +++ b/release-notes/control-leases.md @@ -0,0 +1,53 @@ +--- +impacts: [breaking, migration, operator-action] +--- + +## What changes + +Every claim a writer makes on a service before changing its containers now +lives in one file in the operations agent's own state, +`infra/docker/state/ops-agent/leases.json`. Data Manager asks the agent for its +claims over HTTP; the host CLI, the agent's fixed CLI children and the +configuration helper write the file under its lock. The shared lease directory +on `/data` is gone. + +Stop, restart, start, recreate, update, remove, `compose up` and `compose down`, +and backup create and restore all take a claim first, from the admin panel and +from the host CLI alike. While another operation holds one of the services, the +command is refused at once with a message that names the holding operation and +points to `openmapx control status`; before, `services restart` could run in the +middle of an update. A failure Docker reports releases the claim. A crash, +timeout, killed command or lost Docker daemon connection keeps it until an +operator resolves it. + +The configuration writer, the lease store, the backup store and the CLI's update +receipt use lock files that another writer reclaims once their holder is +provably gone, instead of directories that needed a manual `rmdir` after a crash. + +`openmapx control status|clear|resolve` is the operator exit: it shows held +claims, unfinished operations and lock holders, records the outcome of an +operation whose outcome is unknown (also settling Data Manager's configuration +fence for a configuration maintenance), and drops a claim or lock whose holder +is gone. Every clear and resolve is appended to +`infra/docker/state/ops-agent/control-audit.jsonl`. +`services recover-update` is removed. The database migration drops a column of +Data Manager's configuration fence that the signed certificates used. + +## Who is affected + +Every installation upgrading from an earlier release, and operators whose +scripts stop or restart services during updates or backups, which now refuse +while the services are held. Lease records and lifecycle receipts of the earlier +format are not converted. + +## Required action + +Install this release fresh, or upgrade only while no update, service lifecycle +command, backup or configuration apply is in progress. Remove an earlier +`infra/docker/data/.admin-target-control` directory by hand after the upgrade; +nothing reads it. When a command reports a held service after a crash, follow +[Operations whose outcome is unknown](../docs/docs/install/upgrading.md#operations-whose-outcome-is-unknown): +check the containers, then `openmapx control resolve --op --as +committed|failed`. On macOS, and for a host CLI lock holder seen from a +container, a dead lock holder is not reclaimed automatically; check that it has +exited, then run `openmapx control clear --lock `. diff --git a/release-notes/product-release-channels.md b/release-notes/product-release-channels.md index 1266e9dac..946bb261d 100644 --- a/release-notes/product-release-channels.md +++ b/release-notes/product-release-channels.md @@ -4,12 +4,12 @@ impacts: [breaking, operator-action] ## What changes -Product release selection uses main, preview, an exact prerelease version or metadata digest, or explicit local mode. Existing overlays must contain a coherent six-image selection; incomplete hand-written overlays are rejected. Admin apply uses the exact checked release and fails if its selection configuration changed. +Product release selection uses main, preview, an exact prerelease version or metadata digest, or explicit local mode. A selected release pins its six images in the configuration generation; there is no hand-written release overlay. Admin apply uses the exact checked release and fails if its selection configuration changed. ## Who is affected -Operators using hand-written overlays or local helper build fallback need to update their configuration. The preview channel does not exist until the first named prerelease is deliberately published. +Operators who used hand-written release overlays or local helper build fallback need to update their configuration. The preview channel does not exist until the first named prerelease is deliberately published. ## Required action -Select a release with OPENMAPX_RELEASE and optional OPENMAPX_RELEASE_MANIFEST_REPOSITORY, or opt out with OPENMAPX_RELEASE=local. Re-render and update the operations agent from the host. Explicitly reselect a coherent release with pnpm openmapx compose release before updating services; never select preview as a fallback before publication. Keep deliberate pins until explicitly reselected or cleared. Exact helper pull failures cannot build replacement bytes, even in local mode. Image rollback does not reverse database migrations. +Select a release with OPENMAPX_RELEASE and optional OPENMAPX_RELEASE_MANIFEST_REPOSITORY, or opt out with OPENMAPX_RELEASE=local. Re-render and update the operations agent from the host. Explicitly reselect a coherent release with pnpm openmapx compose release before updating services; never select preview as a fallback before publication. Pins stay in the applied configuration until explicitly reselected or cleared. Exact helper pull failures cannot build replacement bytes, even in local mode. `openmapx control rollback` refuses to return to a release older than the database's newest migration. diff --git a/release-notes/release-pins-in-generation.md b/release-notes/release-pins-in-generation.md new file mode 100644 index 000000000..5040bb4f8 --- /dev/null +++ b/release-notes/release-pins-in-generation.md @@ -0,0 +1,38 @@ +--- +impacts: [breaking, operator-action] +--- + +## What changes + +A configuration generation now renders the release's image pins into its own +compose file and keeps the release manifest beside it. The release overlay +`infra/docker/state/ops-agent/releases/docker-compose.release.yml`, the release +store's `current.json`, `latest.json` and `selections.json` and the release-store +lock are gone. `compose release` renders a new generation pinning the resolved +release, `compose release --clear` and **Clear selected release pin** render the +applied configuration again without pins, and every other render keeps the +applied pins. The agent keeps a cache of the manifests it checked, named by +digest, under `state/ops-agent/releases/`. + +`openmapx control rollback` returns to the configuration generation applied +before the current one, with its release pins, and recreates the services in the +self-update order. It refuses when the database has a migration newer than the +previous release ships; CI now records each release's newest migration in its +release manifest. Rolling back also reverts the configuration made since that +generation: the release selection, the selected services, service configuration +and secrets. + +## Who is affected + +Every installation upgrading from an earlier release, and operators who read or +edited the overlay file. Release manifests published before this change carry no +migration information, so rollback to one of them is refused. + +## Required action + +Install this release fresh, or after upgrading run `pnpm openmapx compose +release` (or apply the release through **System → Updates**) so that the applied +configuration pins it; then remove the old overlay file and the +`state/ops-agent/releases/store` directory by hand. Read the rollback section of +the upgrade guide before using `openmapx control rollback`, and re-apply any +configuration change you still want after a rollback. From 6363799720c45958a4e3eb4f3285813a400bffa1 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 23:08:14 +0100 Subject: [PATCH 17/42] chore(changeset): note the extension CLI's refreshed bundled core --- .changeset/control-leases-release-pins-core.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/control-leases-release-pins-core.md diff --git a/.changeset/control-leases-release-pins-core.md b/.changeset/control-leases-release-pins-core.md new file mode 100644 index 000000000..933285b8b --- /dev/null +++ b/.changeset/control-leases-release-pins-core.md @@ -0,0 +1,10 @@ +--- +"@openmapx/extension-cli": patch +--- + +Refresh the extension CLI's bundled core inputs following the move of target leases +into the operations agent's store, the recoverable lock files and the release image +pins in the configuration generation. This covers its bundled workspace dependency +artifact; the extension scaffolding, validation and packaging command interfaces +remain unchanged. The administration API and stack-management CLI are separate +product surfaces. From ef15585431f3b92530063c5ed26a379972b9799c Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 23:13:41 +0100 Subject: [PATCH 18/42] test(ci): give the promotion fixture a migration journal and pin the manifest's migration head The promotion step now reads the release's migration journal, so the fixture provides one; the test checks the head written into the manifest and that a journal without a valid newest entry stops the promotion. --- scripts/docker-promotion.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/docker-promotion.test.ts b/scripts/docker-promotion.test.ts index be7270b08..57f78b1b6 100644 --- a/scripts/docker-promotion.test.ts +++ b/scripts/docker-promotion.test.ts @@ -32,6 +32,7 @@ function promote( mismatch = false, reusedApps: string[] = [], planOverrides: Record = {}, + journalEntries: unknown[] = [{ idx: 0, tag: "0000_initial", when: 1794754360788 }], ) { const temp = mkdtempSync(join(tmpdir(), "openmapx-promotion-")); try { @@ -42,6 +43,12 @@ function promote( mkdirSync(dir, { recursive: true }); writeFileSync(join(dir, digests[app].slice(7)), ""); } + // The release's migration journal, as the workflow checks it out beside the plan. + mkdirSync(join(temp, "source/apps/api/src/db/migrations/meta"), { recursive: true }); + writeFileSync( + join(temp, "source/apps/api/src/db/migrations/meta/_journal.json"), + JSON.stringify({ entries: journalEntries }), + ); mkdirSync(join(temp, "release-plan")); writeFileSync( join(temp, "release-plan/release-plan.json"), @@ -146,6 +153,7 @@ describe("aggregate Docker promotion", () => { "2026-09-09T12:00:00.000Z", ); expect(Object.keys(manifest.images)).toEqual(apps); + expect(manifest.databaseMigrationHead).toEqual({ tag: "0000_initial", when: 1794754360788 }); expect(manifest.producer).toEqual({ repository: "OpenMapX/openmapx", workflowPath: ".github/workflows/ci.yml", @@ -216,6 +224,15 @@ describe("aggregate Docker promotion", () => { } }); + it.each([[[]], [[{ idx: 0, tag: "latest", when: 1 }]], [[{ idx: 0, tag: "0000_initial" }]]])( + "refuses a release whose migration journal has no valid newest entry (%j)", + (entries) => { + const { status, stderr } = promote("", false, [], {}, entries); + expect(status).not.toBe(0); + expect(stderr).toContain("The migration journal has no valid newest entry"); + }, + ); + it("rejects a main tag that does not resolve to the approved digest", () => { const { calls, status, stderr } = promote("", true); expect(status).toBe(1); From 05fcb2b1c673d978102183fc91697801d6bdfdb0 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 15 Nov 2026 23:22:57 +0100 Subject: [PATCH 19/42] fix(control): name a lock file by its holder's nonce as well as its inode Linux reuses a freed inode number at once, so a holder whose lock was cleared and a late reclaimer could each judge a new holder's file to be the one they knew and remove it. Every holder record now carries a random nonce, and a lock file counts as the same only when its inode and its bytes both match. The existing tests for a cleared holder's release and for a replaced stale file failed on Linux and pass now. --- packages/cli/__tests__/cli-control.test.ts | 1 + .../core/src/services/recoverable-lock.ts | 55 ++++++++++++------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/packages/cli/__tests__/cli-control.test.ts b/packages/cli/__tests__/cli-control.test.ts index 3e4ac5255..78bef588c 100644 --- a/packages/cli/__tests__/cli-control.test.ts +++ b/packages/cli/__tests__/cli-control.test.ts @@ -128,6 +128,7 @@ describe("openmapx control", () => { pidNamespace: "elsewhere", containerId: null, acquiredAt: Date.now(), + nonce: "d".repeat(32), }; writeFileSync(path, JSON.stringify(holder)); expect(services.readLockHolder(path)).toEqual(holder); diff --git a/packages/core/src/services/recoverable-lock.ts b/packages/core/src/services/recoverable-lock.ts index 18ea89cec..cf8649b30 100644 --- a/packages/core/src/services/recoverable-lock.ts +++ b/packages/core/src/services/recoverable-lock.ts @@ -6,7 +6,6 @@ import { fstatSync, fsyncSync, linkSync, - lstatSync, openSync, readFileSync, readlinkSync, @@ -45,6 +44,8 @@ const holderSchema = z.strictObject({ .regex(/^[a-f0-9]{64}$/) .nullable(), acquiredAt: z.number().int().nonnegative().safe(), + /** Unique per acquisition: a freed inode number is reused at once, so it alone names no file. */ + nonce: z.string().regex(/^[a-f0-9]{32}$/), }); export type LockHolder = z.infer; @@ -101,6 +102,7 @@ export function currentLockHolder(purpose: string, now = Date.now()): LockHolder pidNamespace: currentPidNamespace(), containerId: currentContainerId(), acquiredAt: now, + nonce: randomBytes(16).toString("hex"), }; } @@ -156,7 +158,18 @@ export async function lockHolderGone( return false; } -function readHolderFile(path: string): { holder?: LockHolder; ino: number } | undefined { +/** A lock file as one reader saw it: its inode and its bytes, which carry the holder's nonce. */ +interface LockFile { + holder?: LockHolder; + ino: number; + raw: string; +} + +function sameFile(current: LockFile | undefined, judged: LockFile): boolean { + return current !== undefined && current.ino === judged.ino && current.raw === judged.raw; +} + +function readHolderFile(path: string): LockFile | undefined { let descriptor: number; try { // Never block on a FIFO someone placed at the lock path. @@ -169,14 +182,15 @@ function readHolderFile(path: string): { holder?: LockHolder; ino: number } | un const stat = fstatSync(descriptor); // Waiting never turns another kind of file into a lock; leave it to the operator. if (!stat.isFile()) throw new Error(`${path} is not a lock file; remove it by hand`); - if (stat.size > 4096) return { ino: stat.ino }; + if (stat.size > 4096) return { ino: stat.ino, raw: `oversized:${stat.size}` }; const buffer = Buffer.alloc(stat.size); readSync(descriptor, buffer, 0, buffer.length, 0); + const raw = buffer.toString("utf8"); try { - return { holder: holderSchema.parse(JSON.parse(buffer.toString("utf8"))), ino: stat.ino }; + return { holder: holderSchema.parse(JSON.parse(raw)), ino: stat.ino, raw }; } catch { // A holder that died between creating and writing its file left it empty. - return { ino: stat.ino }; + return { ino: stat.ino, raw }; } } finally { closeSync(descriptor); @@ -188,11 +202,11 @@ export function readLockHolder(path: string): LockHolder | undefined { } /** - * Moves the file at `path` aside if it is still the one judged (`ino`); one - * that changed meanwhile is put back. Used for the reclaim guard, and by the + * Moves the file at `path` aside if it is still the one judged; one that + * changed meanwhile is put back. Used for the reclaim guard, and by the * operator's clear, which acts on a holder it named. */ -function moveAside(path: string, ino: number): void { +function moveAside(path: string, judged: LockFile): void { const aside = `${path}.stale-${randomBytes(6).toString("hex")}`; try { renameSync(path, aside); @@ -200,7 +214,7 @@ function moveAside(path: string, ino: number): void { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; } - if (lstatSync(aside).ino !== ino) { + if (!sameFile(readHolderFile(aside), judged)) { try { linkSync(aside, path); } catch (error) { @@ -220,7 +234,7 @@ function moveAside(path: string, ino: number): void { */ async function reclaim( path: string, - ino: number, + judged: LockFile, self: LockHolder, containerRunning?: ContainerRunning, ): Promise { @@ -237,7 +251,7 @@ async function reclaim( // A reclaimer that died while holding the guard leaves it behind. const other = readHolderFile(guard); if (other?.holder && (await lockHolderGone(other.holder, self, containerRunning))) - moveAside(guard, other.ino); + moveAside(guard, other); return false; } try { @@ -246,7 +260,7 @@ async function reclaim( closeSync(descriptor); } try { - if (readHolderFile(path)?.ino !== ino) return true; + if (!sameFile(readHolderFile(path), judged)) return true; unlinkSync(path); fsyncDirectory(dirname(path)); return true; @@ -286,11 +300,12 @@ export async function acquireRecoverableLock( if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; } if (descriptor !== undefined) { - let ino: number; + let own: LockFile; try { - writeSync(descriptor, JSON.stringify(self)); + const raw = JSON.stringify(self); + writeSync(descriptor, raw); fsyncSync(descriptor); - ino = fstatSync(descriptor).ino; + own = { ino: fstatSync(descriptor).ino, raw }; } finally { closeSync(descriptor); } @@ -299,8 +314,8 @@ export async function acquireRecoverableLock( return () => { if (released) return; released = true; - const current = readHolderFile(path); - if (current?.ino !== ino) throw new Error(`${basename(path)} was taken from this holder`); + if (!sameFile(readHolderFile(path), own)) + throw new Error(`${basename(path)} was taken from this holder`); unlinkSync(path); fsyncDirectory(dirname(path)); }; @@ -314,7 +329,7 @@ export async function acquireRecoverableLock( // An unreadable holder record is only trusted as stale once it has stayed // unreadable past the wait, so a holder still writing it is not displaced. if (existing.holder || performance.now() >= deadline) { - if (await reclaim(path, existing.ino, self, options.containerRunning)) continue; + if (await reclaim(path, existing, self, options.containerRunning)) continue; await delay(5, undefined, { signal: options.signal }); continue; } @@ -340,9 +355,9 @@ export async function withRecoverableLock( /** The operator exit for a lock whose holder cannot be proved gone, and for its reclaim guard. */ export function clearRecoverableLock(path: string): LockHolder | undefined { const guard = readHolderFile(`${path}.reclaim`); - if (guard) moveAside(`${path}.reclaim`, guard.ino); + if (guard) moveAside(`${path}.reclaim`, guard); const existing = readHolderFile(path); if (!existing) return undefined; - moveAside(path, existing.ino); + moveAside(path, existing); return existing.holder; } From 8e330b41067037329a3e14860b06f1103be59ac1 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 00:24:53 +0100 Subject: [PATCH 20/42] test(data-manager): give the catalog workflow's agent the address of its data-manager Since target leases moved into the agent's store, the agent reads a data-manager execution's terminal status from data-manager over HTTP. The catalog workflow test started its agent process without a data-manager address, so activation waited forever after the approval. The test now picks data-manager's port before starting the agent and hands it the address and the agent's token; the native data-manager modes take over that port. --- .../catalog-workflow-postgres.test.ts | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts index 72220ac6d..543304173 100644 --- a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts @@ -12,6 +12,7 @@ import { writeFileSync, } from "node:fs"; import { createRequire } from "node:module"; +import { type AddressInfo, createServer } from "node:net"; import { dirname, join, resolve } from "node:path"; import { createInterface } from "node:readline"; import { promisify } from "node:util"; @@ -452,6 +453,20 @@ it const dockerBridge = (await bridge.listen({ host: "127.0.0.1", port: 0 })) + "/docker"; const require = createRequire(import.meta.url), loader = require.resolve("tsx/esm"); + // The agent reads a data-manager execution's terminal status from + // data-manager itself, so it learns the port data-manager will listen on. + const dmPort = await new Promise((resolvePort, rejectPort) => { + const probe = createServer(); + probe.once("error", rejectPort); + probe.listen(0, "127.0.0.1", () => { + const { port } = probe.address() as AddressInfo; + probe.close(() => resolvePort(port)); + }); + }); + const agentDmTokenFile = join(f.root, "dm-agent-token"); + writeFileSync(agentDmTokenFile, `${Buffer.alloc(32, 23).toString("base64url")}-agent`, { + mode: 0o600, + }); async function startAgent(catalogFault: string, listenPort = 0) { const child = spawn( process.execPath, @@ -460,7 +475,14 @@ it loader, resolve("apps/ops-agent/src/test-fixtures/staged-coordinator-request-process.mjs"), ], - { stdio: ["pipe", "pipe", "pipe"] }, + { + stdio: ["pipe", "pipe", "pipe"], + env: { + ...process.env, + DATA_MANAGER_URL: `http://127.0.0.1:${dmPort}`, + DATA_MANAGER_TOKEN_FILE: agentDmTokenFile, + }, + }, ); children.push(child); let stderr = ""; @@ -647,7 +669,7 @@ it operationsPolicy: policy, executionReceiver: receiver, }); - let dmUrl = await dm.listen({ host: "127.0.0.1", port: 0 }), + let dmUrl = await dm.listen({ host: "127.0.0.1", port: dmPort }), transport = sdk.createDataManagerTransport({ baseUrl: dmUrl, authToken: token }); async function startNativeDm(listenPort = 0) { const child = spawn( @@ -716,7 +738,11 @@ it let nativeDm: Awaited> | undefined; if (fault === "dm-pointer" || fault === "runner-refusal" || fault === "before-pointer") { await runtime.close(); - nativeDm = await startNativeDm(); + // The native data-manager takes over the port the agent knows. + dm.server.closeAllConnections(); + await dm.close(); + dm = undefined; + nativeDm = await startNativeDm(dmPort); transport = sdk.createDataManagerTransport({ baseUrl: nativeDm.url, authToken: token }); } async function terminal(id: string) { From 2708d2a5161c6675c4122362c0efd7a9827c464f Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 00:53:01 +0100 Subject: [PATCH 21/42] fix(ops-agent): drop the admitted lease when admission fails before any effect A request that took its targets' lease and then failed while its plan was built, for example an update whose configuration revision is no longer current, kept every target held until an operator cleared it. Nothing has started at that point, so the lease is dropped. --- apps/ops-agent/src/server-foundation.test.ts | 61 ++++++++++++++++++++ apps/ops-agent/src/server.ts | 5 ++ 2 files changed, 66 insertions(+) diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index d705080d5..a7ff706ca 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -940,6 +940,67 @@ describe("one admission path for target effects", () => { } }); + it("drops the lease of an update whose configuration revision is no longer current", async () => { + const control = await import("@openmapx/core/build-artifacts"); + const { installServiceLifecycleRuntime } = await import("./service-lifecycle-runtime"); + const { mkdtempSync, mkdirSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const root = mkdtempSync(join(tmpdir(), "omx-leased-stale-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["redis", "osrm"], + }); + const leaseStore = control.openLeaseStore({ + directory: control.leaseStoreDirectory(root), + purpose: "test", + }); + const runtime = {} as import("./runtime").OpsRuntime; + installServiceLifecycleRuntime(runtime, { + rootDir: root, + infraDir: root, + runFixedCli: async () => { + throw new Error("no child may start"); + }, + }); + const app = await buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAllResources, + controlInstallation: installation, + leaseStore, + runtime, + }); + try { + const response = await app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(), + payload: { + ...request( + { + kind: "service.update", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"c".repeat(43)}`, + }, + "ops1_staleUpdateLease", + key("opk1_staleUpdateLease"), + ), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + }, + }); + expect(response.statusCode).toBeGreaterThanOrEqual(400); + expect(leaseStore.list()).toEqual([]); + } finally { + await app.close(); + control.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }); + it("releases the targets of a stop that reported a failure, and keeps those of a killed one", async () => { const control = await import("@openmapx/core/build-artifacts"); const { runContainedStreamingProcess } = await import("./docker-runtime"); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index b19a81832..eee996be1 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -2953,6 +2953,11 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif retainOwnedConfigurationMaintenance(authority, lease, maintenanceAdmission); } } catch (error) { + // Nothing has started yet, so a lease taken at admission holds nothing. + if (admittedLease && options.leaseStore) { + controlLeases.delete(job.operationId); + await abandonUnstartedLease(options.leaseStore, admittedLease, executionFor(job)); + } forceRecoveryRequired(job); await journal.replace([...jobsById.values()].map(persisted)); throw error; From 769c116b9864c9633a3a01503e8323303728b054 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 00:55:40 +0100 Subject: [PATCH 22/42] fix(data-manager): repeat a lease acquisition whose reply was lost A timeout or transport failure can arrive after the agent recorded the lease. Data-manager treated it as a refusal and never released that lease, so every later data-manager job and host restart of its targets was refused as busy. Only a held target and the agent's definite pre-effect refusals now end the job before any effect; other failures repeat the acquisition, which returns the lease already recorded. --- .../src/jobs/execution/runtime.test.ts | 71 ++++++++++++++++++- .../src/jobs/execution/runtime.ts | 14 +++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts index e39f161b6..c2d7d8aec 100644 --- a/services/data-manager/src/jobs/execution/runtime.test.ts +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -4,7 +4,7 @@ import { connect, createServer } from "node:net"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; -import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { createControlRootOperationKey, OpsClientError } from "@openmapx/core/ops"; import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { describe, expect, it, vi } from "vitest"; @@ -192,6 +192,75 @@ describe.skipIf( } }, ); + it("repeats a lease acquisition whose reply was lost and releases the lease it recorded", async () => { + const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; + if (!url) throw Error("Owned PostgreSQL required"); + const sql = postgres(url, { max: 4 }); + const root = mkdtempSync(join(tmpdir(), "omx-runtime-lost-reply-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const agent = control.agentStoreLeases(root, "ops-agent"); + let lostReplies = 1; + const installation = control.openControlInstallation({ + infraDir: root, + role: "data-manager", + serviceIds: ["data-manager"], + leases: { + ...agent, + // The agent records the lease, and its reply never arrives. + async acquire(input) { + const lease = await agent.acquire(input); + if (lostReplies-- > 0) throw new OpsClientError("timeout"); + return lease; + }, + }, + }), + journal = control.createDataExecutionControlJournal(installation, sql), + store = createPostgresDataExecutionStore(sql, { controlJournal: journal }); + let effects = 0; + const runtime = createDataExecutionRuntime({ + store, + dispatch: async () => { + effects++; + return { kind: "download-fonts", completed: true }; + }, + }); + let id: string | undefined; + try { + const deadline = Date.now() + 60000; + const key = createControlRootOperationKey({ notAfter: deadline, nonce: randomBytes(16) }); + id = ( + await runtime.admit( + { kind: "download-fonts" }, + { operationKey: key, requestedDeadlineAt: deadline }, + ) + ).id; + await expect + .poll(async () => (await runtime.inspect(id!))?.status, { timeout: 15000 }) + .toBe("ok"); + expect(effects).toBe(1); + expect(lostReplies).toBe(-1); + await expect + .poll( + async () => + control + .openLeaseStore({ + directory: control.leaseStoreDirectory(root), + purpose: "test", + }) + .list(), + { timeout: 15000 }, + ) + .toEqual([]); + } finally { + await runtime.close(); + if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; + journal.close(); + control.closeControlInstallation(installation); + await sql.end(); + rmSync(root, { recursive: true, force: true }); + } + }); it("repairs a worker-completion/publication crash from the original signed completion without redispatch", async () => { const url = process.env.OPENMAPX_DATA_EXECUTION_TEST_DATABASE_URL; if (!url) throw Error("Owned PostgreSQL required"); diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index ce2587879..a591d0e51 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -121,6 +121,8 @@ async function releaseOwnLease( await journal.recordContainedTerminal(execution); await releaseArtifactTargetLease(lease); } +/** Agent answers that refuse a lease acquisition before it records anything. */ +const PRE_EFFECT_REFUSALS = new Set(["authorization", "validation", "busy", "conflict"]); export function dataExecutionTargetIds(input: DataExecutionInput): string[] { return ["transit-sync", "catalog.validate", "catalog.activate"].includes(input.kind) ? ["data-manager", "motis", "motis-feed-proxy", "motis-staging"] @@ -285,14 +287,22 @@ export function createDataExecutionRuntime(deps: { // Other executions claim their targets first: a held target, or an agent // that cannot answer, ends the execution as failed before any effect, with // its publication unchanged. + // A timeout or transport failure may come after the agent recorded the lease, + // so the same acquisition, which returns an existing lease, is repeated. let lease: ArtifactTargetLease | undefined; let refusal: unknown; - if (!leaf) + for (let attempt = 1; !leaf && !lease && !refusal; attempt += 1) try { lease = await acquireArtifactTargetLease({ serviceIds: targets, owner: authority }); } catch (error) { if (!(error instanceof TargetBusyError || error instanceof OpsClientError)) throw error; - refusal = error; + if ( + error instanceof TargetBusyError || + PRE_EFFECT_REFUSALS.has(error.errorClass) || + attempt >= 3 + ) + refusal = error; + else await new Promise((resolve) => setTimeout(resolve, 200 * attempt)); } await owner.begin(authority); const controller = new AbortController(); From 8a86280c9123aeb56244aa92d6ff53c170547855 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 00:59:27 +0100 Subject: [PATCH 23/42] fix(control): end a stuck lock reclaim at the wait and name every lock clear accepts A reclaimer that died while holding the reclaim guard made later acquisitions spin past their wait instead of refusing. A failed reclaim now ends in a busy error naming the guard once the wait is over, and a guard whose holder is gone is removed at once. An unreadable holder record is reclaimed only after it stayed the same file for the wait, or a second for a lock that does not wait. control clear --lock accepts every name a busy message prints, including a guard and an update receipt's lock, which is named by its path. --- .../docs/developer/build-artifacts-control.md | 19 +++++-- docs/docs/developer/cli-reference.md | 2 +- packages/cli/__tests__/cli-control.test.ts | 32 +++++++++++ packages/cli/src/commands/control.ts | 27 ++++++--- packages/cli/src/lib/system-update-receipt.ts | 7 ++- .../__tests__/recoverable-lock.test.ts | 40 ++++++++++++++ packages/core/src/services/index.ts | 1 - .../core/src/services/recoverable-lock.ts | 55 +++++++++++++------ 8 files changed, 148 insertions(+), 35 deletions(-) diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index ae1ec0e1f..b46345cf6 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -352,10 +352,15 @@ system update receipt each use one lock file. It records the holder's PID, boot PID namespace, container and process start time, and another writer reclaims it only when that holder is provably gone: the machine booted since, the holder shares this PID namespace and its process ended, or its container is no longer running. -Reclaimers take turns through a short guard file beside the lock. A live holder is -waited for briefly and then named in a busy error. The writer and lease locks are -never held across a Docker effect; the backup-store lock is held for a whole backup -effect, and a second backup waits up to a minute. +A holder record that cannot be read, because its holder died between creating and +writing the file, is reclaimed only after it stayed the same for the wait (one +second for a lock that does not wait). Reclaimers take turns through a short guard +file beside the lock. A live holder, or a guard that is still there when the wait +ends, is named in a busy error. The writer and lease locks are never held across a +Docker effect. Agent backups and prunes hold the backup-store lock for the whole +effect, and a second one waits up to a minute; the host CLI's `backup +create|restore` takes a lease on every target instead, so a second one is refused +as busy. Some dead holders cannot be proved gone, and their lock stays until `openmapx control clear --lock `: @@ -365,8 +370,10 @@ Some dead holders cannot be proved gone, and their lock stays until - any holder on macOS, which records no process start time, so a reused process ID counts as alive. -Check that the named process is gone first. The same command also removes a -reclaim guard that a dead reclaimer left beside the lock. +Check that the named process is gone first. The command accepts the short name, +the file name a busy message prints, and that name with `.reclaim` for a guard a +dead reclaimer left beside the lock; it removes both. An update receipt's lock is +named by its full path, because the receipt lives wherever the operator put it. ### Release pins and rollback diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index ae0dafee5..40229f3df 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -125,7 +125,7 @@ resolve and rollback is appended to `control-audit.jsonl` there and printed. | `control status` | Show the held leases with their owner, targets and children, the agent operations without a final outcome, and the holders of the writer, lease store and backup store locks. | | `control resolve --op --as committed\|failed` | Record the outcome of an operation whose outcome is unknown and release its lease. Prints what Docker reports for the held services first. Resolving a configuration maintenance also settles Data Manager's configuration fence, also after a clear. | | `control clear --op ` | Drop the lease of an operation whose holder is gone. A cleared configuration maintenance leaves the fence prepared until `control resolve`. | -| `control clear --lock ` | Remove a lock (`writer`, `leases` or `backups`, or the file name a busy message prints) whose holder cannot be proved gone, and a reclaim guard a dead reclaimer left beside it. Check that the holder has exited first: on macOS, and for a host CLI holder seen from a container, a dead holder is not reclaimed automatically. | +| `control clear --lock ` | Remove a lock (`writer`, `leases` or `backups`, or the name a busy message prints, including an update receipt's lock path) whose holder cannot be proved gone, and a reclaim guard a dead reclaimer left beside it. Check that the holder has exited first: on macOS, and for a host CLI holder seen from a container, a dead holder is not reclaimed automatically. | | `control rollback [--yes]` | Render the configuration generation applied before the current one again, with its release pins, and recreate the services in the self-update order. Refuses when the database has a migration newer than that release's recorded migration head. Rolling back also reverts the configuration made since: release selection, selected services, service configuration and secrets. Without `--yes` it only prints the plan. | ## `data` diff --git a/packages/cli/__tests__/cli-control.test.ts b/packages/cli/__tests__/cli-control.test.ts index 78bef588c..e85b6316a 100644 --- a/packages/cli/__tests__/cli-control.test.ts +++ b/packages/cli/__tests__/cli-control.test.ts @@ -137,6 +137,38 @@ describe("openmapx control", () => { expect(auditLog()).toMatchObject([{ action: "clear-lock", lock: "leases.lock", holder }]); }); + it("knows every lock name a busy message prints", async () => { + mkdirSync(control.leaseStoreDirectory(infraDir), { recursive: true }); + const receiptDir = join(root, "receipts"); + mkdirSync(receiptDir, { mode: 0o700 }); + const locks = [ + join(control.leaseStoreDirectory(infraDir), "leases.lock"), + services.configurationWriterLockPath(infraDir), + services.backupStoreLockPath(infraDir), + join(receiptDir, ".update.json.lock"), + ]; + for (const [index, path] of locks.entries()) { + mkdirSync(join(path, ".."), { recursive: true }); + const named = index === 3 ? { name: path } : {}; + const busy = await services.withRecoverableLock( + path, + { purpose: "live writer", ...named }, + () => + services + .withRecoverableLock(path, { purpose: "second", waitMs: 0, ...named }, () => "") + .catch((error: Error) => error.message), + ); + const name = String(busy).match(/control clear --lock ([^`]+)`/)?.[1]; + expect(name).toBeDefined(); + writeFileSync(path, ""); + writeFileSync(`${path}.reclaim`, ""); + await run("clear", "--lock", name!); + expect(services.readLockHolder(path)).toBeUndefined(); + await run("clear", "--lock", `${name}.reclaim`); + } + expect(auditLog()).toHaveLength(8); + }); + it("warns that clearing a maintenance leaves its fence, which resolve then settles", async () => { const agent: ExecutionRef = { ...owner(), diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index edcce89fa..82caaf1fa 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { closeSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs"; -import { basename, join } from "node:path"; +import { basename, isAbsolute, join } from "node:path"; import { type LeaseRecord, leaseStoreDirectory, @@ -34,6 +34,20 @@ const LOCKS = { backups: () => services.backupStoreLockPath(repoPaths().infraDir), } as const; +/** + * A busy message names an installation lock, or its reclaim guard, by its file; + * the short name works too. An update receipt's lock is named by its path. + */ +function lockPath(value: string): string { + const name = value.replace(/\.reclaim$/, ""); + const known = Object.entries(LOCKS).find( + ([short, path]) => name === short || name === basename(path()), + ); + if (known) return known[1](); + if (isAbsolute(name) && /^\..+\.lock$/.test(basename(name))) return name; + throw new Error(`Unknown lock ${value}`); +} + function store() { return openLeaseStore({ directory: leaseStoreDirectory(repoPaths().infraDir), @@ -186,14 +200,13 @@ export function registerControlCommands(program: Command): void { .command("clear") .description("Drop the lease of an operation, or a lock, whose holder is gone") .option("--op ", "Operation key that owns or appears in the lease") - .option("--lock ", `Lock to clear: ${Object.keys(LOCKS).join(", ")}`) + .option( + "--lock ", + `Lock to clear: ${Object.keys(LOCKS).join(", ")}, or the name a busy message prints`, + ) .action(async (options: { op?: string; lock?: string }) => { if (options.lock) { - // A busy message names the lock by its file; the short name works too. - const path = Object.entries(LOCKS) - .map(([name, path]) => ({ name, path: path() })) - .find(({ name, path }) => options.lock === name || options.lock === basename(path))?.path; - if (!path) throw new Error(`Unknown lock ${options.lock}`); + const path = lockPath(options.lock); const holder = services.clearRecoverableLock(path); audit({ action: "clear-lock", lock: options.lock, holder: holder ?? null }); completeCliCommand(0); diff --git a/packages/cli/src/lib/system-update-receipt.ts b/packages/cli/src/lib/system-update-receipt.ts index 4eedb7ccc..7c387e1b9 100644 --- a/packages/cli/src/lib/system-update-receipt.ts +++ b/packages/cli/src/lib/system-update-receipt.ts @@ -70,10 +70,11 @@ export class SystemUpdateReceiptStore { } /** One writer at a time; a second update command is refused at once while one runs. */ private async locked(run: () => T): Promise { - const dir = this.parent(); + const lock = join(this.parent(), `.${basename(this.path)}.lock`); + // The receipt is wherever the operator put it, so a busy message names the whole path. return services.withRecoverableLock( - join(dir, `.${basename(this.path)}.lock`), - { purpose: "system-update-receipt", waitMs: 0 }, + lock, + { purpose: "system-update-receipt", waitMs: 0, name: lock }, run, ); } diff --git a/packages/core/src/services/__tests__/recoverable-lock.test.ts b/packages/core/src/services/__tests__/recoverable-lock.test.ts index be1ef22e7..b2c511fce 100644 --- a/packages/core/src/services/__tests__/recoverable-lock.test.ts +++ b/packages/core/src/services/__tests__/recoverable-lock.test.ts @@ -199,4 +199,44 @@ describe("recoverable lock", () => { expect(readLockHolder(path)?.purpose).toBe("new writer"); expect(existsSync(`${path}.reclaim`)).toBe(false); }); + it("refuses with the guard named when a reclaimer died before it wrote its guard", async () => { + const path = lockPath(); + writeFileSync(path, JSON.stringify(foreign({ pid: exitedPid() }))); + writeFileSync(`${path}.reclaim`, ""); + const started = performance.now(); + const error = await acquireRecoverableLock(path, { + purpose: "next writer", + waitMs: 200, + signal: AbortSignal.timeout(3_000), + containerRunning: neverAsked, + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(LockBusyError); + expect((error as LockBusyError).message).toContain("control clear --lock store.lock.reclaim"); + expect(performance.now() - started).toBeLessThan(2_000); + clearRecoverableLock(path); + expect(readdirSync(join(path, ".."))).toEqual([]); + }); + + it("reclaims an unreadable holder only after it stayed the same for a second, with no wait", async () => { + const path = lockPath(); + writeFileSync(path, ""); + const started = performance.now(); + const release = await acquireRecoverableLock(path, { + purpose: "next writer", + waitMs: 0, + containerRunning: neverAsked, + }); + expect(performance.now() - started).toBeGreaterThanOrEqual(1_000); + release(); + }); + + it("does not reclaim a holder that finishes writing its record within the wait", async () => { + const path = lockPath(); + writeFileSync(path, ""); + setTimeout(() => writeFileSync(path, JSON.stringify(currentLockHolder("slow writer"))), 100); + await expect( + acquireRecoverableLock(path, { purpose: "next writer", waitMs: 300 }), + ).rejects.toBeInstanceOf(LockBusyError); + expect(readLockHolder(path)?.purpose).toBe("slow writer"); + }); }); diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index f19ee054a..58808a0be 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -240,7 +240,6 @@ export { } from "./platform-preview"; export { expandPresets, PRESETS, type PresetName, UnknownPresetError } from "./presets"; export { - acquireRecoverableLock, clearRecoverableLock, LockBusyError, readLockHolder, diff --git a/packages/core/src/services/recoverable-lock.ts b/packages/core/src/services/recoverable-lock.ts index cf8649b30..da4c97ed5 100644 --- a/packages/core/src/services/recoverable-lock.ts +++ b/packages/core/src/services/recoverable-lock.ts @@ -22,8 +22,8 @@ import { fsyncDirectory } from "../release-files"; /** * A lock file that records who holds it, so that a holder that died can be told - * apart from one that is still working. It is held only around short file - * updates, never across a Docker effect. + * apart from one that is still working. Most are held only around short file + * updates; the backup-store lock is held for a whole backup. * * Holders run in different PID namespaces: the agent and its fixed CLI in the * agent's container, the configuration helper in its own container and the @@ -53,11 +53,12 @@ export class LockBusyError extends Error { constructor( readonly path: string, readonly holder: LockHolder | undefined, + name = basename(path), ) { super( holder - ? `${basename(path)} is held by ${holder.purpose} (pid ${holder.pid}${holder.containerId ? ` in container ${holder.containerId.slice(0, 12)}` : ""}, since ${new Date(holder.acquiredAt).toISOString()}). If that process is gone, run \`openmapx control status\` and \`openmapx control clear --lock ${basename(path)}\`.` - : `${basename(path)} is busy`, + ? `${name} is held by ${holder.purpose} (pid ${holder.pid}${holder.containerId ? ` in container ${holder.containerId.slice(0, 12)}` : ""}, since ${new Date(holder.acquiredAt).toISOString()}). If that process is gone, run \`openmapx control status\` and \`openmapx control clear --lock ${name}\`.` + : `${name} is busy. If no process holds it, run \`openmapx control clear --lock ${name}\`.`, ); } } @@ -230,7 +231,8 @@ function moveAside(path: string, judged: LockFile): void { * file beside the lock, and the one holding it removes the lock only if it is * still the file it judged: a new holder's file is never removed, because * nobody else removes or replaces the lock while the guard is held. Resolves - * false when another reclaimer holds the guard; the caller looks again. + * true when the caller should look again at once, and false while another + * reclaimer holds the guard. */ async function reclaim( path: string, @@ -250,8 +252,10 @@ async function reclaim( if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; // A reclaimer that died while holding the guard leaves it behind. const other = readHolderFile(guard); - if (other?.holder && (await lockHolderGone(other.holder, self, containerRunning))) + if (other?.holder && (await lockHolderGone(other.holder, self, containerRunning))) { moveAside(guard, other); + return true; + } return false; } try { @@ -273,6 +277,8 @@ export interface RecoverableLockOptions { purpose: string; /** How long to wait for a live holder, 5 seconds by default. */ waitMs?: number; + /** The name busy messages give for `openmapx control clear --lock`; the file name by default. */ + name?: string; signal?: AbortSignal; containerRunning?: ContainerRunning; } @@ -281,7 +287,10 @@ export async function acquireRecoverableLock( path: string, options: RecoverableLockOptions, ): Promise<() => void> { - const deadline = performance.now() + (options.waitMs ?? 5_000); + const waitMs = options.waitMs ?? 5_000; + const deadline = performance.now() + waitMs; + // An unreadable holder record, and since when it has stayed exactly the same. + let unreadable: { file: LockFile; since: number } | undefined; for (;;) { options.signal?.throwIfAborted(); const self = currentLockHolder(options.purpose); @@ -322,19 +331,31 @@ export async function acquireRecoverableLock( } const existing = readHolderFile(path); if (!existing) continue; + // An unreadable holder record is only trusted as stale once it has stayed the + // same past the wait, or a second, so a holder still writing it is not displaced. + if (!existing.holder && !(unreadable && sameFile(existing, unreadable.file))) + unreadable = { file: existing, since: performance.now() }; + const pending = + !existing.holder && performance.now() - (unreadable?.since ?? 0) < (waitMs || 1_000); if ( - !existing.holder || - (await lockHolderGone(existing.holder, self, options.containerRunning)) + existing.holder + ? await lockHolderGone(existing.holder, self, options.containerRunning) + : !pending ) { - // An unreadable holder record is only trusted as stale once it has stayed - // unreadable past the wait, so a holder still writing it is not displaced. - if (existing.holder || performance.now() >= deadline) { - if (await reclaim(path, existing, self, options.containerRunning)) continue; - await delay(5, undefined, { signal: options.signal }); - continue; - } + if (await reclaim(path, existing, self, options.containerRunning)) continue; + // Another reclaimer holds the guard; one that died there is named once the wait is over. + const guard = `${path}.reclaim`; + if (performance.now() >= deadline) + throw new LockBusyError( + guard, + readLockHolder(guard), + `${options.name ?? basename(path)}.reclaim`, + ); + await delay(5, undefined, { signal: options.signal }); + continue; } - if (performance.now() >= deadline) throw new LockBusyError(path, existing.holder); + if (performance.now() >= deadline && !pending) + throw new LockBusyError(path, existing.holder, options.name); await delay(25, undefined, { signal: options.signal }); } } From f5703614223afb13bb3893cc41d5b1b867b7eda0 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:03:20 +0100 Subject: [PATCH 24/42] fix(control): roll back to the configuration applied with the previous release Rollback took its target from the last pointer move, so any render after an update, such as compose up or an admin save, made it target a generation with the same release and the pre-update configuration became unreachable. The target is now the newest retained generation whose pinned release differs from the applied one, and the pointer-move record is gone. Rollback refuses when no such generation is retained, always changes the release, and says that configuration made since is reverted and that services the target does not select keep running. A partial failure tells the operator to finish with compose up rather than roll back again, and a rollback that throws after its commit is still audited. --- .../retained-maintenance-job-postgres.test.ts | 2 +- apps/ops-agent/src/production-runtime.test.ts | 2 +- .../administration/platform-configuration.md | 2 +- .../docs/developer/build-artifacts-control.md | 24 +++--- docs/docs/developer/cli-reference.md | 14 ++-- docs/docs/install/upgrading.md | 51 ++++++++----- .../__tests__/cli-control-rollback.test.ts | 43 ++++++++--- packages/cli/src/commands/control.ts | 76 ++++++++++--------- packages/cli/src/lib/rollback.ts | 28 +++---- .../configuration-generation.test.ts | 33 ++++---- .../src/services/configuration-generation.ts | 56 +++++--------- packages/core/src/services/index.ts | 2 +- release-notes/release-pins-in-generation.md | 16 ++-- 13 files changed, 183 insertions(+), 166 deletions(-) diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index 404d0b67e..a61f461a8 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -882,7 +882,7 @@ dbDescribe("retained finite maintenance requests", () => { expect(services.readAppliedRelease(host.infraDir)).toBeNull(); expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe(pointer); expect( - services.readPreviousConfigurationGeneration( + services.readPreviousReleaseGeneration( host.infraDir, services.currentConfigurationGeneration(host.infraDir)!, ), diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index cfe4fde0e..27765b6bb 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -155,7 +155,7 @@ describe("assembled production service authority", () => { const cleared = services.currentConfigurationGeneration(host.infraDir); expect(cleared).not.toBe(pinned); expect(services.readAppliedRelease(host.infraDir)).toBeNull(); - expect(services.readPreviousConfigurationGeneration(host.infraDir, cleared!)).toBe(pinned); + expect(services.readPreviousReleaseGeneration(host.infraDir, cleared!)).toBe(pinned); expect(services.readGenerationRelease(host.infraDir, pinned!)?.manifest.release).toBe( "checked-pin", ); diff --git a/docs/docs/administration/platform-configuration.md b/docs/docs/administration/platform-configuration.md index 8a2d3eba4..3b9ba5d04 100644 --- a/docs/docs/administration/platform-configuration.md +++ b/docs/docs/administration/platform-configuration.md @@ -75,5 +75,5 @@ changed pin is refused. Clearing renders the applied configuration again as a ne generation without pins; running containers keep their images until they are next recreated, and the configured channel is unchanged. It does not select local images: a later start may pin the configured channel again. Finish any pending -data-manager update first. To return to the previous generation with its pins, use +data-manager update first. To return to the configuration applied with the previous release, use `openmapx control rollback` from the host (see [Upgrading](../install/upgrading.md#rolling-back)). diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index b46345cf6..d72efdd87 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -388,16 +388,20 @@ configuration again without pins. The agent caches the manifests it checked unde `state/ops-agent/releases/.json`; the newest is the release an update can apply. A manifest's `producer` and `buildMetadata` are display metadata. -The configuration directory records the last pointer move, which names the -generation applied before the current one. `openmapx control rollback` renders -that generation's configuration and release again as a new generation and -recreates the services in the self-update order. It refuses when the database's -newest applied migration (`created_at` in `drizzle.__drizzle_migrations`) is newer -than the target release's `databaseMigrationHead`, which CI writes into the release -manifest from the API's migration journal; the migrator alone would run an older -image against a newer schema. Rolling back also reverts the configuration made -since: the release selection, the selected services, service configuration and -secrets. +`openmapx control rollback` targets the newest retained generation, by the age of +its rendered files, whose pinned release digest differs from the applied one: the +configuration that was applied with the previous release. It renders that +generation's configuration and release again as a new generation and recreates the +services in the self-update order, so a rollback always changes the release and a +second one goes forward again. It refuses when no such generation is retained, and +when the database's newest applied migration (`created_at` in +`drizzle.__drizzle_migrations`) is newer than the target release's +`databaseMigrationHead`, which CI writes into the release manifest from the API's +migration journal; the migrator alone would run an older image against a newer +schema. Rolling back also reverts the configuration made since: the release +selection, the selected services, service configuration and secrets. Every +rollback, also one that fails after its commit, is appended to the control audit +file. ### Operator exit diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index 40229f3df..c30fe0fb4 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -120,13 +120,13 @@ The operator exit for the update system. Host only: it reads and writes the operations agent's lease store under `infra/docker/state/ops-agent`. Every clear, resolve and rollback is appended to `control-audit.jsonl` there and printed. -| Command | Description | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `control status` | Show the held leases with their owner, targets and children, the agent operations without a final outcome, and the holders of the writer, lease store and backup store locks. | -| `control resolve --op --as committed\|failed` | Record the outcome of an operation whose outcome is unknown and release its lease. Prints what Docker reports for the held services first. Resolving a configuration maintenance also settles Data Manager's configuration fence, also after a clear. | -| `control clear --op ` | Drop the lease of an operation whose holder is gone. A cleared configuration maintenance leaves the fence prepared until `control resolve`. | -| `control clear --lock ` | Remove a lock (`writer`, `leases` or `backups`, or the name a busy message prints, including an update receipt's lock path) whose holder cannot be proved gone, and a reclaim guard a dead reclaimer left beside it. Check that the holder has exited first: on macOS, and for a host CLI holder seen from a container, a dead holder is not reclaimed automatically. | -| `control rollback [--yes]` | Render the configuration generation applied before the current one again, with its release pins, and recreate the services in the self-update order. Refuses when the database has a migration newer than that release's recorded migration head. Rolling back also reverts the configuration made since: release selection, selected services, service configuration and secrets. Without `--yes` it only prints the plan. | +| Command | Description | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `control status` | Show the held leases with their owner, targets and children, the agent operations without a final outcome, and the holders of the writer, lease store and backup store locks. | +| `control resolve --op --as committed\|failed` | Record the outcome of an operation whose outcome is unknown and release its lease. Prints what Docker reports for the held services first. Resolving a configuration maintenance also settles Data Manager's configuration fence, also after a clear. | +| `control clear --op ` | Drop the lease of an operation whose holder is gone. A cleared configuration maintenance leaves the fence prepared until `control resolve`. | +| `control clear --lock ` | Remove a lock (`writer`, `leases` or `backups`, or the name a busy message prints, including an update receipt's lock path) whose holder cannot be proved gone, and a reclaim guard a dead reclaimer left beside it. Check that the holder has exited first: on macOS, and for a host CLI holder seen from a container, a dead holder is not reclaimed automatically. | +| `control rollback [--yes]` | Render the newest retained configuration that pins another release than the applied one again, with that release, and recreate the services in the self-update order. Check out that release's source first. Refuses when no such configuration is retained, or when the database has a migration newer than that release's recorded migration head. Rolling back reverts the configuration made since: release selection, selected services, service configuration and secrets; services it does not select keep running. After a partial failure, finish with `compose up`. Without `--yes` it only prints the plan. | ## `data` diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index 41e382190..88e9c510d 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -502,34 +502,47 @@ and [OSM data queries](../guides/osm-data-queries.md). ## Rolling back -`openmapx control rollback` returns to the configuration generation that was -applied before the current one, with the release it pins: +`openmapx control rollback` returns to the configuration that was applied with +the previous release, and to that release: ```bash pnpm openmapx control rollback # print the plan; nothing changes pnpm openmapx control rollback --yes # roll back ``` +The target is the newest retained configuration generation that pins another +release than the applied one, so a rollback always changes the release; renders +and configuration saves made since the update do not move it. The command refuses +when no such generation is retained: no release update was applied, or its +generation was pruned (sixteen generations are kept). + +Check out the target release's source first (step 2 of the upgrade, in reverse). +The render compares the checkout's built-in service files with the release's and +refuses on a difference, naming the revision to check out. + It renders that generation's configuration and release again as a new generation and recreates the services in the self-update order: every other service first, then PostGIS, Redis, the web app, the API, Traefik and the operations agent last. -A service that fails to recreate stops the rollback; the services after it are -left as they are. - -**Rolling back also reverts every configuration change made since that -generation:** the release selection, the selected services, service -configuration and secrets all return to the previous generation's. Re-apply any -change you still want afterwards. - -The command refuses when the release would change and the database has a newer -migration than the previous release ships. The migrator only applies migrations -newer than the newest applied one, so an older API would otherwise run against a -newer schema without complaint. CI records each release's newest migration in its -release manifest; a release published before that, or a previous configuration -without pins, cannot be checked and is refused too. In those cases, restore the -backup taken before the update. PostGIS must be running so that the command can -read the database's migrations. Only the latest switch between generations is -recorded, so a second rollback returns to the configuration you rolled back from. +A service that fails to recreate stops the rollback, and the services after it are +left as they are. The previous configuration is applied at that point: fix the +cause and run `pnpm openmapx compose up` to finish. Do not run rollback again, +which would return to the configuration you rolled back from. + +**Rolling back reverts every configuration change made since that +configuration was applied:** the release selection, the selected services, +service configuration and secrets all return to its values. Re-apply any change +you still want afterwards. Services that configuration does not select keep +running; stop them with `pnpm openmapx services stop `. + +The command refuses when the database has a newer migration than the previous +release ships. The migrator only applies migrations newer than the newest applied +one, so an older API would otherwise run against a newer schema without complaint. +CI records each release's newest migration in its release manifest; a release +published before that, or a previous configuration without pins, cannot be +checked and is refused too. In those cases, restore the backup taken before the +update. PostGIS must be running so that the command can read the database's +migrations. A rollback is itself a release change, so a second rollback returns to +the configuration you rolled back from. ## Reclaiming disk after an upgrade diff --git a/packages/cli/__tests__/cli-control-rollback.test.ts b/packages/cli/__tests__/cli-control-rollback.test.ts index ee06255ab..6d5896e30 100644 --- a/packages/cli/__tests__/cli-control-rollback.test.ts +++ b/packages/cli/__tests__/cli-control-rollback.test.ts @@ -111,16 +111,25 @@ async function rollback(...args: string[]) { } describe("rollback plan", () => { - it("names the applied generation and the one applied before it", async () => { + it("targets the configuration applied with the previous release", async () => { expect(() => planRollback(infraDir)).toThrow(/No configuration/); const first = await render(["alpha"], manifest("release-1", "1", 100)); - expect(() => planRollback(infraDir)).toThrow(/first one/); + expect(() => planRollback(infraDir)).toThrow(/No retained configuration pins another release/); const second = await render(["alpha", "app-api"], manifest("release-2", "2", 200)); + // A later render keeps release-2, so the target stays the first generation. + const third = await render(["alpha", "app-api"], manifest("release-2", "2", 200)); const plan = planRollback(infraDir); - expect(plan).toMatchObject({ currentRevisionId: second, targetRevisionId: first }); + expect(plan).toMatchObject({ currentRevisionId: third, targetRevisionId: first }); + expect(second).not.toBe(third); expect(plan.current?.manifest.release).toBe("release-2"); expect(plan.target?.manifest.release).toBe("release-1"); }); + + it("refuses when every retained configuration pins the applied release", async () => { + await render(["alpha"], manifest("release-1", "1", 100)); + await render(["alpha", "app-api"], manifest("release-1", "1", 100)); + expect(() => planRollback(infraDir)).toThrow(/none is retained/); + }); }); describe("the database check", () => { @@ -161,12 +170,6 @@ describe("the database check", () => { /records no database migration head/, ); }); - - it("needs no database when the release does not change", () => { - const same = manifest("release-2", "2", 200); - expect(() => assertDatabaseAllowsRollback(plan(same, same), 999)).not.toThrow(); - expect(() => assertDatabaseAllowsRollback(plan(null, null), 999)).not.toThrow(); - }); }); describe("openmapx control rollback", () => { @@ -178,7 +181,8 @@ describe("openmapx control rollback", () => { const output = printed.join("\n"); expect(output).toContain("(release release-2)"); expect(output).toContain("(release release-1)"); - expect(output).toContain("also reverts every configuration change made since then"); + expect(output).toContain("reverts every configuration change made since"); + expect(output).toContain("Services it does not select keep running"); expect(output).toContain("Run again with --yes"); expect(services.currentConfigurationGeneration(infraDir)).toBe(current); expect(dockerComposeStream).not.toHaveBeenCalled(); @@ -211,7 +215,8 @@ describe("openmapx control rollback", () => { await expect(rollback("--yes")).rejects.toThrow("exit:0"); const applied = services.currentConfigurationGeneration(infraDir); expect(applied).not.toBe(current); - expect(services.readPreviousConfigurationGeneration(infraDir, applied!)).toBe(current); + // A second rollback goes forward again. + expect(services.readPreviousReleaseGeneration(infraDir, applied!)).toBe(current); expect(services.readAppliedRelease(infraDir)?.manifest.release).toBe("release-1"); expect(services.readGenerationConfiguration(infraDir, applied!).selectedRoots).toEqual( services.readGenerationConfiguration(infraDir, previous).selectedRoots, @@ -247,8 +252,24 @@ describe("openmapx control rollback", () => { ["up", "-d", "--no-deps", "postgis"], ]); expect(printed.join("\n")).toContain("Recreating postgis failed (exit 3)"); + expect(printed.join("\n")).toContain("run `pnpm openmapx compose up` to finish"); expect( openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: "test" }).list(), ).toEqual([]); }); + it("records the rollback in the audit file when it fails after its commit", async () => { + await render(["alpha"], manifest("release-1", "1", 100)); + const current = await render(["alpha"], manifest("release-2", "2", 200)); + newestAppliedMigration(100); + vi.mocked(applyGeneratedHardlinksBeforeStart).mockRejectedValueOnce(new Error("links failed")); + await expect(rollback("--yes")).rejects.toThrow("links failed"); + expect(services.currentConfigurationGeneration(infraDir)).not.toBe(current); + const audit = readFileSync(join(infraDir, "state/ops-agent/control-audit.jsonl"), "utf8"); + expect(JSON.parse(audit.trim().split("\n").at(-1) ?? "")).toMatchObject({ + action: "rollback", + from: current, + exitCode: null, + error: "links failed", + }); + }); }); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index 82caaf1fa..f8de44bfe 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -284,7 +284,7 @@ export function registerControlCommands(program: Command): void { control .command("rollback") .description( - "Return to the configuration applied before the current one, with the release it pins", + "Return to the configuration that was applied with the previous release, and that release", ) .option("--yes", "Roll back; without it the plan is printed and nothing changes") .action(async (options: { yes?: boolean }) => { @@ -294,49 +294,57 @@ export function registerControlCommands(program: Command): void { `Applied configuration: ${plan.currentRevisionId} (release ${plan.current?.manifest.release ?? "none"})`, ); log.info( - `Previous configuration: ${plan.targetRevisionId} (release ${plan.target?.manifest.release ?? "none"})`, + `Configuration applied with the previous release: ${plan.targetRevisionId} (release ${plan.target?.manifest.release ?? "none"})`, ); log.warn( - "Rolling back also reverts every configuration change made since then: the release selection, the selected services, service configuration and secrets return to the previous configuration's.", + "Rolling back returns to that configuration and reverts every configuration change made since: the release selection, the selected services, service configuration and secrets. Services it does not select keep running; stop them with `pnpm openmapx services stop `.", ); - const sameRelease = plan.current?.manifestDigest === plan.target?.manifestDigest; - assertDatabaseAllowsRollback(plan, sameRelease ? null : await readNewestAppliedMigration()); + assertDatabaseAllowsRollback(plan, await readNewestAppliedMigration()); if (!options.yes) { log.info("Nothing was changed. Run again with --yes to roll back."); completeCliCommand(0); } const { renderComposeForRepo } = await import("./compose"); const target = services.readGenerationConfiguration(infraDir, plan.targetRevisionId); - const code = await leasedHostEffect("control.rollback", "all", async () => { - const rendered = await renderComposeForRepo({ - services: target.selectedRoots, - release: target.release, - configurationFrom: plan.targetRevisionId, - expectedCurrentRevisionId: plan.currentRevisionId, - }); - log.ok(`Rendered the previous configuration again → ${rendered.revisionId}`); - // Later renders keep the restored selection. - if (services.readDesiredSelection(infraDir).source !== "env") - services.writeServiceSelectionFile(infraDir, target.selectedRoots); - await applyGeneratedHardlinksBeforeStart(); - // The self-update order: the database before its clients, the agent last. - for (const id of services.selfUpdateServiceOrder(rendered.enabledServiceIds)) { - const exit = await dockerComposeStream(["up", "-d", "--no-deps", id]); - if (exit !== 0) { - log.err( - `Recreating ${id} failed (exit ${exit}); the services after it were not touched.`, - ); - return exit; + let code: number | null = null; + let failure: string | undefined; + try { + code = await leasedHostEffect("control.rollback", "all", async () => { + const rendered = await renderComposeForRepo({ + services: target.selectedRoots, + release: target.release, + configurationFrom: plan.targetRevisionId, + expectedCurrentRevisionId: plan.currentRevisionId, + }); + log.ok(`Rendered the previous configuration again → ${rendered.revisionId}`); + // Later renders keep the restored selection. + if (services.readDesiredSelection(infraDir).source !== "env") + services.writeServiceSelectionFile(infraDir, target.selectedRoots); + await applyGeneratedHardlinksBeforeStart(); + // The self-update order: the database before its clients, the agent last. + for (const id of services.selfUpdateServiceOrder(rendered.enabledServiceIds)) { + const exit = await dockerComposeStream(["up", "-d", "--no-deps", id]); + if (exit !== 0) { + log.err( + `Recreating ${id} failed (exit ${exit}); the services after it were not touched. The previous configuration is applied: fix the cause and run \`pnpm openmapx compose up\` to finish. Do not run rollback again, which would return to the configuration rolled back from.`, + ); + return exit; + } } - } - return 0; - }); - audit({ - action: "rollback", - from: plan.currentRevisionId, - to: plan.targetRevisionId, - exitCode: code, - }); + return 0; + }); + } catch (error) { + failure = error instanceof Error ? error.message : String(error); + throw error; + } finally { + audit({ + action: "rollback", + from: plan.currentRevisionId, + to: plan.targetRevisionId, + exitCode: code, + ...(failure ? { error: failure } : {}), + }); + } completeCliCommand(code); }); } diff --git a/packages/cli/src/lib/rollback.ts b/packages/cli/src/lib/rollback.ts index db2cb6111..75f423a05 100644 --- a/packages/cli/src/lib/rollback.ts +++ b/packages/cli/src/lib/rollback.ts @@ -8,27 +8,23 @@ export interface RollbackPlan { target: services.GenerationRelease | null; } -/** The applied generation and the one applied before it, with the releases they pin. */ +/** + * The applied generation and the newest retained one that pins another + * release: the configuration that was applied with the previous release. + */ export function planRollback(infraDir: string): RollbackPlan { const currentRevisionId = services.currentConfigurationGeneration(infraDir); if (!currentRevisionId) throw new Error("No configuration has been applied yet"); - const targetRevisionId = services.readPreviousConfigurationGeneration( - infraDir, - currentRevisionId, - ); + const current = services.readGenerationRelease(infraDir, currentRevisionId); + const targetRevisionId = services.readPreviousReleaseGeneration(infraDir, currentRevisionId); if (!targetRevisionId) - throw new Error("The applied configuration is the first one; there is nothing to roll back to"); - try { - services.configurationGenerationComposeFile(infraDir, targetRevisionId); - } catch (error) { - throw new Error(`The previous configuration ${targetRevisionId} is no longer retained`, { - cause: error, - }); - } + throw new Error( + `No retained configuration pins another release than the applied one (${current?.manifest.release ?? "no release"}). Rollback returns to the configuration applied with the previous release, and none is retained: no release update was applied, or its configuration was pruned.`, + ); return { currentRevisionId, targetRevisionId, - current: services.readGenerationRelease(infraDir, currentRevisionId), + current, target: services.readGenerationRelease(infraDir, targetRevisionId), }; } @@ -38,14 +34,12 @@ export function planRollback(infraDir: string): RollbackPlan { * the migrator only applies migrations newer than the newest applied one, so * an older API would start against the newer schema without complaint. The * newest applied migration must not be newer than the target release's - * recorded migration head. Rolling back within one release changes no image - * and needs no check. + * recorded migration head. */ export function assertDatabaseAllowsRollback( plan: RollbackPlan, newestAppliedMigration: number | null, ): void { - if (plan.current?.manifestDigest === plan.target?.manifestDigest) return; if (!plan.target) throw new Error( "The previous configuration pins no release, so the database migrations its images expect are unknown. Rollback refuses; restore a backup taken before the update instead.", diff --git a/packages/core/src/services/__tests__/configuration-generation.test.ts b/packages/core/src/services/__tests__/configuration-generation.test.ts index 02f9c0879..49e71c1fd 100644 --- a/packages/core/src/services/__tests__/configuration-generation.test.ts +++ b/packages/core/src/services/__tests__/configuration-generation.test.ts @@ -23,7 +23,7 @@ import { readAppliedServiceIds, readGenerationConfiguration, readGenerationRelease, - readPreviousConfigurationGeneration, + readPreviousReleaseGeneration, renderConfiguration, withConfigurationGenerationWriter, } from "../configuration-generation"; @@ -520,33 +520,32 @@ describe("release pins in the generation", () => { expect(compose(infraDir).services["data-manager"]?.environment).toBeUndefined(); }); - it("records the generation it replaced and what it was rendered from", async () => { + it("targets the newest generation applied with another release, and what it was rendered from", async () => { const { infraDir } = checkout(); const first = await commit(infraDir, manifest("release-1", "1")); - expect(readPreviousConfigurationGeneration(infraDir, first.revisionId)).toBeNull(); + expect(readPreviousReleaseGeneration(infraDir, first.revisionId)).toBeNull(); const second = await commit(infraDir, manifest("release-2", "2")); - expect(readPreviousConfigurationGeneration(infraDir, second.revisionId)).toBe(first.revisionId); + // Any later render keeps the release, so the target stays the first generation. + const rendered = await commit(infraDir); + expect(readPreviousReleaseGeneration(infraDir, second.revisionId)).toBe(first.revisionId); + expect(readPreviousReleaseGeneration(infraDir, rendered.revisionId)).toBe(first.revisionId); expect(readGenerationConfiguration(infraDir, first.revisionId)).toMatchObject({ domain: "maps.example.test", selectedRoots: ["app-api", "data-manager", "alpha"], release: manifest("release-1", "1"), }); expect(readGenerationRelease(infraDir, first.revisionId)?.manifest.release).toBe("release-1"); + // A rollback is a release change itself, so the next one goes forward again. + const rolledBack = await commit(infraDir, manifest("release-1", "1")); + expect(readPreviousReleaseGeneration(infraDir, rolledBack.revisionId)).toBe( + rendered.revisionId, + ); }); - it("keeps the recorded predecessor when an applied generation is committed again", async () => { + it("finds no target when every retained generation pins the applied release", async () => { const { infraDir } = checkout(); - const first = await commit(infraDir); - const options = { - infraDir, - services: pinned, - input: input({ selectedRoots: ["app-api", "data-manager", "alpha"] }), - revisionId: `cfg1_${"r".repeat(43)}`, - }; - await commitConfigurationGeneration(options); - await commitConfigurationGeneration(options); - expect(readPreviousConfigurationGeneration(infraDir, options.revisionId)).toBe( - first.revisionId, - ); + await commit(infraDir, manifest("release-1", "1")); + const again = await commit(infraDir); + expect(readPreviousReleaseGeneration(infraDir, again.revisionId)).toBeNull(); }); }); diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index a26d9d38e..3ddf7af7c 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -99,8 +99,6 @@ const MAX_GENERATION_BYTES = 2 * 1024 * 1024; const RESOLVED_CONFIG_FILE = "resolved-config.generated.json"; /** The release whose images the generation pins, as its canonical manifest. */ const RELEASE_MANIFEST_FILE = "release-manifest.json"; -/** Which generation the pointer last moved from, and to which. */ -const POINTER_MOVE_FILE = `${CONFIGURATION_DIRECTORY}/.last-pointer-move.json`; const REVISION = /^cfg1_[A-Za-z0-9_-]{43}$/; /** @@ -942,43 +940,24 @@ export function readAppliedRelease(infraDir: string): GenerationRelease | null { } /** - * The generation that was applied before `revisionId`, or null when that is - * unknown: the first generation, or a pointer move whose record was not - * written. Only the latest move is recorded, so only the applied generation - * has a predecessor. + * The newest retained generation, other than `revisionId`, that pins another + * release than it does, or null when none is retained: the configuration that + * was applied with the previous release. A generation's age is that of its + * rendered files, which are never written again. */ -export function readPreviousConfigurationGeneration( - infraDir: string, - revisionId: string, -): string | null { - const path = join(infraDir, POINTER_MOVE_FILE); - let move: { from?: unknown; to?: unknown }; - try { - move = JSON.parse(readOwnGenerationFile(path, [0o600n])); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; - throw error; - } - if (move.to !== revisionId) return null; - if (typeof move.from !== "string" || !REVISION.test(move.from)) throw new Error(UNSAFE); - return move.from; -} - -async function recordPointerMove(infraDir: string, from: string | null, to: string): Promise { - const path = join(infraDir, POINTER_MOVE_FILE); - if (!from) { - rmSync(path, { force: true }); - return; - } - const temporary = `${path}.tmp`; - rmSync(temporary, { force: true }); - try { - await writeDurable(temporary, `${JSON.stringify({ from, to })}\n`, 0o600); - renameSync(temporary, path); - await syncDirectory(dirname(path)); - } finally { - rmSync(temporary, { force: true }); - } +export function readPreviousReleaseGeneration(infraDir: string, revisionId: string): string | null { + const digest = readGenerationRelease(infraDir, revisionId)?.manifestDigest ?? null; + const generations = join(infraDir, CONFIGURATION_GENERATIONS); + const candidates = readdirSync(generations) + .filter((name) => REVISION.test(name) && name !== revisionId) + .map((name) => ({ + name, + writtenAt: lstatSync(join(generations, name, RESOLVED_CONFIG_FILE)).mtimeMs, + })) + .sort((a, b) => b.writtenAt - a.writtenAt || a.name.localeCompare(b.name)); + for (const { name } of candidates) + if ((readGenerationRelease(infraDir, name)?.manifestDigest ?? null) !== digest) return name; + return null; } /** @@ -1661,7 +1640,6 @@ async function commitOwnedConfigurationFiles( serviceFileReport: [], }; await syncDirectory(dirname(join(infraDir, CURRENT_CONFIGURATION))); - await recordPointerMove(infraDir, currentRevisionId, revisionId); } else { committed = { revisionId, diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 58808a0be..2dcc627e1 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -96,7 +96,7 @@ export { readAppliedTransitCredentialsRevision, readGenerationConfiguration, readGenerationRelease, - readPreviousConfigurationGeneration, + readPreviousReleaseGeneration, renderConfiguration, withConfigurationGenerationWriter, } from "./configuration-generation"; diff --git a/release-notes/release-pins-in-generation.md b/release-notes/release-pins-in-generation.md index 5040bb4f8..eb4a2f384 100644 --- a/release-notes/release-pins-in-generation.md +++ b/release-notes/release-pins-in-generation.md @@ -11,16 +11,16 @@ store's `current.json`, `latest.json` and `selections.json` and the release-stor lock are gone. `compose release` renders a new generation pinning the resolved release, `compose release --clear` and **Clear selected release pin** render the applied configuration again without pins, and every other render keeps the -applied pins. The agent keeps a cache of the manifests it checked, named by +applied pins; a start without pins pins the configured release. The agent keeps a cache of the manifests it checked, named by digest, under `state/ops-agent/releases/`. -`openmapx control rollback` returns to the configuration generation applied -before the current one, with its release pins, and recreates the services in the -self-update order. It refuses when the database has a migration newer than the -previous release ships; CI now records each release's newest migration in its -release manifest. Rolling back also reverts the configuration made since that -generation: the release selection, the selected services, service configuration -and secrets. +`openmapx control rollback` returns to the configuration that was applied with +the previous release, with that release's pins, and recreates the services in the +self-update order. It refuses when no such configuration is retained, and when the +database has a migration newer than the previous release ships; CI now records +each release's newest migration in its release manifest. Rolling back also +reverts the configuration made since: the release selection, the selected +services, service configuration and secrets. ## Who is affected From 83a2277b7f3680422ecd5774b3c8287b4e048bb4 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:07:33 +0100 Subject: [PATCH 25/42] fix(control): wait out a busy target outside the admission lock A busy refusal waited five seconds inside the agent's admission lock, so every other admission and job transition stalled meanwhile, including the progress of the operation holding the target. The agent now tries once inside the lock, waits for the holder outside it and tries again. The host CLI admits through the same rule, so a host effect started right after a data-manager job is not refused during its release. --- apps/ops-agent/src/server-foundation.test.ts | 9 ++- apps/ops-agent/src/server.ts | 72 ++++++++++++++------ packages/cli/src/lib/host-effect.ts | 4 +- 3 files changed, 61 insertions(+), 24 deletions(-) diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index a7ff706ca..44b6a09ef 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -918,11 +918,16 @@ describe("one admission path for target effects", () => { ["service.restart", "denied"], ["service.remove", "denied"], ]); - // Another target is free. + // Another target is free, and admits while a refusal still spends its grace. + const graced = submit({ kind: "service.restart", serviceId: "redis" }, "leasedGrace0000"); + await new Promise((resolve) => setTimeout(resolve, 200)); + const started = Date.now(); expect( (await submit({ kind: "service.restart", serviceId: "osrm" }, "leasedOther0000")) .statusCode, ).toBe(202); + expect(Date.now() - started).toBeLessThan(2_000); + expect((await graced).statusCode).toBe(409); release(); await eventually( async () => leaseStore.list(), @@ -938,7 +943,7 @@ describe("one admission path for target effects", () => { control.closeControlInstallation(installation); rmSync(root, { recursive: true, force: true }); } - }); + }, 40_000); it("drops the lease of an update whose configuration revision is no longer current", async () => { const control = await import("@openmapx/core/build-artifacts"); diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index eee996be1..822655eef 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -2,7 +2,6 @@ import { createHash, randomBytes as nodeRandomBytes, timingSafeEqual } from "nod import { abandonUnstartedLease, acquireArtifactTargetLease, - admitRootLease, assertArtifactTargetLeaseExecution, assertArtifactTargetLeaseForward, assertControlStorageAvailable, @@ -372,6 +371,26 @@ class RequestGuardError extends Error { } } +/** A held target at the first admission attempt; the admission is tried again after a grace. */ +class BusyAdmission extends Error { + constructor(readonly target: string) { + super("Target is busy"); + } +} + +/** Waits until no lease holds `target`, or until `until`. */ +async function waitForTargetRelease( + store: LeaseStore, + target: string, + until: number, + signal: AbortSignal, +): Promise { + while (Date.now() < until && store.list().some((lease) => lease.targets.includes(target))) { + await new Promise((resolve) => setTimeout(resolve, 50)); + signal.throwIfAborted(); + } +} + class ReplayCache { private readonly entries = new Map(); @@ -2595,7 +2614,10 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (policy.execution === "async") { const ownerKey = `${role}:${parsed.operationKey}`; - const admission = await withAdmissionLock(async () => { + // A held target is retried once, after a short grace spent outside the + // admission lock, so a holder that is finishing can release meanwhile. + let graceSpent = false; + const admit = async () => { const retained = jobsByKey.get(ownerKey); if (retained) { if (!matchesRetained(retained)) { @@ -2835,18 +2857,15 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif if (!admissionFailure && LEASED_KINDS.has(operation.kind) && options.leaseStore) try { admittedLease = ( - await admitRootLease( - localLeaseBackend(options.leaseStore), - { - owner: executionFor(job), - ownerKind: operation.kind, - targets: operation.kind.startsWith("valhalla.") ? ["valhalla"] : targets, - }, - claimController.signal, - ) + await localLeaseBackend(options.leaseStore).acquire({ + owner: executionFor(job), + ownerKind: operation.kind, + targets: operation.kind.startsWith("valhalla.") ? ["valhalla"] : targets, + }) ).leaseId; } catch (error) { if (!(error instanceof TargetBusyError)) admissionFailure = { error }; + else if (!graceSpent) throw new BusyAdmission(error.target); else { options.audit?.({ role, @@ -3020,15 +3039,28 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif } } return { type: "new", job } as const; - }).catch(async (error) => { - await claimAdmission?.rollback().catch(() => undefined); - active -= 1; - throw error instanceof RequestGuardError - ? error - : error instanceof TargetBusyError - ? new RequestGuardError("busy", 409, error.message) - : new RequestGuardError("runtime", 500); - }); + }; + const admission = await withAdmissionLock(admit) + .catch(async (error) => { + if (!(error instanceof BusyAdmission) || !options.leaseStore) throw error; + await waitForTargetRelease( + options.leaseStore, + error.target, + Math.min(Date.now() + 5000, normalized.deadlineAt), + claimController.signal, + ); + graceSpent = true; + return withAdmissionLock(admit); + }) + .catch(async (error) => { + await claimAdmission?.rollback().catch(() => undefined); + active -= 1; + throw error instanceof RequestGuardError + ? error + : error instanceof TargetBusyError + ? new RequestGuardError("busy", 409, error.message) + : new RequestGuardError("runtime", 500); + }); if (admission.type === "retained") { await claimAdmission?.rollback().catch(() => undefined); active -= 1; diff --git a/packages/cli/src/lib/host-effect.ts b/packages/cli/src/lib/host-effect.ts index 1b4bc2ab9..a60f04578 100644 --- a/packages/cli/src/lib/host-effect.ts +++ b/packages/cli/src/lib/host-effect.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes, randomUUID } from "node:crypto"; -import { agentStoreLeases, TargetBusyError } from "@openmapx/core/build-artifacts"; +import { admitRootLease, agentStoreLeases, TargetBusyError } from "@openmapx/core/build-artifacts"; import { canonicalJson, createControlRootOperationKey, @@ -66,7 +66,7 @@ export async function withHostTargets( }; let lease: Awaited>; try { - lease = await leases.acquire({ owner, ownerKind: input.kind, targets: body.targets }); + lease = await admitRootLease(leases, { owner, ownerKind: input.kind, targets: body.targets }); } catch (error) { if (error instanceof TargetBusyError) throw new HostTargetsBusyError(error.message); throw error; From 50068a0c0baf8aec3fbd1dd2c7ec347795eec9ff Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:09:20 +0100 Subject: [PATCH 26/42] fix(control): keep the first known outcome of a data-manager child and check the store's directory When the issuer had already settled a data-manager hardlink child from its status, data-manager's own report with a differently mapped outcome threw, aborted its release and repeated at every startup. The first known outcome now wins on both sides. The lease store refuses a directory that is not the agent's own, as the configuration writer lock does, so a CLI run as another user cannot leave a store the agent cannot read. --- apps/ops-agent/src/lease-operations.test.ts | 21 +++++++++---------- .../src/build-artifacts/lease-store.test.ts | 10 ++++++++- .../core/src/build-artifacts/lease-store.ts | 3 +++ .../core/src/build-artifacts/target-lease.ts | 12 ++++------- .../src/services/configuration-ownership.ts | 2 +- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/apps/ops-agent/src/lease-operations.test.ts b/apps/ops-agent/src/lease-operations.test.ts index 7375516a8..027327002 100644 --- a/apps/ops-agent/src/lease-operations.test.ts +++ b/apps/ops-agent/src/lease-operations.test.ts @@ -154,17 +154,16 @@ describe("lease operations for data-manager", () => { settlement, }); expect(store().get(leaseId)?.children[0]).toMatchObject({ state: "settled", settlement }); - // The issuer's own observation of the same result changes nothing. - await runLeaseOperation(installation, store(), "data-manager", { - kind: "lease.settleChild", - settlement: { ...settlement, terminalReceiptDigest: "d".repeat(64) }, - }); - expect(store().get(leaseId)?.children[0]?.settlement).toEqual(settlement); - await expect( - runLeaseOperation(installation, store(), "data-manager", { + // The first known outcome wins: the issuer may have mapped the same result + // from data-manager's status differently, and nothing changes. + for (const later of [ + { ...settlement, terminalReceiptDigest: "d".repeat(64) }, + { ...settlement, outcome: "contained" as const }, + ]) + await runLeaseOperation(installation, store(), "data-manager", { kind: "lease.settleChild", - settlement: { ...settlement, outcome: "contained" }, - }), - ).rejects.toThrow("Original settlement changed"); + settlement: later, + }); + expect(store().get(leaseId)?.children[0]?.settlement).toEqual(settlement); }); }); diff --git a/packages/core/src/build-artifacts/lease-store.test.ts b/packages/core/src/build-artifacts/lease-store.test.ts index a2e27deba..e19abbd20 100644 --- a/packages/core/src/build-artifacts/lease-store.test.ts +++ b/packages/core/src/build-artifacts/lease-store.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -162,4 +162,12 @@ describe("the agent's lease store", () => { expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); expect(leases.listSync().map((lease) => lease.targets)).toEqual([["shared"]]); }); + it("refuses a state directory that its group or others can reach", async () => { + const { infraDir, leases } = backend(); + mkdirSync(leaseStoreDirectory(infraDir), { recursive: true, mode: 0o755 }); + chmodSync(leaseStoreDirectory(infraDir), 0o755); + await expect( + leases.acquire({ owner: execution(), ownerKind: "service.stop", targets: ["redis"] }), + ).rejects.toThrow(/must be a directory owned by uid/); + }); }); diff --git a/packages/core/src/build-artifacts/lease-store.ts b/packages/core/src/build-artifacts/lease-store.ts index adebf6e46..1f5732974 100644 --- a/packages/core/src/build-artifacts/lease-store.ts +++ b/packages/core/src/build-artifacts/lease-store.ts @@ -13,6 +13,7 @@ import { leaseTargetsSchema, } from "../ops/lease-records"; import { atomicWrite } from "../release-files"; +import { assertOwnDirectory } from "../services/configuration-ownership"; import { type ContainerRunning, withRecoverableLock } from "../services/recoverable-lock"; /** @@ -106,6 +107,8 @@ export function openLeaseStore(options: { update(change: (leases: LeaseRecord[]) => T): Promise { const run = queue.then(() => { mkdirSync(options.directory, { recursive: true, mode: 0o700 }); + // Another user, such as a CLI run with sudo, would leave a store the agent cannot read. + assertOwnDirectory(options.directory); return withRecoverableLock( lock, { purpose: options.purpose, containerRunning: options.containerRunning }, diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index 1ab199674..fe169c8d8 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -277,8 +277,9 @@ export const leaseRules = { }, /** * Data-manager reports the outcome of a child it ran under another role's - * lease. The child is found by its identity; an outcome already recorded with - * the same result is kept, and a released lease leaves nothing to settle. + * lease. The child is found by its identity; the first known outcome + * recorded, by data-manager or by the issuer from its status, is kept, and a + * released lease leaves nothing to settle. */ settleChild(leases: LeaseRecord[], settlement: LeaseSettlement) { for (const lease of leases) { @@ -287,12 +288,7 @@ export const leaseRules = { value.recipient === "data-manager" && sameExecution(value.execution, settlement.identity), ); if (!child) continue; - if ( - child.state === "settled" && - child.settlement?.outcome === settlement.outcome && - child.settlement.publication === settlement.publication - ) - return lease; + if (settledKnown(child)) return lease; return leaseRules.settle(leases, { leaseId: lease.leaseId, key: child.key, settlement }); } return undefined; diff --git a/packages/core/src/services/configuration-ownership.ts b/packages/core/src/services/configuration-ownership.ts index 68ae82348..fc198a7c1 100644 --- a/packages/core/src/services/configuration-ownership.ts +++ b/packages/core/src/services/configuration-ownership.ts @@ -22,7 +22,7 @@ function owner(): number { * One of the agent's own directories: a real directory of this user that * nobody else can reach. Refused, never repaired. */ -function assertOwnDirectory(path: string): void { +export function assertOwnDirectory(path: string): void { const stats = lstatSync(path); if ( !stats.isDirectory() || From b3e307c15e36d205c8405865a15b7b37be69535b Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:10:33 +0100 Subject: [PATCH 27/42] fix(control): leave data-manager's leases to data-manager and hide ended operations control resolve of a data-manager-owned lease removed the store row but left data-manager's job unknown, so its fence kept refusing maintenance and its recovery kept settling a lease that was gone. Resolve now refuses such a lease and points to control clear as the explicit release. control status no longer lists agent operations the audit file records as resolved or cleared. --- .../administration/services-administration.md | 4 +- .../docs/developer/build-artifacts-control.md | 6 +- docs/docs/developer/cli-reference.md | 4 +- packages/cli/__tests__/cli-control.test.ts | 62 ++++++++++++++++++ packages/cli/src/commands/control.ts | 64 ++++++++++++------- 5 files changed, 114 insertions(+), 26 deletions(-) diff --git a/docs/docs/administration/services-administration.md b/docs/docs/administration/services-administration.md index 440a0fc41..2e0f9da6b 100644 --- a/docs/docs/administration/services-administration.md +++ b/docs/docs/administration/services-administration.md @@ -503,7 +503,9 @@ host: prints what Docker reports for the held services), then record the outcome with `pnpm openmapx control resolve --op --as committed|failed`. This releases the lease; for a configuration maintenance it also settles Data - Manager's configuration fence. + Manager's configuration fence. Data Manager settles its own jobs when it + starts and during recovery, so `control resolve` refuses a lease Data Manager + owns; `control clear` is the explicit release for one it cannot settle. 3. For a lease whose holder is gone, `pnpm openmapx control clear --op ` drops it. A cleared configuration maintenance leaves the fence prepared until you resolve the same operation. diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index d72efdd87..fff84afe8 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -413,7 +413,11 @@ host-only, print what Docker reports for the held targets first, and are appende to `state/ops-agent/control-audit.jsonl` and printed. Clearing a configuration maintenance leaves the data-manager's configuration fence prepared; `control resolve` of the same operation settles it through the CLI's data-manager token, -also after a clear. `control clear --lock ` is described under [Locks](#locks). +also after a clear. `control resolve` refuses a lease data-manager owns: +data-manager settles its own jobs at startup and in recovery, and `control clear` +is the explicit release. `control status` leaves out agent operations the audit +file records as resolved or cleared; they stay in the agent's journal until it is +pruned. `control clear --lock ` is described under [Locks](#locks). ### Authority keys and the control prune diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index c30fe0fb4..0843e0bdd 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -122,8 +122,8 @@ resolve and rollback is appended to `control-audit.jsonl` there and printed. | Command | Description | | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `control status` | Show the held leases with their owner, targets and children, the agent operations without a final outcome, and the holders of the writer, lease store and backup store locks. | -| `control resolve --op --as committed\|failed` | Record the outcome of an operation whose outcome is unknown and release its lease. Prints what Docker reports for the held services first. Resolving a configuration maintenance also settles Data Manager's configuration fence, also after a clear. | +| `control status` | Show the held leases with their owner, targets and children, the agent operations without a final outcome (except those an operator resolved or cleared), and the holders of the writer, lease store and backup store locks. | +| `control resolve --op --as committed\|failed` | Record the outcome of an operation whose outcome is unknown and release its lease. Prints what Docker reports for the held services first. Resolving a configuration maintenance also settles Data Manager's configuration fence, also after a clear. Refuses a lease Data Manager owns: Data Manager settles its own jobs when it starts and during recovery, and `control clear` is the explicit release. | | `control clear --op ` | Drop the lease of an operation whose holder is gone. A cleared configuration maintenance leaves the fence prepared until `control resolve`. | | `control clear --lock ` | Remove a lock (`writer`, `leases` or `backups`, or the name a busy message prints, including an update receipt's lock path) whose holder cannot be proved gone, and a reclaim guard a dead reclaimer left beside it. Check that the holder has exited first: on macOS, and for a host CLI holder seen from a container, a dead holder is not reclaimed automatically. | | `control rollback [--yes]` | Render the newest retained configuration that pins another release than the applied one again, with that release, and recreate the services in the self-update order. Check out that release's source first. Refuses when no such configuration is retained, or when the database has a migration newer than that release's recorded migration head. Rolling back reverts the configuration made since: release selection, selected services, service configuration and secrets; services it does not select keep running. After a partial failure, finish with `compose up`. Without `--yes` it only prints the plan. | diff --git a/packages/cli/__tests__/cli-control.test.ts b/packages/cli/__tests__/cli-control.test.ts index e85b6316a..d2e9fd413 100644 --- a/packages/cli/__tests__/cli-control.test.ts +++ b/packages/cli/__tests__/cli-control.test.ts @@ -97,6 +97,68 @@ describe("openmapx control", () => { expect(text).toContain("targets: alpha"); }); + it("lists agent operations without a final outcome until an operator ends them", async () => { + const journal = services.agentStatePaths(infraDir).journal; + mkdirSync(join(journal, ".."), { recursive: true }); + const job = (operationKey: string, state: string, errorClass?: string) => ({ + operationKey, + operation: { kind: "service.restart" }, + state, + ...(errorClass ? { errorClass } : {}), + }); + writeFileSync( + journal, + JSON.stringify({ + jobs: [ + job("opk1_running", "running"), + job("opk1_unknown", "failed", "recovery_required"), + job("opk1_finished", "succeeded"), + ], + }), + ); + await run("status"); + let text = printed.join("\n"); + expect(text).toContain("Agent operations without a final outcome:"); + expect(text).toContain("service.restart opk1_running (running)"); + expect(text).toContain("service.restart opk1_unknown (recovery_required)"); + expect(text).not.toContain("opk1_finished"); + writeFileSync( + join(control.leaseStoreDirectory(infraDir), "control-audit.jsonl"), + `${JSON.stringify({ action: "resolve", operationKey: "opk1_unknown", as: "failed" })}\n`, + ); + printed = []; + await run("status"); + text = printed.join("\n"); + expect(text).toContain("opk1_running"); + expect(text).not.toContain("opk1_unknown"); + }); + + it("leaves a data-manager job's lease to data-manager and points to clear", async () => { + const execution = { + ...owner(), + executor: "data-manager" as const, + principal: "data-manager" as const, + }; + await control + .localLeaseBackend(store()) + .acquire({ owner: execution, ownerKind: "download-fonts", targets: ["alpha"] }); + const program = new Command().exitOverride(); + registerControlCommands(program); + await expect( + program.parseAsync([ + "node", + "openmapx", + "control", + "resolve", + "--op", + execution.operationKey, + "--as", + "failed", + ]), + ).rejects.toThrow(/Data-manager settles its own jobs.*control clear --op/); + expect(store().list()).toHaveLength(1); + }); + it("clears the lease of an operation, prints what runs and audits it", async () => { const execution = await held(); await run("clear", "--op", execution.operationKey); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index f8de44bfe..7cfc35398 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -104,32 +104,40 @@ function findLease(operationKey: string): LeaseRecord { return lease; } -/** - * A maintenance root an operator cleared, from the audit file: its fence can - * still be settled with `control resolve`. - */ -function clearedMaintenance(operationKey: string): LeaseRecord | undefined { - let lines: string[]; +interface AuditEntry { + action?: string; + operationKey?: string; + lease?: LeaseRecord | null; +} + +function readAudit(): AuditEntry[] { try { - lines = readFileSync( + return readFileSync( join(leaseStoreDirectory(repoPaths().infraDir), "control-audit.jsonl"), "utf8", - ).split("\n"); + ) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as AuditEntry); } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; } - for (const line of lines.reverse()) { - if (!line) continue; - const entry = JSON.parse(line) as { action?: string; lease?: LeaseRecord | null }; - if ( - entry.action === "clear" && - entry.lease?.owner.operationKey === operationKey && - entry.lease.ownerKind === "configurationMaintenance.prepare" - ) - return entry.lease; - } - return undefined; +} + +/** + * A maintenance root an operator cleared, from the audit file: its fence can + * still be settled with `control resolve`. + */ +function clearedMaintenance(operationKey: string): LeaseRecord | undefined { + return readAudit() + .reverse() + .find( + (entry) => + entry.action === "clear" && + entry.lease?.owner.operationKey === operationKey && + entry.lease.ownerKind === "configurationMaintenance.prepare", + )?.lease as LeaseRecord | undefined; } function hasFence(lease: LeaseRecord): boolean { @@ -176,10 +184,18 @@ export function registerControlCommands(program: Command): void { const leases = store().list(); log.info(leases.length ? "Held leases:" : "No leases are held."); for (const lease of leases) for (const line of describe(lease)) log.info(line); + // An operation an operator resolved or cleared stays in the agent's journal + // until it is pruned; the audit file records that it was ended. + const ended = new Set( + readAudit() + .filter((entry) => entry.action === "resolve" || entry.action === "clear") + .flatMap((entry) => [entry.operationKey, entry.lease?.owner.operationKey]), + ); const open = readJournal().filter( (job) => - ["queued", "running", "termination_pending"].includes(job.state) || - job.errorClass === "recovery_required", + !ended.has(job.operationKey) && + (["queued", "running", "termination_pending"].includes(job.state) || + job.errorClass === "recovery_required"), ); if (open.length) { log.info("Agent operations without a final outcome:"); @@ -243,6 +259,10 @@ export function registerControlCommands(program: Command): void { const lease = cleared ?? findLease(options.op); if (lease.owner.operationKey !== options.op) throw new Error("Resolve the operation that owns the lease"); + if (lease.owner.executor === "data-manager") + throw new Error( + `Data-manager settles its own jobs when it starts and during recovery; resolving its lease here would leave its job unsettled. If data-manager cannot settle it, release the targets with \`openmapx control clear --op ${options.op}\`.`, + ); for (const line of describe(lease)) log.info(line); await printLive(lease.targets); // A maintenance root also holds the data-manager fence; the same idempotent From 42d3193d217f8da35c2a760a1d80c11b392c42b5 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:11:47 +0100 Subject: [PATCH 28/42] chore(release): drop the unused compose release asset and stale overlay text Nothing consumes the docker-compose.release.yml asset of a product release any more; release-manifest.json beside it carries the same data. Its INSTALL text, and comments and notes that still described the removed release overlay and release-store lock, now describe the pins in the generation and control rollback. The lease docs now say which failures release a lease in the agent and on the host, and that a busy target is waited for briefly. --- .../configuration-maintenance-postgres.test.ts | 6 +++--- .../src/configuration-release-downloads.ts | 4 ++-- apps/ops-agent/src/owned-services.ts | 4 ++-- docs/docs/developer/build-artifacts-control.md | 12 ++++++++---- docs/docs/install/upgrading.md | 10 +++++----- .../release-community-and-integration-trust.md | 2 +- scripts/product-release.mjs | 15 ++------------- scripts/product-release.test.ts | 13 ++----------- 8 files changed, 25 insertions(+), 41 deletions(-) diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index 1bacbb289..740a9af54 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -179,7 +179,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( let dm: Awaited> | undefined; let actualPosts: Map; beforeEach(async (context) => { - // Synthetic example/* images have no published release overlay. + // Synthetic example/* images have no published release. vi.stubEnv("OPENMAPX_RELEASE", "local"); fixture = await createConfigurationPostgresFixture(); state.fixture = fixture; @@ -2303,7 +2303,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); }, 60000); it("uses one ownership-only maintenance decision before a durable nonempty coordinator credential continuation", async () => { - // Synthetic example/* images intentionally have no published release overlay. + // Synthetic example/* images intentionally have no published release. vi.stubEnv("OPENMAPX_RELEASE", "local"); vi.stubEnv("DOMAIN", "example.test"); dm = await ownedDataManager({ @@ -4980,7 +4980,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( it.each(["journal"])( "settles actual durable nonempty unused maintenance through the original API control and retains its canonical delivery (%s)", async () => { - // This synthetic stack uses local example images and selects no product release overlay. + // This synthetic stack uses local example images and pins no product release. vi.stubEnv("OPENMAPX_RELEASE", "local"); dm = await ownedDataManager({ sql: fixture.sql, diff --git a/apps/ops-agent/src/configuration-release-downloads.ts b/apps/ops-agent/src/configuration-release-downloads.ts index df29de361..40f3a258f 100644 --- a/apps/ops-agent/src/configuration-release-downloads.ts +++ b/apps/ops-agent/src/configuration-release-downloads.ts @@ -115,8 +115,8 @@ export async function inspectConfigurationReleaseDownloads(input: { images = await Promise.all(tasks); } catch (error) { controller.abort(Error("Release metadata inspection canceled")); - // The caller owns the release-store lock until every contained process - // has finished, including siblings still shutting down after cancellation. + // Every contained process finishes before the error is raised, including + // siblings still shutting down after cancellation. await Promise.allSettled(tasks); throw error; } diff --git a/apps/ops-agent/src/owned-services.ts b/apps/ops-agent/src/owned-services.ts index f8d77eedf..fc0b53cd2 100644 --- a/apps/ops-agent/src/owned-services.ts +++ b/apps/ops-agent/src/owned-services.ts @@ -553,8 +553,8 @@ export function createOwnedServiceResolver(options: { configurations.set(key, configuration); } // Deselection removes Compose membership, but the registered manifest - // remains authority for its effective image. Release overlays still win - // when they provide an explicit image for that service. + // remains authority for its effective image. A release pin in the + // generation wins when it names an image for that service. const registered = registry.get(serviceId); const reference = configuration[serviceId]?.image ?? diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index fff84afe8..bd893fdbc 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -339,10 +339,14 @@ removal, backup create and restore. Pull, build, exec and probes take none. A he target refuses the operation before anything is recorded, with a busy error that names the holder's executor, operation key, kind and state and points to `openmapx control status`. On the host, `services stop|restart`, `compose up|down` -and `backup create|restore` take the same leases. A Docker command that reports a -failure (a non-zero exit, or an error from Docker) records a contained, known -outcome and releases the lease; a crash, timeout, killed process or lost daemon -connection leaves the outcome unknown and the lease held. +and `backup create|restore` take the same leases. In the agent, only a Docker +command that ran to its own end and reported a failure with an exit code, while +the daemon still answered, records a contained, known outcome and releases the +lease; any other error, including every other failure of an agent backup, leaves +the outcome unknown and the lease held. On the host, an exit code from 1 to 127, +or an error that is not a signal or timeout while the daemon still answers, +releases the lease; a crash, timeout, killed process or lost daemon connection +leaves the outcome unknown and the lease held. ### Locks diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index 88e9c510d..76526e1b3 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -304,11 +304,11 @@ stays interrupted until the operation's deadline plus 30 seconds has passed. Every command that changes containers (stop, restart, start, recreate, update, remove, `compose up` and `compose down`, backup create and restore) first claims its services in the operations agent's lease store. While one holds a service, -another refuses at once with a message that names the holder's operation key and -points to `openmapx control status`. A failure Docker reports releases the -services; a crash, timeout, killed command or lost connection to the Docker -daemon leaves the outcome unknown, and the services stay held until you resolve -it. On the host: +another waits up to five seconds for it and then refuses with a message that +names the holder's operation key and points to `openmapx control status`. A +failure Docker reports with an exit code releases the services; a crash, timeout, +killed command or lost connection to the Docker daemon leaves the outcome +unknown, and the services stay held until you resolve it. On the host: ```bash pnpm openmapx control status # held leases, unfinished operations, lock holders diff --git a/release-notes/release-community-and-integration-trust.md b/release-notes/release-community-and-integration-trust.md index 587b2b287..dca2d1c83 100644 --- a/release-notes/release-community-and-integration-trust.md +++ b/release-notes/release-community-and-integration-trust.md @@ -10,7 +10,7 @@ permissions. A checkout written under a umask of `002`, as `git clone` and requests without any `chmod`. The checkout must still be writable only by the operator's account, because services mount configuration files from it, `services build` uses its build contexts, and `infra/docker/.env` and the -release overlay live in it. A private group per user, the Debian and Ubuntu +applied configuration generations live in it. A private group per user, the Debian and Ubuntu default, meets this with a umask of `002`; where the group has other members, clone and pull with umask `022`. diff --git a/scripts/product-release.mjs b/scripts/product-release.mjs index c34ef235c..8fcaeb199 100644 --- a/scripts/product-release.mjs +++ b/scripts/product-release.mjs @@ -14,20 +14,10 @@ import { validateReadiness, } from "./product-release-plan.mjs"; -const fixedAssets = [ - "candidate.json", - "release-manifest.json", - "docker-compose.release.yml", - "INSTALL.md", - "notes.md", -]; +const fixedAssets = ["candidate.json", "release-manifest.json", "INSTALL.md", "notes.md"]; function main(ports) { assert(ports.ref === "refs/heads/main", "Named release orchestration must run on main"); } -function overlay(manifest) { - const images = manifest.images; - return `# Exact rehearsal selection; use the matching source checkout.\nx-openmapx-release: ${canonicalCandidate({ release: manifest.release, images, product: manifest.product })}\nservices:\n app-api:\n image: ${images.api}\n app-web:\n image: ${images.web}\n data-manager:\n image: ${images["data-manager"]}\n environment:\n OPENMAPX_TRANSITOUS_TOOLS_IMAGE: ${images["transitous-tools"]}\n ops-agent:\n image: ${images["ops-agent"]}\n transitous-runner:\n image: ${images["transitous-runner"]}\n`; -} function assetsFor(candidate, digest, notes) { const envelope = { candidate, @@ -37,9 +27,8 @@ function assetsFor(candidate, digest, notes) { const assets = { "candidate.json": `${canonicalCandidate(envelope)}\n`, "release-manifest.json": `${canonicalCandidate(candidate.manifest)}\n`, - "docker-compose.release.yml": overlay(candidate.manifest), "notes.md": notes, - "INSTALL.md": `# Candidate ${candidate.version}\n\nNot published until the owner approves the protected publishing job.\n\nCheck out ${candidate.repository} at exact source ${candidate.sourceRevision}. Use the installation instructions at that source. Set OPENMAPX_RELEASE=${digest} and OPENMAPX_RELEASE_MANIFEST_REPOSITORY=ghcr.io/${candidate.repository.toLowerCase().split("/")[0]}/release-manifest, then run pnpm openmapx compose release for a clean installation. For an existing installation, use compose release --check and the authenticated system update workflow in the upgrading guide; do not replace its running Data Manager pin with bare compose release. Both resolve the exact staged candidate; no preview fallback is permitted. The accompanying overlay is an auditable six-image summary; use the CLI-generated selection for normal host operations.\n\nRehearse clean installation and ${candidate.manifest.product.upgradeOrigins.join(", ")} upgrade, selected service health and critical flows. Image rollback does not reverse database migrations. Record actual recovery limits and evidence in readiness.json; no backups are created automatically by these instructions.\n`, + "INSTALL.md": `# Candidate ${candidate.version}\n\nNot published until the owner approves the protected publishing job.\n\nCheck out ${candidate.repository} at exact source ${candidate.sourceRevision}. Use the installation instructions at that source. Set OPENMAPX_RELEASE=${digest} and OPENMAPX_RELEASE_MANIFEST_REPOSITORY=ghcr.io/${candidate.repository.toLowerCase().split("/")[0]}/release-manifest, then run pnpm openmapx compose release for a clean installation. For an existing installation, use compose release --check and the authenticated system update workflow in the upgrading guide; do not replace its running Data Manager pin with bare compose release. Both resolve the exact staged candidate; no preview fallback is permitted.\n\nRehearse clean installation and ${candidate.manifest.product.upgradeOrigins.join(", ")} upgrade, selected service health and critical flows. openmapx control rollback refuses to return to a release older than the database's newest migration; restore a backup for that. Record actual recovery limits and evidence in readiness.json; no backups are created automatically by these instructions.\n`, }; assets.SHA256SUMS = fixedAssets.map((name) => `${sha256(assets[name])} ${name}\n`).join(""); return assets; diff --git a/scripts/product-release.test.ts b/scripts/product-release.test.ts index efc9a24ef..54122e57d 100644 --- a/scripts/product-release.test.ts +++ b/scripts/product-release.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it } from "vitest"; -import { parse } from "yaml"; import { fixture } from "./product-release.fixtures"; import { inspectPublication, @@ -278,22 +277,14 @@ it.each([ expect(t.state.preview).toBeNull(); }); -it("provides an auditable usable overlay and exact evidence links before publishing approval", async () => { +it("provides exact evidence links before publishing approval", async () => { const t = setup(); const review = await t.stage(); - const summary = parse(t.state.draft.assets["docker-compose.release.yml"]) as { - "x-openmapx-release": { release: string; images: Record }; - services: Record; - }; - expect(summary["x-openmapx-release"]).toMatchObject({ - release: t.candidate.version, - images: t.candidate.manifest.images, - }); - expect(summary.services["app-api"]?.image).toBe(t.candidate.manifest.images.api); expect(review.readiness.checks.cleanInstall.evidence).toBe( "https://example.invalid/synthetic/cleanInstall", ); expect(review.assets["readiness.json"]).toBe(sha256(t.state.draft.assets["readiness.json"])); + expect(Object.keys(t.state.draft.assets)).not.toContain("docker-compose.release.yml"); }); it("reconciles a lost registry acknowledgement after the final preview write", async () => { From 5719b2539cc4895f1ecd36d278f34c3a5cb0b3b3 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:22:14 +0100 Subject: [PATCH 29/42] refactor(control): drop test-only lease exports and the crash engine's exec support heldTargets moves to a core test fixture, tests verify a root release with the synchronous local check, and the lease-store busy alias gives way to the lock's own error. The unused lease reason field, the remember helper and the separate admitLease step, folded into the lease rules' acquire, are gone. The crash-point engine no longer emulates compose exec and ps --services, which only the removed exec lease transport used. --- ...configuration-maintenance-postgres.test.ts | 10 +-- .../coordinator-lifecycle-postgres.test.ts | 8 +-- .../services/__tests__/crash-points/driver.ts | 30 --------- .../__tests__/crash-points/support/engine.ts | 35 ---------- .../data-configuration-fence-postgres.test.ts | 14 ++-- ...orm-configuration-runtime-postgres.test.ts | 14 ++-- .../artifact-transfer/import-runtime.test.ts | 12 ++-- .../src/configuration-handoff-control.test.ts | 8 +-- .../src/configuration-handoff-control.ts | 6 +- .../src/control-terminal-restart.test.ts | 3 +- .../src/data-workflow-runtime.test.ts | 5 +- apps/ops-agent/src/integration-store.test.ts | 2 +- .../src/traffic-target-ownership.test.ts | 9 +-- .../cli/__tests__/cli-host-effect.test.ts | 3 +- .../__tests__/cli-service-lifecycle.test.ts | 3 +- .../__fixtures__/held-targets.ts | 7 ++ .../configuration-pre-effect.test.ts | 6 +- .../src/build-artifacts/lease-store.test.ts | 2 +- .../core/src/build-artifacts/lease-store.ts | 40 ------------ .../publication-native.test.ts | 6 +- .../core/src/build-artifacts/target-lease.ts | 64 +++++++++---------- packages/core/src/catalog-pins/store.test.ts | 2 +- packages/core/src/ops/lease-records.ts | 1 - .../catalog-workflow-postgres.test.ts | 9 +-- .../src/jobs/execution/fixed-cli.test.ts | 3 +- .../src/jobs/execution/full-update.test.ts | 5 +- .../src/jobs/execution/hardlinks.test.ts | 3 +- .../legacy-admission-postgres.test.ts | 3 +- .../src/jobs/execution/ops-children.test.ts | 7 +- .../src/jobs/execution/runtime.test.ts | 9 +-- 30 files changed, 118 insertions(+), 211 deletions(-) create mode 100644 packages/core/src/build-artifacts/__fixtures__/held-targets.ts diff --git a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts index 740a9af54..5aec07b70 100644 --- a/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts +++ b/apps/api/src/services/__tests__/configuration-maintenance-postgres.test.ts @@ -19,7 +19,6 @@ import { setTimeout as nativeDelay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { closeControlInstallation, - heldTargets, leaseStoreDirectory, lookupControlChildPhaseAttempt, lookupControlChildPhaseBinding, @@ -33,6 +32,7 @@ import { canonicalJson } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; import { openOpsJobJournal } from "../../../../ops-agent/src/journal"; @@ -1333,7 +1333,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), authority = restoreAdmittedControlExecution(installation, owner.plan.prepare); core.assertConfigurationMaintenanceRootClosure(authority); - await core.verifyRetainedArtifactTargetRootRelease(authority); + core.verifyLocalArtifactTargetRootRelease(authority); expect(existsSync(candidatePath)).toBe(false); expect(rootLease()).toBeNull(); expect(await fixture.db.select().from(schema.serviceSecret)).toEqual([]); @@ -1418,7 +1418,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ).receipt, core = await import("@openmapx/core/build-artifacts"); expect(final.prepare.operationKey).toBe(candidate.prepareOperationKey); - await core.verifyRetainedArtifactTargetRootRelease( + core.verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, final.prepare), ); expect((await start().exited).code).toBe(2); @@ -4921,7 +4921,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), records.configurationMaintenanceOwnerRecordSchema, ), - released = await core.verifyRetainedArtifactTargetRootRelease(authority), + released = core.verifyLocalArtifactTargetRootRelease(authority), originalFinal = JSON.stringify(owner.final), originalInspection = JSON.stringify(owner.inspection), [originalSql] = @@ -5909,7 +5909,7 @@ if(args.includes("--force-recreate")){${mode === "recreate" ? "process.exit(9);" expect(calls.filter((args) => args.includes("pull"))).toHaveLength(1); expect(calls.filter((args) => args.includes("up"))).toHaveLength(recreated ? 1 : 0); expect(heldTargets(host.infraDir).includes("motis")).toBe(true); - await expect(core.verifyRetainedArtifactTargetRootRelease(root)).rejects.toThrow(); + expect(() => core.verifyLocalArtifactTargetRootRelease(root)).toThrow(); if (mode === "missing-receipt") { await expect(receiptRead).rejects.toThrow(); expect(readdirSync(receiptDirectory)).toEqual([]); diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index c7878bf48..7beff635c 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -19,7 +19,6 @@ import { acquireArtifactTargetLease, assertCoordinatorPullLeaseCompatibility, closeControlInstallation, - heldTargets, leaseStoreDirectory, lookupControlChildPhase, openControlInstallation, @@ -31,7 +30,7 @@ import { retainControlEffectReceipt, settleArtifactTargetLeaseChild, verifyControlEffectSettlement, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, } from "@openmapx/core/build-artifacts"; import { type CoordinatorPullSubmission, @@ -44,6 +43,7 @@ import { services } from "@openmapx/core/server"; import { eq } from "drizzle-orm"; import Fastify, { type FastifyRequest } from "fastify"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { releaseProvenance } from "../../../../../packages/core/src/services/__fixtures__/release-provenance"; import { ownedDataManager } from "../../../../ops-agent/src/__fixtures__/owned-data-manager"; import { ownedServiceHost } from "../../../../ops-agent/src/__fixtures__/owned-service-host"; @@ -1755,7 +1755,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(effect.outcome).toBe("failed_without_effect"); const originalRoot = restoreAdmittedControlExecution(installation, admitted.execution); expect(lookupControlChildPhase(originalRoot, { phase: "lifecycle", ordinal: 0 })).toBeNull(); - await verifyRetainedArtifactTargetRootRelease(originalRoot); + verifyLocalArtifactTargetRootRelease(originalRoot); const repaired = await api.inject({ method: "POST", url: "/api/admin/services/bulk-action", @@ -1846,7 +1846,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( expect(state).toMatchObject({ state: "failed", errorClass: "configuration_prerequisite" }); const authority = restoreAdmittedControlExecution(installation, source.execution); expect(lookupControlChildPhase(authority, { phase: "lifecycle", ordinal: 0 })).toBeNull(); - await verifyRetainedArtifactTargetRootRelease(authority); + verifyLocalArtifactTargetRootRelease(authority); expect(services.currentConfigurationGeneration(host.infraDir)).toBe( host.initial.revisionId, ); diff --git a/apps/api/src/services/__tests__/crash-points/driver.ts b/apps/api/src/services/__tests__/crash-points/driver.ts index 7f9df3df0..4686b9ae6 100644 --- a/apps/api/src/services/__tests__/crash-points/driver.ts +++ b/apps/api/src/services/__tests__/crash-points/driver.ts @@ -338,8 +338,6 @@ export class CrashHost { stopRole: async (service) => this.stopRoleProcess(ROLE_OF_SERVICE[service] as Role), roleState: (service) => this.roleState(ROLE_OF_SERVICE[service] as Role), runContainer: (container, request) => this.runContainer(container, request.env), - execRole: (service, container, command) => - this.execRole(ROLE_OF_SERVICE[service] as Role, container, command), }); this.proxies = { "api->agent": new FaultProxy(`http://127.0.0.1:${ports.agent}`), @@ -869,34 +867,6 @@ export class CrashHost { return "running"; } - /** - * `compose exec` in the agent's container: the command runs from the agent's - * bundle with the role's environment and mounts, as `docker exec` would. - */ - private async execRole(role: Role, container: ContainerRecord, command: string[]) { - if (role !== "agent") - return { code: 126, stdout: "", stderr: "crash-point engine execs only in the agent\n" }; - const bundles = this.bundles as RuntimeBundles; - const argv = command.map((value, index) => - index === 0 && value === "node" - ? process.execPath - : value.startsWith("dist/") - ? join(bundles.opsAgent, value.slice("dist/".length)) - : value, - ); - const launched = launchContained({ - name: `exec-${role}-${Date.now()}`, - workDir: this.workDir, - argv, - env: this.roleEnvironment(role, container), - binds: this.binds(container), - ...this.identity(container), - cwd: join(REPO, "apps/ops-agent"), - }); - const { code } = await launched.exited; - return { code: code ?? 137, stdout: launched.stdout(), stderr: launched.stderr() }; - } - /** `docker run` of a release image: its command runs from the matching bundle. */ private runContainer(container: ContainerRecord, callerEnv: Record) { const bundles = this.bundles as RuntimeBundles; diff --git a/apps/api/src/services/__tests__/crash-points/support/engine.ts b/apps/api/src/services/__tests__/crash-points/support/engine.ts index 8a39c74aa..95ad3bfa0 100644 --- a/apps/api/src/services/__tests__/crash-points/support/engine.ts +++ b/apps/api/src/services/__tests__/crash-points/support/engine.ts @@ -121,12 +121,6 @@ export interface EngineHooks { stopRole(service: string): Promise; /** Whether a role service's process is up and answering. */ roleState(service: string): "running" | "starting" | "exited"; - /** `compose exec` of a command in a role service's running container. */ - execRole( - service: string, - container: ContainerRecord, - command: string[], - ): Promise<{ code: number; stdout: string; stderr: string }>; /** `docker run` of an image's command, such as the configuration helper. */ runContainer( container: ContainerRecord, @@ -1045,9 +1039,6 @@ export class FakeDockerEngine { const rest = args.slice(index + 1); if (sub === "config") return this.realCompose([...global, "config", ...rest], request); if (sub === "version") return { code: 0, stdout: "Docker Compose version v2.40.3\n" }; - // A command in a running container changes no container, so it never - // waits behind another caller's Compose effect. - if (sub === "exec") return this.composeExec(global, rest, request); // Mutating compose commands of different callers never interleave, as in // Docker where each container operation is atomic on the daemon. const run = async () => { @@ -1318,11 +1309,6 @@ export class FakeDockerEngine { services.includes(c.Config.Labels["com.docker.compose.service"] as string), ); if (quiet) return { code: 0, stdout: rows.map((c) => `${c.Id}\n`).join("") }; - if (args.includes("--services")) - return { - code: 0, - stdout: rows.map((c) => `${c.Config.Labels["com.docker.compose.service"]}\n`).join(""), - }; const format = flagValue(args, "--format"); const objects = rows.map((c) => ({ ID: c.Id, @@ -1346,27 +1332,6 @@ export class FakeDockerEngine { return { code: 0, stdout: objects.map((o) => `${o.Name} ${o.State}\n`).join("") }; } - /** `compose exec [-T] ` in a role service's running container. */ - private async composeExec(global: string[], args: string[], request: Request): Promise { - let index = 0; - while (args[index] === "-T" || args[index] === "--no-TTY") index++; - const service = args[index]; - const command = args.slice(index + 1); - if (!service || args[index]?.startsWith("-") || command.length === 0) - return this.unsupportedAnswer(["compose", "exec", ...args]); - const { project } = await this.composeModel(global, request); - const container = this.composeContainers(project).find( - (c) => c.Config.Labels["com.docker.compose.service"] === service, - ); - if ( - !container?.State.Running || - !ROLE_SERVICES.has(service) || - this.hooks.roleState(service) !== "running" - ) - return { code: 1, stderr: `service "${service}" is not running\n` }; - return this.hooks.execRole(service, container, command); - } - private servicesOf(args: string[], skipValue: string[]): string[] { const services: string[] = []; for (let i = 0; i < args.length; i++) { diff --git a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts index 7a1bd44e0..341fa1997 100644 --- a/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts +++ b/apps/api/src/services/__tests__/data-configuration-fence-postgres.test.ts @@ -267,7 +267,7 @@ describe.skipIf( existsSync(join(f.host.queue, ".candidates", `${candidate.sealedOperationKey}.json`)), ).toBe(false); expect(readFileSync(canonicalPath)).toEqual(canonical); - expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(control.verifyLocalArtifactTargetRootRelease(f.authority)).toBeDefined(); expect(f.data.preparePosts).toBe(0); expect(f.host.calls.some((args) => args.includes("up") || args.includes("restart"))).toBe( false, @@ -370,7 +370,7 @@ describe.skipIf( await tx`SELECT pg_advisory_xact_lock(${0x4f4d58},2)`; expect((await request()).status).not.toBe(200); }); - await expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).rejects.toThrow(); + expect(() => control.verifyLocalArtifactTargetRootRelease(f.authority)).toThrow(); const assertion = JSON.parse(Buffer.from(f.assertion, "base64url").toString("utf8")); expect(storeChild(f.root, assertion.leaseId, assertion.delegationId)?.state).toBe("held"); } finally { @@ -403,7 +403,7 @@ describe.skipIf( pollIntervalMs: 1, }), ).toBe("pending"); - await expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).rejects.toThrow(); + expect(() => control.verifyLocalArtifactTargetRootRelease(f.authority)).toThrow(); vi.stubEnv("DATA_MANAGER_URL", f.data.baseUrl); const cancel = client.cancel.bind(client), legacy = vi.spyOn(client, "cancel").mockImplementation(async (...args) => { @@ -417,7 +417,7 @@ describe.skipIf( pollIntervalMs: 1, }), ).toBe("pending"); - await expect(control.verifyRetainedArtifactTargetRootRelease(f.authority)).rejects.toThrow(); + expect(() => control.verifyLocalArtifactTargetRootRelease(f.authority)).toThrow(); expect( await cancelAdminJobOperations(f.ctx.checkpointResult, client, { context: f.ctx, @@ -477,7 +477,7 @@ describe.skipIf( expect(finished?.status).toBe("failed"); expect(finished?.finished_at).not.toBeNull(); expect(finished?.result._cancelRequestedAt).toBeDefined(); - expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(control.verifyLocalArtifactTargetRootRelease(f.authority)).toBeDefined(); expect(await client.status("configurationMaintenance.prepare", f.owner.executionId)).toEqual( original, ); @@ -562,7 +562,7 @@ describe.skipIf( fault.mockRestore(); expect(faulted).toBe(true); await maintenance.recoverUnadmittedConfigurationMaintenance(f.host.runtime, f.authority); - expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(control.verifyLocalArtifactTargetRootRelease(f.authority)).toBeDefined(); expect(readFileSync(canonicalPath)).toEqual(canonical); expect(f.data.preparePosts).toBe(0); expect(f.data.recoveryPosts).toBe(1); @@ -627,7 +627,7 @@ describe.skipIf( await expect( maintenance.recoverUnadmittedConfigurationMaintenance(f.host.runtime, f.authority), ).resolves.toMatchObject({ maintenanceId: f.input.maintenanceId }); - expect(await control.verifyRetainedArtifactTargetRootRelease(f.authority)).toBeDefined(); + expect(control.verifyLocalArtifactTargetRootRelease(f.authority)).toBeDefined(); } finally { clock?.mockRestore(); await f.close(); diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index 4b08743d6..ef6a2fed8 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -29,7 +29,7 @@ import { pruneControlRecords, readConfigurationHandoffSpawnIntent, restoreAdmittedControlExecution, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, verifyRetainedArtifactTargetSettlement, } from "@openmapx/core/build-artifacts"; import { configurationMaintenanceSettleRequestDigest } from "@openmapx/core/ops"; @@ -1845,8 +1845,8 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( }); expect(dm.settlePosts).toBe(0); await expect(dm.assertDataAdmission()).rejects.toThrow(); - await expect( - verifyRetainedArtifactTargetRootRelease( + expect(() => + verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), ), ).rejects.toThrow(); @@ -1872,9 +1872,9 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ).toHaveLength(1); expect(dm.settlePosts).toBe(0); - await expect(dm.assertDataAdmission()).rejects.toThrow(); - await expect( - verifyRetainedArtifactTargetRootRelease( + await expect(dm.assertDataAdmission()).toThrow(); + expect(() => + verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), ), ).rejects.toThrow(); @@ -1907,7 +1907,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( `configuration-handoff-${owner.plan.handoff!.handoffId}.completion.json`, ), before = readFileSync(file); - await expect(dm.assertDataAdmission()).rejects.toThrow(); + await expect(dm.assertDataAdmission()).toThrow(); const { inspectConfigurationHandoff } = await import( "../../../../ops-agent/src/configuration-handoff-store" ); diff --git a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts index 373e55bc2..0cbe7a1c0 100644 --- a/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts +++ b/apps/ops-agent/src/artifact-transfer/import-runtime.test.ts @@ -9,7 +9,7 @@ import { openControlInstallation, restoreAdmittedControlExecution, validateBuildBundle, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, writeBuildBundle, } from "@openmapx/core/build-artifacts"; import { @@ -321,9 +321,7 @@ it("imports through the real registered server and original target root after a fingerprint: status.fingerprint, deadlineAt: status.deadlineAt, }); - await verifyRetainedArtifactTargetRootRelease( - restoreAdmittedControlExecution(f.installation, execution), - ); + verifyLocalArtifactTargetRootRelease(restoreAdmittedControlExecution(f.installation, execution)); expect( f.calls.some((args) => args.includes("up") || args.includes("stop") || args.includes("pull")), ).toBe(false); @@ -500,9 +498,7 @@ it("reconciles the original prepared publication after exact foreign-inode repai fingerprint: status.fingerprint, deadlineAt: status.deadlineAt, }); - await verifyRetainedArtifactTargetRootRelease( - restoreAdmittedControlExecution(installation, execution), - ); + verifyLocalArtifactTargetRootRelease(restoreAdmittedControlExecution(installation, execution)); expect( journal.records().filter((job) => job.operation.kind === "artifactTransfer.import"), ).toHaveLength(1); @@ -631,7 +627,7 @@ it("observes committed publication after result loss without rewriting its origi fingerprint: original.fingerprint, deadlineAt: original.deadlineAt, }); - await verifyRetainedArtifactTargetRootRelease( + verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(next.installation, execution), ); expect(readFileSync(join(f.infraDir, "data/osrm-graph/region.osrm"), "utf8")).toBe(files); diff --git a/apps/ops-agent/src/configuration-handoff-control.test.ts b/apps/ops-agent/src/configuration-handoff-control.test.ts index adee9b428..d96decdef 100644 --- a/apps/ops-agent/src/configuration-handoff-control.test.ts +++ b/apps/ops-agent/src/configuration-handoff-control.test.ts @@ -24,8 +24,8 @@ vi.mock("node:timers/promises", async (original) => ({ new Promise((resolve) => setTimeout(resolve, milliseconds)), })); -import { LeaseStoreBusyError } from "@openmapx/core/build-artifacts"; import { createControlRootOperationKey } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; import { closeConfigurationHandoffChild, openConfigurationHandoffChild, @@ -83,7 +83,7 @@ afterEach(() => vi.useRealTimers()); it("retries only a nominal pre-adoption refusal with the original identity", async () => { const f = fixture(); mock.adoptArtifactTargetLease.mockRejectedValueOnce( - new LeaseStoreBusyError("/state/leases.lock", undefined), + new services.LockBusyError("/state/leases.lock", undefined), ); const pending = openConfigurationHandoffChild(f.input); void pending.catch(() => {}); @@ -109,7 +109,7 @@ it("retries only a nominal pre-adoption refusal with the original identity", asy it("stops at the five-second contention budget without adopting or retaining authority", async () => { const f = fixture(), - error = new LeaseStoreBusyError("/state/leases.lock", undefined); + error = new services.LockBusyError("/state/leases.lock", undefined); mock.adoptArtifactTargetLease.mockRejectedValue(error); const failed = expect(openConfigurationHandoffChild(f.input)).rejects.toBe(error); await vi.advanceTimersByTimeAsync(5050); @@ -122,7 +122,7 @@ it("stops at the five-second contention budget without adopting or retaining aut it("stops at the original deadline before the five-second wait budget", async () => { const f = fixture(), - error = new LeaseStoreBusyError("/state/leases.lock", undefined); + error = new services.LockBusyError("/state/leases.lock", undefined); f.input.execution.deadlineAt = Date.now() + 20; mock.adoptArtifactTargetLease.mockRejectedValue(error); const failed = expect(openConfigurationHandoffChild(f.input)).rejects.toBe(error); diff --git a/apps/ops-agent/src/configuration-handoff-control.ts b/apps/ops-agent/src/configuration-handoff-control.ts index 872590ab6..414e76355 100644 --- a/apps/ops-agent/src/configuration-handoff-control.ts +++ b/apps/ops-agent/src/configuration-handoff-control.ts @@ -15,7 +15,6 @@ import { closeControlInstallation, controlDigest, executionAuthorityState, - LeaseStoreBusyError, openFixedChildInstallation, readConfigurationHandoffSpawnIntent, readPrivateRecord, @@ -30,6 +29,7 @@ import { verifyFixedCliControlPeer, } from "@openmapx/core/build-artifacts"; import { canonicalJson, type ExecutionRef, executionRefSchema } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; import { type ConfigurationHandoffIntentRecord, configurationHandoffRecordDomains, @@ -50,7 +50,7 @@ const children = new WeakMap< } >(); function sourceReadBusy(error: unknown): boolean { - return error instanceof LeaseStoreBusyError; + return error instanceof services.LockBusyError; } /** Retry only the caller's certified pre-write contention. The original * deadline remains authoritative; no lock is removed or stolen. */ @@ -145,7 +145,7 @@ export async function openConfigurationHandoffChild(input: { serviceIds: intent.operation.serviceIds, }), // Any other error may follow partial adoption and is never retried. - (error) => error instanceof LeaseStoreBusyError, + (error) => error instanceof services.LockBusyError, ); const authority = retainAdmittedControlExecution(installation, { execution, diff --git a/apps/ops-agent/src/control-terminal-restart.test.ts b/apps/ops-agent/src/control-terminal-restart.test.ts index 4e6b74dbe..6f88aa9a0 100644 --- a/apps/ops-agent/src/control-terminal-restart.test.ts +++ b/apps/ops-agent/src/control-terminal-restart.test.ts @@ -16,6 +16,7 @@ import { import { services as coreServices } from "@openmapx/core/server"; import { build } from "esbuild"; import { afterAll, beforeAll, expect, it, vi } from "vitest"; +import { heldTargets } from "../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { openOpsJobJournal, type PersistedOpsJob } from "./journal"; import { buildOpsAgentServer } from "./server"; import { initializeTerminalClaimCleanup } from "./trusted-config-source"; @@ -508,7 +509,7 @@ it("repairs a borrowed terminal worker after the actual canonical-digest restart }); await app.ready(); expect(journal.records()).toEqual([stopped.job]); - expect(c.heldTargets(root).includes("motis")).toBe(true); + expect(heldTargets(root).includes("motis")).toBe(true); c.retainControlEffectReceipt(authority, { outcome: "failed_without_effect", publication: "not_started", diff --git a/apps/ops-agent/src/data-workflow-runtime.test.ts b/apps/ops-agent/src/data-workflow-runtime.test.ts index af984a688..34ca707c3 100644 --- a/apps/ops-agent/src/data-workflow-runtime.test.ts +++ b/apps/ops-agent/src/data-workflow-runtime.test.ts @@ -8,6 +8,7 @@ import { opsOperationFingerprint, } from "@openmapx/core/ops"; import { afterEach, expect, it, vi } from "vitest"; +import { heldTargets } from "../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { ownedServiceHost } from "./__fixtures__/owned-service-host"; const roots: string[] = []; @@ -67,7 +68,7 @@ it("freezes distinct original data/finalization slots and refuses targets before serviceIds: ids, }), ).rejects.toThrow("data phase not admitted"); - expect(control.heldTargets(host.infraDir).includes("data-manager")).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); const child = await control.bindControlChildPhase(authority, plan.dataReservation), key = control.prepareFixedControlChild(authority, plan.dataDelegationId); control.retainFixedCliSpawnIntent(authority, { @@ -91,7 +92,7 @@ it("freezes distinct original data/finalization slots and refuses targets before serviceIds: ids, }), ).rejects.toThrow(); - expect(control.heldTargets(host.infraDir).includes("data-manager")).toBe(false); + expect(heldTargets(host.infraDir).includes("data-manager")).toBe(false); } finally { control.closeControlInstallation(installation); } diff --git a/apps/ops-agent/src/integration-store.test.ts b/apps/ops-agent/src/integration-store.test.ts index cc0cb88c3..16bc78d10 100644 --- a/apps/ops-agent/src/integration-store.test.ts +++ b/apps/ops-agent/src/integration-store.test.ts @@ -14,7 +14,6 @@ import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { closeControlInstallation, - heldTargets, openControlInstallation, provisionControlInstallation, } from "@openmapx/core/build-artifacts"; @@ -22,6 +21,7 @@ import { createControlRootOperationKey, type OpsResourcePolicy } from "@openmapx import { services } from "@openmapx/core/server"; import { packageIntegration } from "@openmapx/integration-framework/installer"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { heldTargets } from "../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { IntegrationStore, installIntegrationRuntime } from "./integration-store"; import { openOpsJobJournal } from "./journal"; import { createUnavailableRuntime } from "./runtime"; diff --git a/apps/ops-agent/src/traffic-target-ownership.test.ts b/apps/ops-agent/src/traffic-target-ownership.test.ts index 766a035da..537fdce67 100644 --- a/apps/ops-agent/src/traffic-target-ownership.test.ts +++ b/apps/ops-agent/src/traffic-target-ownership.test.ts @@ -10,6 +10,7 @@ import { } from "@openmapx/core/ops"; import { services as coreServices } from "@openmapx/core/server"; import { expect, it } from "vitest"; +import { heldTargets } from "../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { createDockerRuntime } from "./docker-runtime"; import { openOpsJobJournal } from "./journal"; import { createProductionRegistryResourceClaimer, denyAllTrustedOpsData } from "./policy"; @@ -91,7 +92,7 @@ it.each(["succeeded", "canceled"] as const)( if (admitted.execution !== "async") throw Error("Async traffic root required"); await started; expect(effects).toBe(1); - expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(true); + expect(heldTargets(dirname(data)).includes("valhalla")).toBe(true); if (mode === "canceled") { const status = await client.status(operation.kind, admitted.operationId), lossy = new OpsClient({ @@ -113,7 +114,7 @@ it.each(["succeeded", "canceled"] as const)( expect((await client.status(operation.kind, admitted.operationId)).state).toBe( "termination_pending", ); - expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(true); + expect(heldTargets(dirname(data)).includes("valhalla")).toBe(true); if (!status.fingerprint || !status.deadlineAt) throw Error("Original dated traffic identity required"); const authority = control.restoreAdmittedControlExecution(installation, { @@ -132,7 +133,7 @@ it.each(["succeeded", "canceled"] as const)( timeout: 10000, }) .toBe("failed"); - expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(true); + expect(heldTargets(dirname(data)).includes("valhalla")).toBe(true); } else { release(); await expect @@ -140,7 +141,7 @@ it.each(["succeeded", "canceled"] as const)( timeout: 10000, }) .toBe("succeeded"); - expect(control.heldTargets(dirname(data)).includes("valhalla")).toBe(false); + expect(heldTargets(dirname(data)).includes("valhalla")).toBe(false); } } finally { release(); diff --git a/packages/cli/__tests__/cli-host-effect.test.ts b/packages/cli/__tests__/cli-host-effect.test.ts index 80f1c8ca6..d3b4ee7e5 100644 --- a/packages/cli/__tests__/cli-host-effect.test.ts +++ b/packages/cli/__tests__/cli-host-effect.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import * as control from "@openmapx/core/build-artifacts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { heldTargets } from "../../core/src/build-artifacts/__fixtures__/held-targets"; import { dockerDaemonReachable } from "../src/lib/docker"; import { leasedHostEffect, withHostTargets } from "../src/lib/host-effect"; @@ -32,7 +33,7 @@ afterEach(() => { vi.unstubAllEnvs(); rmSync(root, { recursive: true, force: true }); }); -const held = () => control.heldTargets(infraDir); +const held = () => heldTargets(infraDir); describe("host effects under a lease", () => { it("holds the targets while the effect runs and releases them after", async () => { diff --git a/packages/cli/__tests__/cli-service-lifecycle.test.ts b/packages/cli/__tests__/cli-service-lifecycle.test.ts index ce21f6270..b6643fb97 100644 --- a/packages/cli/__tests__/cli-service-lifecycle.test.ts +++ b/packages/cli/__tests__/cli-service-lifecycle.test.ts @@ -14,6 +14,7 @@ import * as control from "@openmapx/core/build-artifacts"; import { services } from "@openmapx/core/server"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { heldTargets } from "../../core/src/build-artifacts/__fixtures__/held-targets"; import { releaseProvenance } from "../../core/src/services/__fixtures__/release-provenance"; import { renderComposeForRepo } from "../src/commands/compose"; import { registerServicesCommands } from "../src/commands/services"; @@ -132,7 +133,7 @@ async function retainedReceipt() { return receipts[0]!; } function targetLocks() { - return control.heldTargets(join(root, "infra/docker")); + return heldTargets(join(root, "infra/docker")); } /** A selection whose rendered hardlink plan has one entry, so a Start issues a child to data-manager. */ async function renderWithHardlinkPlan() { diff --git a/packages/core/src/build-artifacts/__fixtures__/held-targets.ts b/packages/core/src/build-artifacts/__fixtures__/held-targets.ts new file mode 100644 index 000000000..80c1a2350 --- /dev/null +++ b/packages/core/src/build-artifacts/__fixtures__/held-targets.ts @@ -0,0 +1,7 @@ +import { leaseStoreDirectory, openLeaseStore } from "../lease-store"; + +/** Every target some lease in the installation's store holds. */ +export function heldTargets(infraDir: string): string[] { + const store = openLeaseStore({ directory: leaseStoreDirectory(infraDir), purpose: "test" }); + return [...new Set(store.list().flatMap((lease) => lease.targets))].sort(); +} diff --git a/packages/core/src/build-artifacts/configuration-pre-effect.test.ts b/packages/core/src/build-artifacts/configuration-pre-effect.test.ts index fd53f864d..db8f9e844 100644 --- a/packages/core/src/build-artifacts/configuration-pre-effect.test.ts +++ b/packages/core/src/build-artifacts/configuration-pre-effect.test.ts @@ -32,7 +32,7 @@ import { acquireArtifactTargetLease, releaseArtifactTargetLease, settleArtifactTargetLeaseChild, - verifyRetainedArtifactTargetRootRelease, + verifyLocalArtifactTargetRootRelease, } from "./target-lease"; let root: string, installation: ReturnType; @@ -115,7 +115,7 @@ it("settles the actual unissued root and consumes only its indexed not-started e verifyControlEffectSettlement(installation, owner.execution), ); await releaseArtifactTargetLease(lease); - await verifyRetainedArtifactTargetRootRelease(owner); + verifyLocalArtifactTargetRootRelease(owner); expect(() => beginControlEffect(owner)).toThrow(); }); it.each(["begun", "bound", "issued"])( @@ -149,7 +149,7 @@ it.each(["begun", "bound", "issued"])( } expect(() => retainConfigurationPreEffectRefusal(owner)).toThrow(); expect(() => inspectConfigurationPreEffectRefusal(owner)).toThrow(); - await expect(verifyRetainedArtifactTargetRootRelease(owner)).rejects.toThrow(); + expect(() => verifyLocalArtifactTargetRootRelease(owner)).toThrow(); expect( openLeaseStore({ directory: leaseStoreDirectory(root), purpose: "test" }) .list() diff --git a/packages/core/src/build-artifacts/lease-store.test.ts b/packages/core/src/build-artifacts/lease-store.test.ts index e19abbd20..a1a4e2b9c 100644 --- a/packages/core/src/build-artifacts/lease-store.test.ts +++ b/packages/core/src/build-artifacts/lease-store.test.ts @@ -4,8 +4,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { ExecutionRef } from "../ops/execution-ref"; +import { heldTargets } from "./__fixtures__/held-targets"; import { - heldTargets, type LeaseSettlement, leaseStoreDirectory, openLeaseStore, diff --git a/packages/core/src/build-artifacts/lease-store.ts b/packages/core/src/build-artifacts/lease-store.ts index 1f5732974..7824bf982 100644 --- a/packages/core/src/build-artifacts/lease-store.ts +++ b/packages/core/src/build-artifacts/lease-store.ts @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { lstatSync, mkdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { z } from "zod"; @@ -10,7 +9,6 @@ import { type LeaseRecord, type LeaseSettlement, leaseRecordSchema, - leaseTargetsSchema, } from "../ops/lease-records"; import { atomicWrite } from "../release-files"; import { assertOwnDirectory } from "../services/configuration-ownership"; @@ -33,8 +31,6 @@ export const LEASE_STORE_FILE = "leases.json"; export const LEASE_STORE_LOCK = "leases.lock"; const MAX_LEASES = 256; -/** The store's lock is held by another live writer past the wait. */ -export { LockBusyError as LeaseStoreBusyError } from "../services/recoverable-lock"; export type { LeaseChild, LeaseRecord, LeaseSettlement }; const storeSchema = z.strictObject({ @@ -138,42 +134,6 @@ export function leaseStoreDirectory(infraDir: string): string { return join(infraDir, "state", "ops-agent"); } -/** Every target some lease in the installation's store holds, read without the lock. */ -export function heldTargets(infraDir: string): string[] { - return [ - ...new Set( - readStore(join(leaseStoreDirectory(infraDir), LEASE_STORE_FILE)).flatMap( - (lease) => lease.targets, - ), - ), - ].sort(); -} - -/** Creates a held lease for `owner`, or refuses with the first holder named. */ -export function admitLease( - leases: LeaseRecord[], - input: { owner: ExecutionRef; ownerKind: string; targets: readonly string[]; now: number }, -): LeaseRecord { - const targets = leaseTargetsSchema.parse([...new Set(input.targets)].sort()); - if (leases.some((lease) => lease.owner.operationKey === input.owner.operationKey)) - throw new Error("Execution already acquired a root lease; reconcile its original identity"); - for (const target of targets) { - const holder = leases.find((lease) => lease.targets.includes(target)); - if (holder) throw new TargetBusyError(target, holder); - } - const record: LeaseRecord = { - leaseId: randomUUID(), - owner: input.owner, - ownerKind: input.ownerKind, - targets, - state: "held", - children: [], - createdAt: input.now, - }; - leases.push(record); - return record; -} - /** * Drops a lease whose owner never recorded its attempt, so nothing ran under * it. A lease that already has children is left for its owner. diff --git a/packages/core/src/build-artifacts/publication-native.test.ts b/packages/core/src/build-artifacts/publication-native.test.ts index c55e00c16..4344b513e 100644 --- a/packages/core/src/build-artifacts/publication-native.test.ts +++ b/packages/core/src/build-artifacts/publication-native.test.ts @@ -114,7 +114,7 @@ it.each(["old-directory", "build-record"])( outcome: "not_committed", reason: "failed", }); - await expect(leases.verifyRetainedArtifactTargetRootRelease(authority)).rejects.toThrow(); + expect(() => leases.verifyLocalArtifactTargetRootRelease(authority)).toThrow(); control.retainControlEffectReceipt(authority, { outcome: "contained", publication: "not_started", @@ -128,9 +128,7 @@ it.each(["old-directory", "build-record"])( control.verifyControlEffectSettlement(installation, authority.execution), ); await leases.releaseArtifactTargetLease(lease); - expect( - (await leases.verifyRetainedArtifactTargetRootRelease(authority)).releasedAt, - ).toBeGreaterThan(0); + expect(leases.verifyLocalArtifactTargetRootRelease(authority).releasedAt).toBeGreaterThan(0); }, 60000, ); diff --git a/packages/core/src/build-artifacts/target-lease.ts b/packages/core/src/build-artifacts/target-lease.ts index fe169c8d8..30f834786 100644 --- a/packages/core/src/build-artifacts/target-lease.ts +++ b/packages/core/src/build-artifacts/target-lease.ts @@ -8,6 +8,8 @@ import { type VerifiedParentExecution, } from "../ops/execution-identity"; import { opsOperationFingerprint } from "../ops/fingerprint"; +// biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. +import { leaseTargetsSchema } from "../ops/lease-records"; import { verifiedParentOrigins, verifiedParents } from "../ops/verified-parent-internal"; // biome-ignore-start lint/suspicious/noImportCycles: Exact and reserved children share the same live guards; phase schemas are lazy and these cross-module calls run after initialization. import { @@ -73,12 +75,10 @@ import { withPrivateEffectWriter, writePrivateRecord, } from "./control-authority"; -// biome-ignore-end lint/suspicious/noImportCycles: Live protocol dependency described above. import { controlChildOperationKey } from "./control-child-records"; import { controlDigest, targetSchema, uuid, verifyControl } from "./control-records"; import { assertFixedConfigurationMaintenancePlan } from "./fixed-configuration-maintenance"; import { - admitLease, knownSettlement, type LeaseChild, type LeaseRecord, @@ -207,7 +207,25 @@ export const leaseRules = { return existing; } if (now >= input.owner.deadlineAt) throw Error("Target closure or execution deadline mismatch"); - return admitLease(leases, { ...input, now }); + // Another execution with the same key is a conflict, never a second lease. + if (leases.some((lease) => lease.owner.operationKey === input.owner.operationKey)) + throw Error("Execution already acquired a root lease; reconcile its original identity"); + const targets = leaseTargetsSchema.parse([...new Set(input.targets)].sort()); + for (const target of targets) { + const holder = leases.find((lease) => lease.targets.includes(target)); + if (holder) throw new TargetBusyError(target, holder); + } + const record: LeaseRecord = { + leaseId: randomUUID(), + owner: input.owner, + ownerKind: input.ownerKind, + targets, + state: "held", + children: [], + createdAt: now, + }; + leases.push(record); + return record; }, delegate(leases: LeaseRecord[], input: Parameters[0], now: number) { const lease = leases.find((value) => value.leaseId === input.leaseId); @@ -386,9 +404,6 @@ async function fresh(s: State): Promise { s.seen = await s.backend.read(s.leaseId); return s.seen; } -function remember(s: State, record: LeaseRecord | undefined) { - s.seen = record; -} export function verifyFiniteControlEdge( role: ControlRecipient | "host-cli", @@ -641,13 +656,10 @@ export async function delegateArtifactTargetLease( true, ); }); - remember( - s, - await s.backend.delegate({ - leaseId: s.leaseId, - child: { ...row, ...(child.evidence ? { evidence: child.evidence } : {}) }, - }), - ); + s.seen = await s.backend.delegate({ + leaseId: s.leaseId, + child: { ...row, ...(child.evidence ? { evidence: child.evidence } : {}) }, + }); return assertion; } @@ -806,10 +818,7 @@ export async function settleArtifactTargetLeaseChild( if (owner.kind === "configurationMaintenance.prepare") assertConfigurationMaintenanceRootClosure(s.authority, await observeClosure(s, s.authority)); } - remember( - s, - await s.backend.settle({ leaseId: s.leaseId, key: s.node, settlement: proof.evidence }), - ); + s.seen = await s.backend.settle({ leaseId: s.leaseId, key: s.node, settlement: proof.evidence }); } export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Promise { @@ -821,7 +830,7 @@ export async function releaseArtifactTargetLease(lease: ArtifactTargetLease): Pr ) assertConfigurationMaintenanceRootClosure(s.authority, await observeClosure(s, s.authority)); await s.backend.release({ leaseId: s.leaseId }); - remember(s, undefined); + s.seen = undefined; if (s.authority) rootReleasedAt(s.authority); } @@ -831,7 +840,7 @@ export async function cancelArtifactTargetLease( const s = stateOf(lease); s.canceled = true; const record = await s.backend.cancel({ leaseId: s.leaseId, key: s.node }); - remember(s, record); + s.seen = record; // These immutable attempted identities are the adapter's cancellation outbox. return Object.freeze( (record ? descendantsOf(record, s.node) : []).map((child) => @@ -930,19 +939,10 @@ export function restoreArtifactTargetLeaseForSettlement( settlementOnly: true, }) as ArtifactTargetLease; } -/** Whether the execution's own root lease is gone from the store, after its own effect settled. */ -export async function verifyRetainedArtifactTargetRootRelease( - authority: VerifiedExecutionAuthority, -): Promise> { - const a = executionAuthorityState(authority); - const leaseId = executionLeaseId(authority); - childSettlementState(verifyControlEffectSettlement(a.installation.installation, a.execution)); - if (await a.installation.leases.read(leaseId)) throw Error("Original root release is not proved"); - return Object.freeze({ releasedAt: rootReleasedAt(authority) }); -} /** - * The same without waiting: from the release this owner recorded after the - * store dropped its lease, or from the store for a role that reads it. + * Whether the execution's own root lease is gone, after its own effect settled: + * from the release this owner recorded after the store dropped its lease, or + * from the store for a role that reads it. */ export function verifyLocalArtifactTargetRootRelease( authority: VerifiedExecutionAuthority, @@ -1303,7 +1303,7 @@ export async function settleIssuedLeaseChild( return child.settlement as ChildSettlement; } const settlement: ChildSettlement = { identity: execution, ...evidence }; - remember(s, await s.backend.settle({ leaseId: s.leaseId, key: child.key, settlement })); + s.seen = await s.backend.settle({ leaseId: s.leaseId, key: child.key, settlement }); return settlement; } /** The settlement of an original owned child, read through the owner's lease reference. */ diff --git a/packages/core/src/catalog-pins/store.test.ts b/packages/core/src/catalog-pins/store.test.ts index a11560313..4a38bd170 100644 --- a/packages/core/src/catalog-pins/store.test.ts +++ b/packages/core/src/catalog-pins/store.test.ts @@ -20,6 +20,7 @@ vi.mock("node:fs", async (importOriginal) => { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { heldTargets } from "../build-artifacts/__fixtures__/held-targets"; import { type ControlInstallation, closeControlInstallation, @@ -27,7 +28,6 @@ import { provisionControlInstallation, retainAdmittedControlExecution, } from "../build-artifacts/control-authority"; -import { heldTargets } from "../build-artifacts/lease-store"; import { type ArtifactTargetLease, acquireArtifactTargetLease, diff --git a/packages/core/src/ops/lease-records.ts b/packages/core/src/ops/lease-records.ts index 87b4078c0..779704ae0 100644 --- a/packages/core/src/ops/lease-records.ts +++ b/packages/core/src/ops/lease-records.ts @@ -54,7 +54,6 @@ export const leaseRecordSchema = z.strictObject({ state: nodeStateSchema, settlement: leaseSettlementSchema.optional(), children: z.array(leaseChildSchema).max(64), - reason: z.string().max(512).optional(), createdAt: z.number().int().nonnegative().safe(), }); export type LeaseRecord = z.infer; diff --git a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts index 543304173..775f7f144 100644 --- a/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts +++ b/services/data-manager/__tests__/transitous/catalog-workflow-postgres.test.ts @@ -48,6 +48,7 @@ import { createMigratedPostgresFixture } from "../../../../apps/api/src/test/pos import { ownedServiceHost } from "../../../../apps/ops-agent/src/__fixtures__/owned-service-host"; import { runContainedProcess } from "../../../../apps/ops-agent/src/docker-runtime"; import { buildTransitousRunnerServer } from "../../../../apps/transitous-runner/src/server"; +import { heldTargets } from "../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { ensureMotisSlotLayout } from "../../src/jobs/transitous/slot-state"; import { buildTinyGtfsFeeds } from "./fixtures/build-tiny-gtfs"; import { BERLIN_TINY_OSM_PBF_BASE64 } from "./fixtures/tiny-osm"; @@ -791,7 +792,7 @@ it const [row] = await state.db .sql`SELECT finished_at FROM data_manager.jobs WHERE id=${validation.id}`; expect(row!.finished_at).toBeNull(); - expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); + expect(heldTargets(dirname(data)).includes("motis")).toBe(true); return; } expect( @@ -831,7 +832,7 @@ it child.execution, ), ).toBeNull(); - expect(control.heldTargets(dirname(data)).includes("motis")).toBe(false); + expect(heldTargets(dirname(data)).includes("motis")).toBe(false); const validationBytes = readFileSync( join(data, "motis/proposals", proposalId, `validation-${validation.id}.json`), ); @@ -893,7 +894,7 @@ it ); expect(await cancelResponse).toBe(true); expect(lostCancellationResponse).toBe(true); - expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); + expect(heldTargets(dirname(data)).includes("motis")).toBe(true); } if (nativeDm) { // Freeze between lifecycle writes before injecting the pin-pointer crash. @@ -959,7 +960,7 @@ it const [row] = await state.db .sql`SELECT finished_at FROM data_manager.jobs WHERE id=${activation.id}`; expect(row!.finished_at).toBeNull(); - expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); + expect(heldTargets(dirname(data)).includes("motis")).toBe(true); const next = await sdk.startDataExecution( { input: { kind: "catalog.validate", proposalId, countries: ["de"] }, diff --git a/services/data-manager/src/jobs/execution/fixed-cli.test.ts b/services/data-manager/src/jobs/execution/fixed-cli.test.ts index 1a495e2ad..1251f2b0d 100644 --- a/services/data-manager/src/jobs/execution/fixed-cli.test.ts +++ b/services/data-manager/src/jobs/execution/fixed-cli.test.ts @@ -30,6 +30,7 @@ import { import { openOpsJobJournal } from "../../../../../apps/ops-agent/src/journal"; import { createUnavailableRuntime } from "../../../../../apps/ops-agent/src/runtime"; import { buildOpsAgentServer } from "../../../../../apps/ops-agent/src/server"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { registerApi } from "../../api"; import { registerAuth } from "../../auth"; import { StateStore } from "../../state"; @@ -245,7 +246,7 @@ it.skipIf( rows[0]!.execution, rows[0]!.status, ); - expect(control.heldTargets(dirname(data)).includes("data-manager")).toBe(false); + expect(heldTargets(dirname(data)).includes("data-manager")).toBe(false); } finally { await runtime.close(); agent.server.closeAllConnections(); diff --git a/services/data-manager/src/jobs/execution/full-update.test.ts b/services/data-manager/src/jobs/execution/full-update.test.ts index fef5fbef4..5df145165 100644 --- a/services/data-manager/src/jobs/execution/full-update.test.ts +++ b/services/data-manager/src/jobs/execution/full-update.test.ts @@ -28,6 +28,7 @@ import { createProductionRegistryResourceClaimer } from "../../../../../apps/ops import { dispatchOpsOperation } from "../../../../../apps/ops-agent/src/runtime"; import { buildOpsAgentServer } from "../../../../../apps/ops-agent/src/server"; import { createFileTrustedOpsDataSource } from "../../../../../apps/ops-agent/src/trusted-config-source"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { registerApi } from "../../api"; import { registerAuth } from "../../auth"; import { DATA_MANAGER_CONTROL_SERVICE_IDS } from "../../control-targets"; @@ -322,7 +323,7 @@ it.skipIf( expect(existsSync(commands) ? readFileSync(commands, "utf8") : "").not.toContain( "osrm-extract", ); - expect(control.heldTargets(dirname(data)).includes("osrm")).toBe(false); + expect(heldTargets(dirname(data)).includes("osrm")).toBe(false); releaseTransit(); const until = Date.now() + 90000; let terminal = await client.status(operation.kind, operationId!); @@ -353,7 +354,7 @@ it.skipIf( expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe( host.initial.revisionId, ); - expect(control.heldTargets(dirname(data)).includes("osrm")).toBe(false); + expect(heldTargets(dirname(data)).includes("osrm")).toBe(false); } catch (error) { const witnessProblems: Record = {}; for (const row of await store.listUnsettled()) { diff --git a/services/data-manager/src/jobs/execution/hardlinks.test.ts b/services/data-manager/src/jobs/execution/hardlinks.test.ts index 2750f3a0b..35dd8d660 100644 --- a/services/data-manager/src/jobs/execution/hardlinks.test.ts +++ b/services/data-manager/src/jobs/execution/hardlinks.test.ts @@ -30,6 +30,7 @@ import { import Fastify from "fastify"; import postgres from "postgres"; import { expect, it } from "vitest"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { registerApi } from "../../api"; import { registerAuth } from "../../auth"; import { resetOpsClient } from "../../ops-client"; @@ -259,7 +260,7 @@ it timeout: 15000, }) .toBe("ok"); - expect(control.heldTargets(dirname(data)).includes("data-manager")).toBe(true); + expect(heldTargets(dirname(data)).includes("data-manager")).toBe(true); } const terminal = await observeDataExecution( { id: admitted.id, operationKey: child.operationKey, fingerprint: child.fingerprint }, diff --git a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts index 87efe6b44..3ef5babea 100644 --- a/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts +++ b/services/data-manager/src/jobs/execution/legacy-admission-postgres.test.ts @@ -510,6 +510,7 @@ it.skipIf(!enabled).each(["git", "proposal"] as const)( import { randomUUID } from "node:crypto"; import Fastify from "fastify"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { registerApi } from "../../api"; import { registerAuth } from "../../auth"; import { resolveOperationsProfile } from "../transitous/operations-profile"; @@ -1125,7 +1126,7 @@ it.skipIf(!enabled)( expect((await f.runtime.inspect(id))?.publication).toBe("not_started"); // Terminal status is published before asynchronous lease settlement finishes. await expect - .poll(() => control.heldTargets(root).includes("motis"), { + .poll(() => heldTargets(root).includes("motis"), { timeout: 10000, }) .toBe(false); diff --git a/services/data-manager/src/jobs/execution/ops-children.test.ts b/services/data-manager/src/jobs/execution/ops-children.test.ts index 1df61f87c..4b2974940 100644 --- a/services/data-manager/src/jobs/execution/ops-children.test.ts +++ b/services/data-manager/src/jobs/execution/ops-children.test.ts @@ -446,7 +446,7 @@ it.skipIf( const pending = await store.inspect(id); expect(pending?.status.status).toBe("cancel_pending"); expect(pending?.finishedAt).toBeNull(); - expect(control.heldTargets(root).includes("motis")).toBe(true); + expect(heldTargets(root).includes("motis")).toBe(true); send({ type: "probe" }); expect(await next()).toMatchObject({ forward: false, @@ -464,7 +464,7 @@ it.skipIf( changed: true, }); await expect - .poll(() => control.heldTargets(root).includes("motis"), { + .poll(() => heldTargets(root).includes("motis"), { timeout: 10000, }) .toBe(false); @@ -511,6 +511,7 @@ it.skipIf( import { readFileSync, renameSync, statSync } from "node:fs"; import { startDataExecution } from "@openmapx/core/data-manager-sdk"; import { drizzle } from "drizzle-orm/postgres-js"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { createCandidateManifest } from "../transitous/candidate.js"; import { retainTransitPhaseCandidate, runTransitOwnedPhase } from "../transitous/owned-ops.js"; import { buildJobContext } from "../transitous/pipeline.js"; @@ -1141,7 +1142,7 @@ it.each(["before-proof", "before-terminal", "after-terminal"] as const)( const jobs = JSON.parse(readFileSync(join(root, "state/ops-agent/jobs-v1.json"), "utf8")); expect(JSON.stringify(jobs)).toContain(bound.execution.operationKey); await expect(control.releaseArtifactTargetLease(lease)).rejects.toThrow(); - expect(control.heldTargets(dirname(data)).includes("motis-feed-proxy")).toBe(true); + expect(heldTargets(dirname(data)).includes("motis-feed-proxy")).toBe(true); // No POST is sent to the restarted process; original admission remains. } finally { await first?.close(); diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts index c2d7d8aec..9a2233f20 100644 --- a/services/data-manager/src/jobs/execution/runtime.test.ts +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -8,6 +8,7 @@ import { createControlRootOperationKey, OpsClientError } from "@openmapx/core/op import { drizzle } from "drizzle-orm/postgres-js"; import postgres from "postgres"; import { describe, expect, it, vi } from "vitest"; +import { heldTargets } from "../../../../../packages/core/src/build-artifacts/__fixtures__/held-targets"; import { DataManagerReadiness } from "../../readiness.js"; import { initializeRequiredSubsystems } from "../../startup.js"; import { StateStore } from "../../state.js"; @@ -330,7 +331,7 @@ describe.skipIf( }); expect((await store.inspect(id))?.finishedAt).not.toBeNull(); expect(dispatches).toBe(1); - expect(control.heldTargets(root).includes("data-manager")).toBe(false); + expect(heldTargets(root).includes("data-manager")).toBe(false); const recovered = await store.inspect(id); expect( control.readDataControlTerminalObservation(installation, recovered!.execution).status, @@ -644,7 +645,7 @@ describe.skipIf( expect(forwards).toBe(0); expect((await store.inspect(id))?.finishedAt).toBeNull(); expect((await store.inspect(id))?.status.publication).toBe("unknown"); - expect(control.heldTargets(dirname(data)).includes("motis")).toBe(true); + expect(heldTargets(dirname(data)).includes("motis")).toBe(true); } finally { await runtime.close(); if (id) await sql`DELETE FROM data_manager.jobs WHERE id=${id}`; @@ -698,7 +699,7 @@ describe.skipIf( .poll(async () => (await store.inspect(id!))?.status.status, { timeout: 15000 }) .toBe("ok"); await expect - .poll(async () => !control.heldTargets(dirname(data)).includes("data-manager"), { + .poll(async () => !heldTargets(dirname(data)).includes("data-manager"), { timeout: 15000, }) .toBe(true); @@ -875,7 +876,7 @@ describe.skipIf( ); id = admitted.id; await readyGate; - const held = () => control.heldTargets(root).includes("data-manager"); + const held = () => heldTargets(root).includes("data-manager"); expect(held()).toBe(true); if (mode === "consume-after-abort") { start(); From 8cdcdbeff8d82e8b98ad39f289c03fe69dcb4773 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:22:17 +0100 Subject: [PATCH 30/42] refactor(ops-agent): read checked releases and pinned images through core directly The one-call wrapper around the checked release reader is gone, and the system inspection takes the pinned service images from the release manifest helper instead of building the map by hand. --- apps/ops-agent/src/administrative-runtime.ts | 10 ++-------- .../src/configuration-release-downloads.ts | 4 ++-- .../src/configuration-release-preflight.ts | 7 +++++-- apps/ops-agent/src/configuration-release.test.ts | 10 +++++++++- apps/ops-agent/src/configuration-release.ts | 13 ------------- 5 files changed, 18 insertions(+), 26 deletions(-) delete mode 100644 apps/ops-agent/src/configuration-release.ts diff --git a/apps/ops-agent/src/administrative-runtime.ts b/apps/ops-agent/src/administrative-runtime.ts index 49bf9101d..94b295a5c 100644 --- a/apps/ops-agent/src/administrative-runtime.ts +++ b/apps/ops-agent/src/administrative-runtime.ts @@ -711,16 +711,10 @@ export function createDefaultReleaseEffects( } else if (composeReady) { composeReady = false; } - const serviceImages = { - "app-api": manifest?.images.api, - "app-web": manifest?.images.web, - "data-manager": manifest?.images["data-manager"], - "ops-agent": manifest?.images["ops-agent"], - "transitous-runner": manifest?.images["transitous-runner"], - } as const; + const serviceImages = manifest ? coreServices.releaseServiceImages(manifest) : undefined; const services: OpsResultFor<"system.inspect">["services"] = []; for (const serviceId of INSPECT_RELEASE_SERVICE_IDS) { - const pinnedImage = serviceImages[serviceId]; + const pinnedImage = serviceImages?.[serviceId]; let containerId: string | undefined; // A thrown or unparseable `ps -q` is an absent observation, not an // observed-absent container. Only an exact empty result proves the diff --git a/apps/ops-agent/src/configuration-release-downloads.ts b/apps/ops-agent/src/configuration-release-downloads.ts index 40f3a258f..b8d61f3bc 100644 --- a/apps/ops-agent/src/configuration-release-downloads.ts +++ b/apps/ops-agent/src/configuration-release-downloads.ts @@ -9,8 +9,8 @@ import { configurationReleaseDownloadsSchema, configurationReleasePlatformSchema, } from "@openmapx/core/ops"; +import type { services } from "@openmapx/core/server"; import { z } from "zod"; -import type { CheckedConfigurationRelease } from "./configuration-release"; import type { DockerRuntimeOptions } from "./docker-runtime"; const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/), @@ -51,7 +51,7 @@ const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/), }); export async function inspectConfigurationReleaseDownloads(input: { - release: CheckedConfigurationRelease; + release: services.CheckedConfigurationRelease; execFile: NonNullable; signal: AbortSignal; }): Promise { diff --git a/apps/ops-agent/src/configuration-release-preflight.ts b/apps/ops-agent/src/configuration-release-preflight.ts index 6d113c763..f0ec6f469 100644 --- a/apps/ops-agent/src/configuration-release-preflight.ts +++ b/apps/ops-agent/src/configuration-release-preflight.ts @@ -5,7 +5,6 @@ import { type ConfigurationReleaseSnapshot, configurationReleaseSnapshotSchema, } from "../../../packages/core/src/build-artifacts/configuration-release-snapshot"; -import { readCheckedConfigurationRelease } from "./configuration-release"; import { inspectConfigurationReleaseDownloads } from "./configuration-release-downloads"; import { inspectConfigurationReleaseImages } from "./configuration-release-images"; import type { DockerRuntimeOptions } from "./docker-runtime"; @@ -20,7 +19,11 @@ export async function preflightConfigurationRelease(input: { const channel = services.releaseSelection(); if (channel.kind === "local") throw new services.ReleaseChannelDisabledError(); // A cached manifest is named by its digest, so it cannot change under this read. - const release = readCheckedConfigurationRelease(input.infraDir, input.selection), + const release = services.readCheckedConfigurationRelease( + input.infraDir, + input.selection, + services.readAppliedReleaseSelection(input.infraDir), + ), downloads = await inspectConfigurationReleaseDownloads({ release, execFile: input.execFile, diff --git a/apps/ops-agent/src/configuration-release.test.ts b/apps/ops-agent/src/configuration-release.test.ts index 0db8c3a41..4b9313fb1 100644 --- a/apps/ops-agent/src/configuration-release.test.ts +++ b/apps/ops-agent/src/configuration-release.test.ts @@ -6,9 +6,17 @@ import { CONFIGURATION_RELEASE_IMAGE_NAMES } from "@openmapx/core/ops"; import { services } from "@openmapx/core/server"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { releaseProvenance } from "../../../packages/core/src/services/__fixtures__/release-provenance"; -import { readCheckedConfigurationRelease } from "./configuration-release"; import { inspectConfigurationReleaseDownloads } from "./configuration-release-downloads"; +const readCheckedConfigurationRelease = ( + infraDir: string, + input: { releaseId: string; manifestDigest: string }, +) => + services.readCheckedConfigurationRelease( + infraDir, + input, + services.readAppliedReleaseSelection(infraDir), + ); let root: string, directory: string, raw: string, manifestDigest: string; const releaseId = "checked-release-01"; beforeEach(() => { diff --git a/apps/ops-agent/src/configuration-release.ts b/apps/ops-agent/src/configuration-release.ts deleted file mode 100644 index b972060d7..000000000 --- a/apps/ops-agent/src/configuration-release.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** Reuse canonical checked data from the agent's own manifest cache. */ -import { services } from "@openmapx/core/server"; -export function readCheckedConfigurationRelease( - infraDir: string, - input: { releaseId: string; manifestDigest: string }, -) { - return services.readCheckedConfigurationRelease( - infraDir, - input, - services.readAppliedReleaseSelection(infraDir), - ); -} -export type CheckedConfigurationRelease = ReturnType; From cf72b75cfbee66c3f8ff4d2c40ca23369bcae6bd Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:24:02 +0100 Subject: [PATCH 31/42] test(control): pin the shared admission directory gate and a start's lease on every service The host-authority gate refuses the removed shared admission directory in source, tests and scripts and skips installed dependencies. A service start holds every registered service, so removing another service while it runs is refused as busy. --- apps/ops-agent/src/server-foundation.test.ts | 73 ++++++++++++++++++++ scripts/gate-c-host-authority.test.ts | 21 ++++++ 2 files changed, 94 insertions(+) diff --git a/apps/ops-agent/src/server-foundation.test.ts b/apps/ops-agent/src/server-foundation.test.ts index 44b6a09ef..2d81ffcb1 100644 --- a/apps/ops-agent/src/server-foundation.test.ts +++ b/apps/ops-agent/src/server-foundation.test.ts @@ -945,6 +945,79 @@ describe("one admission path for target effects", () => { } }, 40_000); + it("holds every registered service during a start, so another service cannot be removed", async () => { + const control = await import("@openmapx/core/build-artifacts"); + const { mkdtempSync, mkdirSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const root = mkdtempSync(join(tmpdir(), "omx-leased-start-")); + mkdirSync(join(root, "data"), { mode: 0o700 }); + await control.provisionControlInstallation(root); + const installation = control.openControlInstallation({ + infraDir: root, + role: "ops-agent", + serviceIds: ["redis", "osrm"], + }); + const leaseStore = control.openLeaseStore({ + directory: control.leaseStoreDirectory(root), + purpose: "test", + }); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + const app = await buildOpsAgentServer({ + tokens: { api: apiToken, "data-manager": dataManagerToken }, + resourcePolicy: allowAllResources, + controlInstallation: installation, + leaseStore, + dispatch: async (operation) => { + if (operation.kind === "service.start") await blocked; + return { changed: true }; + }, + }); + try { + const submit = (operation: OpsOperation, name: string) => + app.inject({ + method: "POST", + url: "/v1/operations", + headers: auth(), + payload: { + ...request(operation, `ops1_${name}Start`, key(`opk1_${name}Start`)), + issuedAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + 20_000).toISOString(), + }, + }); + expect( + ( + await submit( + { + kind: "service.start", + serviceId: "redis", + expectedConfigurationRevisionId: `cfg1_${"d".repeat(43)}`, + }, + "leasedStart0000", + ) + ).statusCode, + ).toBe(202); + await eventually( + async () => leaseStore.list(), + (leases) => leases.some((lease) => lease.targets.includes("osrm")), + ); + const refused = await submit( + { kind: "service.remove", serviceId: "osrm" }, + "leasedRemove000", + ); + expect(refused.statusCode).toBe(409); + expect(refused.json().error.message).toContain("Target osrm is busy"); + } finally { + release(); + await app.close(); + control.closeControlInstallation(installation); + rmSync(root, { recursive: true, force: true }); + } + }, 30_000); + it("drops the lease of an update whose configuration revision is no longer current", async () => { const control = await import("@openmapx/core/build-artifacts"); const { installServiceLifecycleRuntime } = await import("./service-lifecycle-runtime"); diff --git a/scripts/gate-c-host-authority.test.ts b/scripts/gate-c-host-authority.test.ts index e150ee063..cf251727e 100644 --- a/scripts/gate-c-host-authority.test.ts +++ b/scripts/gate-c-host-authority.test.ts @@ -280,6 +280,27 @@ describe("Gate C — host authority", () => { } }); + it("refuses the removed shared admission directory anywhere in code, tests or scripts", () => { + const root = mkdtempSync(join(tmpdir(), "openmapx-authority-")); + try { + const name = [".admin", "target", "control"].join("-"); + mkdirSync(join(root, "packages", "core", "src"), { recursive: true }); + mkdirSync(join(root, "scripts", "node_modules", "dep"), { recursive: true }); + writeFileSync(join(root, "packages", "core", "src", "lease.ts"), `const dir = "${name}";\n`); + writeFileSync(join(root, "scripts", "smoke.sh"), `rmdir "$DATA/${name}"\n`); + writeFileSync(join(root, "scripts", "node_modules", "dep", "index.js"), `"${name}"\n`); + writeFileSync(join(root, "packages", "core", "src", "notes.md"), `${name}\n`); + expect( + findAuthorityViolations(root) + .filter((violation) => violation.detail.includes(name)) + .map((violation) => violation.file) + .sort(), + ).toEqual(["packages/core/src/lease.ts", "scripts/smoke.sh"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it("rejects a manifest that mounts the operations agent's state", () => { const root = mkdtempSync(join(tmpdir(), "openmapx-authority-")); try { From ebc8e071ef357639957c318a22860664ad832622 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:32:16 +0100 Subject: [PATCH 32/42] test(api): check a native release's unreleased root synchronously The root release check these modes use is synchronous, so the assertion expects a throw rather than a rejection. --- .../__tests__/platform-configuration-runtime-postgres.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index ef6a2fed8..f4f717390 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -1849,7 +1849,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), ), - ).rejects.toThrow(); + ).toThrow(); return; } if (version === "launch-native-killed") { @@ -1877,7 +1877,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), ), - ).rejects.toThrow(); + ).toThrow(); const { inspectConfigurationHandoff } = await import( "../../../../ops-agent/src/configuration-handoff-store" ), From 6d61ad89bed35f3caa50fac7e76a688cea7b72c1 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 01:56:52 +0100 Subject: [PATCH 33/42] test(cli): answer the rollback test's daemon check without a running Docker A recreation that fails with an exit code releases the lease only while the Docker daemon still answers. The test asked the real daemon, so it failed on a machine where Docker was not running. --- packages/cli/__tests__/cli-control-rollback.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cli/__tests__/cli-control-rollback.test.ts b/packages/cli/__tests__/cli-control-rollback.test.ts index 6d5896e30..4fafba769 100644 --- a/packages/cli/__tests__/cli-control-rollback.test.ts +++ b/packages/cli/__tests__/cli-control-rollback.test.ts @@ -19,6 +19,8 @@ vi.mock("../src/lib/docker", async (original) => ({ ...(await original()), dockerCompose: vi.fn(), dockerComposeStream: vi.fn(), + // A recreation that fails with an exit code reports a known outcome while the daemon answers. + dockerDaemonReachable: vi.fn(async () => true), })); vi.mock("../src/lib/hardlinks", async (original) => ({ ...(await original()), From 7109c53481025274ec88d10b01bb6b4d57057b9e Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 02:01:07 +0100 Subject: [PATCH 34/42] fix(control): read a lock holder's record in full across short reads A lock file is identified by its inode and bytes. A single read could return part of the record, so a holder's release saw other bytes and refused as if the lock had been taken from it. --- packages/core/src/services/recoverable-lock.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/core/src/services/recoverable-lock.ts b/packages/core/src/services/recoverable-lock.ts index da4c97ed5..f12363fd9 100644 --- a/packages/core/src/services/recoverable-lock.ts +++ b/packages/core/src/services/recoverable-lock.ts @@ -184,9 +184,15 @@ function readHolderFile(path: string): LockFile | undefined { // Waiting never turns another kind of file into a lock; leave it to the operator. if (!stat.isFile()) throw new Error(`${path} is not a lock file; remove it by hand`); if (stat.size > 4096) return { ino: stat.ino, raw: `oversized:${stat.size}` }; + // A read may return fewer bytes than asked; the holder's bytes identify it. const buffer = Buffer.alloc(stat.size); - readSync(descriptor, buffer, 0, buffer.length, 0); - const raw = buffer.toString("utf8"); + let length = 0; + while (length < buffer.length) { + const read = readSync(descriptor, buffer, length, buffer.length - length, length); + if (read === 0) break; + length += read; + } + const raw = buffer.subarray(0, length).toString("utf8"); try { return { holder: holderSchema.parse(JSON.parse(raw)), ino: stat.ino, raw }; } catch { From b6554f9b46c9995558a361d6947b64e0f68401a7 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 09:23:21 +0100 Subject: [PATCH 35/42] fix(control): skip configurations without pins when choosing a rollback target Clearing the release pin leaves a generation without pins, which rollback counted as a release of its own: after an update, a pin clear and compose up, it targeted the unpinned generation and refused although the previous release's configuration was still retained. Rollback now skips generations without pins and refuses while the applied generation pins no release. --- docs/docs/install/upgrading.md | 11 +++---- .../__tests__/cli-control-rollback.test.ts | 30 ++++++++++++------- packages/cli/src/commands/control.ts | 4 +-- packages/cli/src/lib/rollback.ts | 24 +++++++-------- .../src/services/configuration-generation.ts | 16 ++++++---- 5 files changed, 47 insertions(+), 38 deletions(-) diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index 76526e1b3..028e03a21 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -512,9 +512,11 @@ pnpm openmapx control rollback --yes # roll back The target is the newest retained configuration generation that pins another release than the applied one, so a rollback always changes the release; renders -and configuration saves made since the update do not move it. The command refuses -when no such generation is retained: no release update was applied, or its -generation was pruned (sixteen generations are kept). +and configuration saves made since the update do not move it. Generations left +without pins by clearing the release pin are skipped, and the command refuses +while the applied generation pins no release. It also refuses when no such +generation is retained: no release update was applied, or its generation was +pruned (sixteen generations are kept). Check out the target release's source first (step 2 of the upgrade, in reverse). The render compares the checkout's built-in service files with the release's and @@ -538,8 +540,7 @@ The command refuses when the database has a newer migration than the previous release ships. The migrator only applies migrations newer than the newest applied one, so an older API would otherwise run against a newer schema without complaint. CI records each release's newest migration in its release manifest; a release -published before that, or a previous configuration without pins, cannot be -checked and is refused too. In those cases, restore the backup taken before the +published before that cannot be checked and is refused too. In those cases, restore the backup taken before the update. PostGIS must be running so that the command can read the database's migrations. A rollback is itself a release change, so a second rollback returns to the configuration you rolled back from. diff --git a/packages/cli/__tests__/cli-control-rollback.test.ts b/packages/cli/__tests__/cli-control-rollback.test.ts index 4fafba769..34d5cf4d3 100644 --- a/packages/cli/__tests__/cli-control-rollback.test.ts +++ b/packages/cli/__tests__/cli-control-rollback.test.ts @@ -123,8 +123,8 @@ describe("rollback plan", () => { const plan = planRollback(infraDir); expect(plan).toMatchObject({ currentRevisionId: third, targetRevisionId: first }); expect(second).not.toBe(third); - expect(plan.current?.manifest.release).toBe("release-2"); - expect(plan.target?.manifest.release).toBe("release-1"); + expect(plan.current.manifest.release).toBe("release-2"); + expect(plan.target.manifest.release).toBe("release-1"); }); it("refuses when every retained configuration pins the applied release", async () => { @@ -132,20 +132,29 @@ describe("rollback plan", () => { await render(["alpha", "app-api"], manifest("release-1", "1", 100)); expect(() => planRollback(infraDir)).toThrow(/none is retained/); }); + + it("skips a configuration left without pins by clearing the pin", async () => { + const first = await render(["alpha"], manifest("release-1", "1", 100)); + await render(["alpha"], manifest("release-2", "2", 200)); + await render(["alpha"], null); + expect(() => planRollback(infraDir)).toThrow(/applied configuration pins no release/); + const current = await render(["alpha"], manifest("release-2", "2", 200)); + expect(planRollback(infraDir)).toMatchObject({ + currentRevisionId: current, + targetRevisionId: first, + }); + }); }); describe("the database check", () => { const plan = ( - target: services.ReleaseManifest | null, - current: services.ReleaseManifest | null = manifest("release-2", "2", 200), + target: services.ReleaseManifest, + current: services.ReleaseManifest = manifest("release-2", "2", 200), ): RollbackPlan => ({ currentRevisionId: `cfg1_${"b".repeat(43)}`, targetRevisionId: `cfg1_${"a".repeat(43)}`, - current: current && { - manifest: current, - manifestDigest: services.releaseManifestDigest(current), - }, - target: target && { manifest: target, manifestDigest: services.releaseManifestDigest(target) }, + current: { manifest: current, manifestDigest: services.releaseManifestDigest(current) }, + target: { manifest: target, manifestDigest: services.releaseManifestDigest(target) }, }); it("refuses when the database has a migration newer than the target release's newest", () => { @@ -166,8 +175,7 @@ describe("the database check", () => { ).not.toThrow(); }); - it("refuses a target that pins no release or records no migration head", () => { - expect(() => assertDatabaseAllowsRollback(plan(null), 1)).toThrow(/pins no release/); + it("refuses a target that records no migration head", () => { expect(() => assertDatabaseAllowsRollback(plan(manifest("release-1", "1")), 1)).toThrow( /records no database migration head/, ); diff --git a/packages/cli/src/commands/control.ts b/packages/cli/src/commands/control.ts index 7cfc35398..a53e24979 100644 --- a/packages/cli/src/commands/control.ts +++ b/packages/cli/src/commands/control.ts @@ -311,10 +311,10 @@ export function registerControlCommands(program: Command): void { const infraDir = repoPaths().infraDir; const plan = planRollback(infraDir); log.info( - `Applied configuration: ${plan.currentRevisionId} (release ${plan.current?.manifest.release ?? "none"})`, + `Applied configuration: ${plan.currentRevisionId} (release ${plan.current.manifest.release})`, ); log.info( - `Configuration applied with the previous release: ${plan.targetRevisionId} (release ${plan.target?.manifest.release ?? "none"})`, + `Configuration applied with the previous release: ${plan.targetRevisionId} (release ${plan.target.manifest.release})`, ); log.warn( "Rolling back returns to that configuration and reverts every configuration change made since: the release selection, the selected services, service configuration and secrets. Services it does not select keep running; stop them with `pnpm openmapx services stop `.", diff --git a/packages/cli/src/lib/rollback.ts b/packages/cli/src/lib/rollback.ts index 75f423a05..844163c05 100644 --- a/packages/cli/src/lib/rollback.ts +++ b/packages/cli/src/lib/rollback.ts @@ -4,8 +4,8 @@ import { dockerCompose } from "./docker"; export interface RollbackPlan { currentRevisionId: string; targetRevisionId: string; - current: services.GenerationRelease | null; - target: services.GenerationRelease | null; + current: services.GenerationRelease; + target: services.GenerationRelease; } /** @@ -16,17 +16,17 @@ export function planRollback(infraDir: string): RollbackPlan { const currentRevisionId = services.currentConfigurationGeneration(infraDir); if (!currentRevisionId) throw new Error("No configuration has been applied yet"); const current = services.readGenerationRelease(infraDir, currentRevisionId); + if (!current) + throw new Error( + "The applied configuration pins no release, and rollback compares releases. Pin one with `openmapx compose release` first.", + ); const targetRevisionId = services.readPreviousReleaseGeneration(infraDir, currentRevisionId); - if (!targetRevisionId) + const target = targetRevisionId && services.readGenerationRelease(infraDir, targetRevisionId); + if (!targetRevisionId || !target) throw new Error( - `No retained configuration pins another release than the applied one (${current?.manifest.release ?? "no release"}). Rollback returns to the configuration applied with the previous release, and none is retained: no release update was applied, or its configuration was pruned.`, + `No retained configuration pins another release than the applied one (${current.manifest.release}). Rollback returns to the configuration applied with the previous release, and none is retained: no release update was applied, or its configuration was pruned.`, ); - return { - currentRevisionId, - targetRevisionId, - current, - target: services.readGenerationRelease(infraDir, targetRevisionId), - }; + return { currentRevisionId, targetRevisionId, current, target }; } /** @@ -40,10 +40,6 @@ export function assertDatabaseAllowsRollback( plan: RollbackPlan, newestAppliedMigration: number | null, ): void { - if (!plan.target) - throw new Error( - "The previous configuration pins no release, so the database migrations its images expect are unknown. Rollback refuses; restore a backup taken before the update instead.", - ); const head = plan.target.manifest.databaseMigrationHead; if (!head) throw new Error( diff --git a/packages/core/src/services/configuration-generation.ts b/packages/core/src/services/configuration-generation.ts index 3ddf7af7c..5a93f6512 100644 --- a/packages/core/src/services/configuration-generation.ts +++ b/packages/core/src/services/configuration-generation.ts @@ -941,12 +941,14 @@ export function readAppliedRelease(infraDir: string): GenerationRelease | null { /** * The newest retained generation, other than `revisionId`, that pins another - * release than it does, or null when none is retained: the configuration that - * was applied with the previous release. A generation's age is that of its - * rendered files, which are never written again. + * release than it does, or null when none is retained or `revisionId` pins + * none: the configuration that was applied with the previous release. + * Generations without pins, left by clearing the pin, are skipped. A + * generation's age is that of its rendered files, which are never written again. */ export function readPreviousReleaseGeneration(infraDir: string, revisionId: string): string | null { - const digest = readGenerationRelease(infraDir, revisionId)?.manifestDigest ?? null; + const digest = readGenerationRelease(infraDir, revisionId)?.manifestDigest; + if (!digest) return null; const generations = join(infraDir, CONFIGURATION_GENERATIONS); const candidates = readdirSync(generations) .filter((name) => REVISION.test(name) && name !== revisionId) @@ -955,8 +957,10 @@ export function readPreviousReleaseGeneration(infraDir: string, revisionId: stri writtenAt: lstatSync(join(generations, name, RESOLVED_CONFIG_FILE)).mtimeMs, })) .sort((a, b) => b.writtenAt - a.writtenAt || a.name.localeCompare(b.name)); - for (const { name } of candidates) - if ((readGenerationRelease(infraDir, name)?.manifestDigest ?? null) !== digest) return name; + for (const { name } of candidates) { + const other = readGenerationRelease(infraDir, name)?.manifestDigest; + if (other && other !== digest) return name; + } return null; } From 6b4364714bceac9874adb85692c69e39dc3f3ce3 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 09:23:23 +0100 Subject: [PATCH 36/42] fix(data-manager): keep repeating a lost lease acquisition across an agent restart Three attempts over 600 ms did not cover an agent restart, so a lease the agent recorded before it died stayed held until control clear. Data-manager now repeats the acquisition for about 30 seconds within the job's deadline. The agent answers a busy store lock or a file system error as a runtime error rather than a conflict, so data-manager repeats those too instead of treating them as a refusal that wrote nothing. --- apps/ops-agent/src/server.ts | 5 +++++ .../data-manager/src/jobs/execution/runtime.test.ts | 6 ++++-- services/data-manager/src/jobs/execution/runtime.ts | 13 +++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/apps/ops-agent/src/server.ts b/apps/ops-agent/src/server.ts index 822655eef..fbb76e927 100644 --- a/apps/ops-agent/src/server.ts +++ b/apps/ops-agent/src/server.ts @@ -106,6 +106,7 @@ import { type VerifiedParentExecution, verifyApiJobParentExecution, } from "@openmapx/core/ops"; +import { services } from "@openmapx/core/server"; import Fastify, { type FastifyInstance } from "fastify"; import { inspectConfigurationGenerationNonCommit } from "../../../packages/core/src/build-artifacts/control-authority"; import { ArtifactTransferError } from "./artifact-transfer/errors"; @@ -2289,6 +2290,10 @@ export function buildOpsAgentServer(options: BuildOpsAgentServerOptions): Fastif ); } catch (error) { if (error instanceof LeaseRefused) throw new RequestGuardError("authorization", 403); + // A busy store lock or a file system error may come after the store + // was written; only a lease rule refusal certainly wrote nothing. + if (error instanceof services.LockBusyError || (error as { code?: unknown }).code) + throw new RequestGuardError("runtime", 500); throw new RequestGuardError("conflict", 409); } return reply diff --git a/services/data-manager/src/jobs/execution/runtime.test.ts b/services/data-manager/src/jobs/execution/runtime.test.ts index 9a2233f20..c165f2d86 100644 --- a/services/data-manager/src/jobs/execution/runtime.test.ts +++ b/services/data-manager/src/jobs/execution/runtime.test.ts @@ -201,7 +201,8 @@ describe.skipIf( mkdirSync(join(root, "data"), { mode: 0o700 }); await control.provisionControlInstallation(root); const agent = control.agentStoreLeases(root, "ops-agent"); - let lostReplies = 1; + // Four lost replies span about four seconds, as an agent restart would. + let lostReplies = 4; const installation = control.openControlInstallation({ infraDir: root, role: "data-manager", @@ -211,7 +212,8 @@ describe.skipIf( // The agent records the lease, and its reply never arrives. async acquire(input) { const lease = await agent.acquire(input); - if (lostReplies-- > 0) throw new OpsClientError("timeout"); + if (lostReplies-- > 0) + throw new OpsClientError(lostReplies % 2 ? "runtime" : "timeout"); return lease; }, }, diff --git a/services/data-manager/src/jobs/execution/runtime.ts b/services/data-manager/src/jobs/execution/runtime.ts index a591d0e51..f15a26be2 100644 --- a/services/data-manager/src/jobs/execution/runtime.ts +++ b/services/data-manager/src/jobs/execution/runtime.ts @@ -123,6 +123,8 @@ async function releaseOwnLease( } /** Agent answers that refuse a lease acquisition before it records anything. */ const PRE_EFFECT_REFUSALS = new Set(["authorization", "validation", "busy", "conflict"]); +/** Repeats of a lost acquisition: 250 ms doubling to 8 s, about 32 s in all. */ +const ACQUIRE_RETRIES = 8; export function dataExecutionTargetIds(input: DataExecutionInput): string[] { return ["transit-sync", "catalog.validate", "catalog.activate"].includes(input.kind) ? ["data-manager", "motis", "motis-feed-proxy", "motis-staging"] @@ -288,21 +290,24 @@ export function createDataExecutionRuntime(deps: { // that cannot answer, ends the execution as failed before any effect, with // its publication unchanged. // A timeout or transport failure may come after the agent recorded the lease, - // so the same acquisition, which returns an existing lease, is repeated. + // so the same acquisition, which returns an existing lease, is repeated for + // about 30 seconds, long enough for the agent to restart. let lease: ArtifactTargetLease | undefined; let refusal: unknown; - for (let attempt = 1; !leaf && !lease && !refusal; attempt += 1) + for (let attempt = 0; !leaf && !lease && !refusal; attempt += 1) try { lease = await acquireArtifactTargetLease({ serviceIds: targets, owner: authority }); } catch (error) { if (!(error instanceof TargetBusyError || error instanceof OpsClientError)) throw error; + const delayMs = Math.min(250 * 2 ** attempt, 8000); if ( error instanceof TargetBusyError || PRE_EFFECT_REFUSALS.has(error.errorClass) || - attempt >= 3 + attempt >= ACQUIRE_RETRIES || + now() + delayMs >= record.status.deadlineAt ) refusal = error; - else await new Promise((resolve) => setTimeout(resolve, 200 * attempt)); + else await new Promise((resolve) => setTimeout(resolve, delayMs)); } await owner.begin(authority); const controller = new AbortController(); From b79b81878ac13f4823537965e0e83a8c4fa927aa Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 09:27:41 +0100 Subject: [PATCH 37/42] fix(control): check and read the lease store through one descriptor The store was checked by path and then read by path, so a file swapped in between would be read unchecked. It is now opened without following links and checked and read through the same descriptor. The coordinator lifecycle test's state snapshot reads its files the same way. --- .../coordinator-lifecycle-postgres.test.ts | 32 ++++++++++++------- .../core/src/build-artifacts/lease-store.ts | 14 +++++--- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts index 560c5d0f4..aaaf6463b 100644 --- a/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts +++ b/apps/api/src/services/__tests__/coordinator-lifecycle-postgres.test.ts @@ -1362,12 +1362,16 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .map(String) .sort() .map((name) => { - const path = join(directory, name), - stat = statSync(path); - return [ - `${base}/${name}`, - `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, - ]; + const fd = filesystem.openSync(join(directory, name), "r"); + try { + const stat = filesystem.fstatSync(fd); + return [ + `${base}/${name}`, + `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(fd)).digest("hex") : "directory"}`, + ]; + } finally { + filesystem.closeSync(fd); + } }); }), ), @@ -1444,12 +1448,16 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( .map(String) .sort() .map((name) => { - const path = join(directory, name), - stat = statSync(path); - return [ - `${base}/${name}`, - `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(path)).digest("hex") : "directory"}`, - ]; + const fd = filesystem.openSync(join(directory, name), "r"); + try { + const stat = filesystem.fstatSync(fd); + return [ + `${base}/${name}`, + `${stat.dev}:${stat.ino}:${stat.mode}:${stat.isFile() ? createHash("sha256").update(readFileSync(fd)).digest("hex") : "directory"}`, + ]; + } finally { + filesystem.closeSync(fd); + } }); }), ), diff --git a/packages/core/src/build-artifacts/lease-store.ts b/packages/core/src/build-artifacts/lease-store.ts index 7824bf982..cf34acc73 100644 --- a/packages/core/src/build-artifacts/lease-store.ts +++ b/packages/core/src/build-artifacts/lease-store.ts @@ -1,4 +1,4 @@ -import { lstatSync, mkdirSync, readFileSync } from "node:fs"; +import { closeSync, constants, fstatSync, mkdirSync, openSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { z } from "zod"; import { canonicalJson } from "../ops/canonical-json"; @@ -78,13 +78,19 @@ export interface LeaseStore { function readStore(path: string): LeaseRecord[] { let text: string; + let fd: number | undefined; try { - const stats = lstatSync(path); + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const stats = fstatSync(fd); if (!stats.isFile() || stats.size > 8 * 1024 * 1024) throw new Error("Unsafe lease store file"); - text = readFileSync(path, "utf8"); + text = readFileSync(fd, "utf8"); } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") return []; + if (code === "ELOOP") throw new Error("Unsafe lease store file"); throw error; + } finally { + if (fd !== undefined) closeSync(fd); } return storeSchema.parse(JSON.parse(text)).leases; } From caa156356aebf3a17c2f22d83e5b7cfbd1281423 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 09:43:02 +0100 Subject: [PATCH 38/42] fix(cli): start the stack with compose up without claiming every service compose up claimed every service, so it was refused whenever a data job held Data Manager, including the notable-places build Data Manager starts on a fresh install. It takes no lease again, like services start and like compose up before; it only starts containers, and the commands that stop or replace them keep their leases. --- docs/docs/developer/build-artifacts-control.md | 6 ++++-- docs/docs/install/upgrading.md | 6 ++++-- packages/cli/src/commands/compose.ts | 5 +---- release-notes/control-leases.md | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/docs/developer/build-artifacts-control.md b/docs/docs/developer/build-artifacts-control.md index bd893fdbc..b1cdf1eef 100644 --- a/docs/docs/developer/build-artifacts-control.md +++ b/docs/docs/developer/build-artifacts-control.md @@ -338,8 +338,10 @@ path: stop, restart, start, recreate, update, remove, `stack.stop`, repository removal, backup create and restore. Pull, build, exec and probes take none. A held target refuses the operation before anything is recorded, with a busy error that names the holder's executor, operation key, kind and state and points to -`openmapx control status`. On the host, `services stop|restart`, `compose up|down` -and `backup create|restore` take the same leases. In the agent, only a Docker +`openmapx control status`. On the host, `services stop|restart`, `compose down` +and `backup create|restore` take the same leases; `compose up` and +`services start` take none, so a running data job does not refuse them. In the +agent, only a Docker command that ran to its own end and reported a failure with an exit code, while the daemon still answered, records a contained, known outcome and releases the lease; any other error, including every other failure of an agent backup, leaves diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index 028e03a21..6cd578fa5 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -302,8 +302,10 @@ stays interrupted until the operation's deadline plus 30 seconds has passed. ### Operations whose outcome is unknown Every command that changes containers (stop, restart, start, recreate, update, -remove, `compose up` and `compose down`, backup create and restore) first claims -its services in the operations agent's lease store. While one holds a service, +remove, `compose down`, backup create and restore) first claims its services in +the operations agent's lease store. The host's `compose up` and `services start` +claim nothing, as before, so that they still work while a data job runs; do not +run them during an update. While one holds a service, another waits up to five seconds for it and then refuses with a message that names the holder's operation key and points to `openmapx control status`. A failure Docker reports with an exit code releases the services; a crash, timeout, diff --git a/packages/cli/src/commands/compose.ts b/packages/cli/src/commands/compose.ts index b09a716b8..0ec6fe3a9 100644 --- a/packages/cli/src/commands/compose.ts +++ b/packages/cli/src/commands/compose.ts @@ -531,10 +531,7 @@ export function registerComposeCommands(program: Command): void { log.dim("No enabled services to start"); process.exit(0); } - // Compose may also start dependencies, so every service is claimed. - const code = await leasedHostEffect("compose.up", "all", () => - dockerComposeStream(["up", "-d", ...enabledServiceIds]), - ); + const code = await dockerComposeStream(["up", "-d", ...enabledServiceIds]); process.exit(code); }); diff --git a/release-notes/control-leases.md b/release-notes/control-leases.md index 76c1feb7b..b73284291 100644 --- a/release-notes/control-leases.md +++ b/release-notes/control-leases.md @@ -11,9 +11,9 @@ claims over HTTP; the host CLI, the agent's fixed CLI children and the configuration helper write the file under its lock. The shared lease directory on `/data` is gone. -Stop, restart, start, recreate, update, remove, `compose up` and `compose down`, -and backup create and restore all take a claim first, from the admin panel and -from the host CLI alike. While another operation holds one of the services, the +Stop, restart, start, recreate, update, remove, `compose down`, and backup +create and restore all take a claim first, from the admin panel and from the +host CLI alike; the host's `compose up` and `services start` still take none. While another operation holds one of the services, the command is refused at once with a message that names the holding operation and points to `openmapx control status`; before, `services restart` could run in the middle of an update. A failure Docker reports releases the claim. A crash, From a73489987c84134a7d105c69b3196d3e07649d60 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 11:11:00 +0100 Subject: [PATCH 39/42] test(control): expect no rollback target from a generation without pins Clearing the release pin leaves a generation without pins, which rollback now refuses to roll back from. The two clear tests checked that the cleared generation pointed back at the pinned one; they now check that it has no rollback target and that the pinned generation still pins its release. --- .../__tests__/retained-maintenance-job-postgres.test.ts | 9 +++------ apps/ops-agent/src/production-runtime.test.ts | 3 ++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts index a61f461a8..0aa2b8afe 100644 --- a/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts +++ b/apps/api/src/services/__tests__/retained-maintenance-job-postgres.test.ts @@ -881,12 +881,9 @@ dbDescribe("retained finite maintenance requests", () => { // Clearing renders the applied configuration again without the pin. expect(services.readAppliedRelease(host.infraDir)).toBeNull(); expect(services.currentConfigurationGeneration(host.infraDir)).not.toBe(pointer); - expect( - services.readPreviousReleaseGeneration( - host.infraDir, - services.currentConfigurationGeneration(host.infraDir)!, - ), - ).toBe(pointer); + expect(services.readGenerationRelease(host.infraDir, pointer!)?.manifest.release).toBe( + "checked-pin", + ); expect( journal.records().filter((row) => row.operation.kind === "release.clear"), ).toHaveLength(1); diff --git a/apps/ops-agent/src/production-runtime.test.ts b/apps/ops-agent/src/production-runtime.test.ts index 27765b6bb..04c5fa779 100644 --- a/apps/ops-agent/src/production-runtime.test.ts +++ b/apps/ops-agent/src/production-runtime.test.ts @@ -155,7 +155,8 @@ describe("assembled production service authority", () => { const cleared = services.currentConfigurationGeneration(host.infraDir); expect(cleared).not.toBe(pinned); expect(services.readAppliedRelease(host.infraDir)).toBeNull(); - expect(services.readPreviousReleaseGeneration(host.infraDir, cleared!)).toBe(pinned); + // A generation without pins has no release to roll back from. + expect(services.readPreviousReleaseGeneration(host.infraDir, cleared!)).toBeNull(); expect(services.readGenerationRelease(host.infraDir, pinned!)?.manifest.release).toBe( "checked-pin", ); From 99b79c3114b1e954196d6ae0fddfa3b6dfd16de2 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 16:44:35 +0100 Subject: [PATCH 40/42] test(ops-agent): list data-manager's lease operations in the authority inventory The inventory maps every typed operation kind to its role and a business effect. The lease.acquire, lease.read, lease.update and lease.settleChild kinds data-manager uses to claim targets in the agent's store were missing from it. --- scripts/ops-authority-inventory.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/ops-authority-inventory.json b/scripts/ops-authority-inventory.json index f4c061896..66af54969 100644 --- a/scripts/ops-authority-inventory.json +++ b/scripts/ops-authority-inventory.json @@ -131,6 +131,10 @@ "data-manager": [ "feedProxy.validateAndReload", "gbfsCatalogLock.inspect", + "lease.acquire", + "lease.read", + "lease.settleChild", + "lease.update", "motis.primary.promote", "motis.primary.restart", "motis.primary.stop", @@ -374,6 +378,15 @@ "valhalla.traffic.disable" ] }, + { + "id": "data-manager.target-leases", + "callSiteIds": [], + "semanticSources": [ + "apps/ops-agent/src/lease-operations.ts#runLeaseOperation", + "services/data-manager/src/jobs/execution/runtime.ts#createDataExecutionRuntime" + ], + "typedKinds": ["lease.acquire", "lease.read", "lease.update", "lease.settleChild"] + }, { "id": "data-manager.catalog-locks", "callSiteIds": [], From c3623bcc5ed2aac4774a281bdd72e725ed8beaea Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 17:36:20 +0100 Subject: [PATCH 41/42] fix(cli): release an interrupted backup's lease without probing the Docker daemon An interrupted backup stops its dump and removes its partial archive before it reports the interruption, so its outcome is known. The host lease wrapper still asked the Docker daemon whether it was reachable before releasing, which delayed the CLI's exit after SIGTERM past what the backup cancellation test allows. An effect can now name the errors it raised after undoing its own work; those release at once. --- .../cli/__tests__/cli-host-effect.test.ts | 19 +++++++++++++++++++ packages/cli/src/commands/backup.ts | 10 ++++++++-- packages/cli/src/lib/host-effect.ts | 10 +++++++--- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/cli/__tests__/cli-host-effect.test.ts b/packages/cli/__tests__/cli-host-effect.test.ts index d3b4ee7e5..de1cdcb8f 100644 --- a/packages/cli/__tests__/cli-host-effect.test.ts +++ b/packages/cli/__tests__/cli-host-effect.test.ts @@ -120,6 +120,25 @@ describe("host effects under a lease", () => { expect(held()).toEqual(["postgis"]); }); + it("releases an effect that undid its own work without asking the Docker daemon", async () => { + vi.mocked(dockerDaemonReachable).mockResolvedValue(false); + const interrupted = Error("Backup interrupted"); + await expect( + withHostTargets( + { + kind: "backup.create", + targets: ["postgis"], + contained: (error) => error === interrupted, + }, + async () => { + throw interrupted; + }, + ), + ).rejects.toBe(interrupted); + expect(held()).toEqual([]); + expect(dockerDaemonReachable).not.toHaveBeenCalled(); + }); + it("leases a service the agent has never seen, such as a community service just installed", async () => { expect( await withHostTargets({ kind: "services.restart", targets: ["community-late"] }, async () => diff --git a/packages/cli/src/commands/backup.ts b/packages/cli/src/commands/backup.ts index 59b96efd8..b28f6cd9d 100644 --- a/packages/cli/src/commands/backup.ts +++ b/packages/cli/src/commands/backup.ts @@ -1744,8 +1744,14 @@ export function registerBackupCommands(program: Command): void { .action(async (options: { name?: string }) => { try { // Backups stop the services they copy, so every service is claimed. - await withHostTargets({ kind: "backup.create", targets: "all" }, () => - createBackup({ name: options.name }), + await withHostTargets( + { + kind: "backup.create", + targets: "all", + // An interrupted backup has stopped its dump and removed its partial archive. + contained: (error) => error instanceof BackupInterruptedError, + }, + () => createBackup({ name: options.name }), ); } catch (err) { log.err((err as Error).message); diff --git a/packages/cli/src/lib/host-effect.ts b/packages/cli/src/lib/host-effect.ts index a60f04578..33c5beed5 100644 --- a/packages/cli/src/lib/host-effect.ts +++ b/packages/cli/src/lib/host-effect.ts @@ -45,6 +45,8 @@ export async function withHostTargets( budgetMs?: number; /** How the effect's result ended, for effects that report an exit code. */ outcomeOf?: (result: T) => Outcome; + /** Errors the effect raised after undoing its own work, such as a handled interruption. */ + contained?: (error: unknown) => boolean; }, run: () => Promise, ): Promise { @@ -71,9 +73,10 @@ export async function withHostTargets( if (error instanceof TargetBusyError) throw new HostTargetsBusyError(error.message); throw error; } - const finish = async (outcome: Outcome) => { + const finish = async (outcome: Outcome, handled = false) => { // A command that failed because the daemon went away may have run in part. - if (outcome === "contained" && !(await dockerDaemonReachable())) outcome = "unknown"; + if (outcome === "contained" && !handled && !(await dockerDaemonReachable())) + outcome = "unknown"; const known = outcome !== "unknown"; await leases.settle({ leaseId: lease.leaseId, @@ -97,7 +100,8 @@ export async function withHostTargets( try { result = await run(); } catch (error) { - await finish(thrownOutcome(error)); + if (input.contained?.(error)) await finish("contained", true); + else await finish(thrownOutcome(error)); throw error; } await finish(input.outcomeOf ? input.outcomeOf(result) : "committed"); From 21ec3ea696c684d8dc543e5fe8616dbac2528a00 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 16 Nov 2026 17:58:52 +0100 Subject: [PATCH 42/42] test(api): await the data admission refusal as a rejection again Two assertions on data-manager's admission during a native helper launch lost their rejects when the neighbouring root release check became synchronous, so they asserted a promise to be a function and left the refusal unhandled. --- .../__tests__/platform-configuration-runtime-postgres.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts index f4f717390..451af39f0 100644 --- a/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts +++ b/apps/api/src/services/__tests__/platform-configuration-runtime-postgres.test.ts @@ -1872,7 +1872,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( ), ).toHaveLength(1); expect(dm.settlePosts).toBe(0); - await expect(dm.assertDataAdmission()).toThrow(); + await expect(dm.assertDataAdmission()).rejects.toThrow(); expect(() => verifyLocalArtifactTargetRootRelease( restoreAdmittedControlExecution(installation, owner.plan.prepare), @@ -1907,7 +1907,7 @@ describe.skipIf(process.env.OPENMAPX_RUN_DATABASE_TESTS !== "1")( `configuration-handoff-${owner.plan.handoff!.handoffId}.completion.json`, ), before = readFileSync(file); - await expect(dm.assertDataAdmission()).toThrow(); + await expect(dm.assertDataAdmission()).rejects.toThrow(); const { inspectConfigurationHandoff } = await import( "../../../../ops-agent/src/configuration-handoff-store" );