diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index a4404fa78..b1fde4c87 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -646,6 +646,48 @@ graph history, remove data or establish browser readiness. Doctor's ordinary `--fix` never applies it. Run the normal project command separately after reviewing the result; its existing native graph and ownership checks still apply. +#### A run mapping from another native home + +Each launcher version has its own native home, but the run mapping lives in the +project. After an upgrade, a mapping the previous version left behind names that +version's home. Project commands from the new home (`ps`, `up`, `down`, +`doctor --native-cleanup inspect`) refuse it with a line that names the owning home +and what to do: + +- **Stopped cleanly.** The owning home's frontend records show that the run's latest + `hack up` acknowledged confirmed graph cleanup, HTTPS retirement and down hooks. + The refusal names the exact release command. Inspection reports + `state: "foreign-home"`, `owner: "stopped"` and the `selectionSha256` it needs: + + ```sh + ./hack-v5 doctor --path /absolute/project --native-run-mapping inspect --json + ./hack-v5 doctor --path /absolute/project --native-run-mapping release --expect-selection --json + ``` + +- **Live.** The owning frontend's PID still exists. The refusal names the home and + PID. Stop it with Ctrl-C in that `hack up`, or run `hack down` with the version + that owns that home, then retry. Release refuses. +- **Interrupted or unproven.** The frontend exited without acknowledging cleanup, or + the home has no readable record for the run (for example a legacy mapping, or a + home that was moved or replaced). Use the owning version's + `hack doctor --native-cleanup inspect` and `hack down`. Release refuses. + +Release proves ownership; it never just overwrites. The stored scope must differ +from this checkout in its native home only, that home must still be the recorded +directory, and the release holds the owning home's finalization lock, so that home +cannot start the run again meanwhile. A retained `hack up` in the owning home records +an admission under that same lock before its last mapping check and before any +runtime effect, and keeps it until it registers its own frontend lifetime. Release +refuses while an admission exists (a live startup is reported as live; one whose +startup exited is reported as interrupted). Owning homes from versions without this +admission record are not covered by it. A changed selection, held project locks or a +pending restart refuse, as does a down-hook journal that is owed, unknown or another +run's. A completed record of the same run, which a successful stop keeps, is allowed. +Release removes only the project's mapping and keeps its exact bytes in an audit +file beside it. The previous home's +graph and named volumes stay in that home; this home's next `hack up` starts the +instance with fresh data. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 146b40069..75629722b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1431,7 +1431,7 @@ hack doctor [options] | `--browser-result unknown|works|fails|permission-denied` | Your manual browser observation for --browser-url (default: unknown) | | `--branch ` | Run against a branch-specific instance (compose name + hostnames) | | `--native-cleanup inspect` | Inspect one mapped native graph's pending cleanup without recovery or stop replay | -| `--native-run-mapping inspect|repair` | Inspect a native run mapping, or explicitly repair it after filesystem device renumbering | +| `--native-run-mapping inspect|repair|release` | Inspect a native run mapping, explicitly repair it after filesystem device renumbering, or release one another native home left after a clean stop | | `--expect-selection <64-hex>` | Require the exact run-mapping recovery inspection selection | | `--accept-legacy-device-rebind` | Explicitly accept legacy migration without proof of original filesystem volume continuity | | `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | diff --git a/src/backends/native-project-finalization.ts b/src/backends/native-project-finalization.ts index 4b9ab307c..35c6bfa57 100644 --- a/src/backends/native-project-finalization.ts +++ b/src/backends/native-project-finalization.ts @@ -221,6 +221,17 @@ async function selected( async function read( path: string ): Promise { + const value = await readRecord(path); + if (value === null) { + return null; + } + if (!isNativeProjectFinalizationToken(value)) { + throw refused(); + } + return value; +} +/** One private, singly linked, bounded JSON record; absence is null. */ +async function readRecord(path: string): Promise { let file: Awaited>; try { file = await open( @@ -259,22 +270,12 @@ async function read( ) { throw refused(); } - const value: unknown = JSON.parse( - bytes.subarray(0, bytesRead).toString("utf8") - ); - if (!isNativeProjectFinalizationToken(value)) { - throw refused(); - } - return value; + return JSON.parse(bytes.subarray(0, bytesRead).toString("utf8")); } finally { await file.close(); } } -async function write( - root: string, - name: string, - token: NativeProjectFinalizationToken -) { +async function write(root: string, name: string, token: unknown) { const temporary = join(root, `${randomBytes(16).toString("hex")}.tmp`); const file = await open( temporary, @@ -312,7 +313,7 @@ async function locked(root: string, fn: () => Promise): Promise { } } function matches( - token: NativeProjectFinalizationToken, + token: FinalizationIdentity, hash: string, run: NativeProjectRun ): boolean { @@ -530,3 +531,205 @@ export async function waitNativeProjectFinalization(opts: { await Bun.sleep(Math.min(100, Math.max(1, end - performance.now()))); } } + +/** + * What a native home's own frontend records say about one run's latest lifetime, + * read without changing anything: + * - `completed`: that lifetime acknowledged confirmed graph cleanup, HTTPS + * retirement and down hooks (a clean stop). Nothing else proves it. + * - `live`: its frontend PID still exists (or cannot be signalled), so it may own + * resources. PID reuse can only make an exited owner look live. + * - `interrupted`: its frontend exited without acknowledging cleanup. + * - `unproven`: no readable lifetime, a legacy token without a PID, or an + * explicitly recovered dead owner whose cleanup this record does not attest. + */ +export type NativeProjectFinalizationLifetime = + | { readonly state: "completed"; readonly attempt: string } + | { readonly state: "live"; readonly pid: number } + | { readonly state: "interrupted"; readonly pid: number } + | { readonly state: "unproven" }; + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return !(isRecord(error) && error.code === "ESRCH"); + } +} + +/** + * A retained startup's admission: present from just before its last run-mapping check + * (so before any runtime effect) until it registers its own frontend lifetime. While + * it exists, the run's latest lifetime record is not the whole truth. + */ +type FinalizationAdmission = FinalizationIdentity & { + readonly version: 1; + readonly pid: number; +}; +const ADMISSION = "admission.json"; +function isAdmission(value: unknown): value is FinalizationAdmission { + return ( + isRecord(value) && + Object.keys(value).sort().join() === + "attempt,namespace,owner,pid,planId,run,scope,version" && + value.version === 1 && + validPid(value.pid) && + [value.attempt, value.run, value.owner].every( + (v) => typeof v === "string" && HEX32.test(v) + ) && + [value.scope, value.namespace, value.planId].every( + (v) => typeof v === "string" && HEX64.test(v) + ) + ); +} +async function readAdmission( + root: string +): Promise { + const value = await readRecord(join(root, ADMISSION)); + if (value !== null && !isAdmission(value)) { + throw refused(); + } + return value; +} + +/** + * Admit a retained startup of `run` in this native home before its last mapping + * check and any runtime effect. Under the run's finalization lock, so a release of + * the mapping from another home either completed first (the caller's mapping check + * then refuses) or sees this admission and refuses. Another live admission refuses. + * `retire` removes only this admission: after the new lifetime is registered, or + * when startup ends before any runtime effect. Otherwise it stays as evidence of an + * interrupted startup until a later admission of this home replaces it. + */ +export async function admitNativeProjectFinalization(opts: { + readonly scope: NativeProjectRunScope; + readonly run: NativeProjectRun; +}): Promise<{ readonly retire: () => Promise }> { + const { root, hash } = await selected(opts.scope, opts.run, true); + const admission: FinalizationAdmission = { + version: 1, + attempt: randomBytes(16).toString("hex"), + scope: hash, + run: opts.run.run, + owner: opts.run.owner, + namespace: opts.run.namespace, + planId: opts.run.planId, + pid: process.pid, + }; + await locked(root, async () => { + const prior = await readAdmission(root); + if (prior && prior.pid !== process.pid && processExists(prior.pid)) { + throw refused(); + } + await write(root, ADMISSION, admission); + }); + let retired = false; + return { + retire: async () => { + if (retired) { + return; + } + await locked(root, async () => { + const current = await readAdmission(root); + if (current?.attempt === admission.attempt) { + await unlink(join(root, ADMISSION)); + const dir = await open( + root, + constants.O_RDONLY | constants.O_NOFOLLOW + ); + try { + await dir.sync(); + } finally { + await dir.close(); + } + } + }); + retired = true; + }, + }; +} + +async function lifetimeIn( + root: string, + hash: string, + run: NativeProjectRun, + alive: (pid: number) => boolean +): Promise { + const admission = await readAdmission(root); + if (admission) { + // A startup is (or was, until interrupted) restoring this run; its records + // below may still describe the previous lifetime. + if (!matches(admission, hash, run)) { + return { state: "unproven" }; + } + return alive(admission.pid) + ? { state: "live", pid: admission.pid } + : { state: "interrupted", pid: admission.pid }; + } + const active = await read(join(root, "active.json")); + if (!(active && matches(active, hash, run))) { + return { state: "unproven" }; + } + const completed = await read(join(root, "completed.json")); + if (completed && same(active, completed)) { + return { state: "completed", attempt: active.attempt }; + } + if (active.version === 1) { + return { state: "unproven" }; + } + if (alive(active.pid)) { + return { state: "live", pid: active.pid }; + } + const recovered = await read(join(root, "recovered.json")); + return recovered && same(active, recovered) + ? { state: "unproven" } + : { state: "interrupted", pid: active.pid }; +} + +/** + * Classify the latest frontend lifetime of `run` under `scope.nativeHome`, which may + * be another native home than the caller's own. Unreadable or unsafe records are + * `unproven`, never an error, so callers can always name the owning home. + */ +export async function inspectNativeProjectFinalizationLifetime(opts: { + readonly scope: NativeProjectRunScope; + readonly run: NativeProjectRun; + readonly isAlive?: (pid: number) => boolean; +}): Promise { + try { + const { root, hash } = await selected(opts.scope, opts.run, false); + return await lifetimeIn( + root, + hash, + opts.run, + opts.isAlive ?? processExists + ); + } catch { + return { state: "unproven" }; + } +} + +/** + * Run `action` while holding the home's finalization operation lock for `run`, and + * only while its latest lifetime is still the exact completed `attempt`. A frontend + * of that home cannot begin, complete or recover a lifetime for this run meanwhile. + */ +export async function withCompletedNativeProjectFinalization(opts: { + readonly scope: NativeProjectRunScope; + readonly run: NativeProjectRun; + readonly attempt: string; + readonly action: () => Promise; +}): Promise { + if (!HEX32.test(opts.attempt)) { + throw refused(); + } + const { root, hash } = await selected(opts.scope, opts.run, false); + return await locked(root, async () => { + const lifetime = await lifetimeIn(root, hash, opts.run, processExists); + if (lifetime.state !== "completed" || lifetime.attempt !== opts.attempt) { + throw refused(); + } + return await opts.action(); + }); +} diff --git a/src/backends/native-project-mapping-command.ts b/src/backends/native-project-mapping-command.ts index 2b3f7620c..f6fd8b091 100644 --- a/src/backends/native-project-mapping-command.ts +++ b/src/backends/native-project-mapping-command.ts @@ -7,9 +7,17 @@ import { resolveWorktreeAutoBranch, sanitizeBranchSlug, } from "../lib/project.ts"; +import type { + NativeProjectRunForeignHomeInspection, + NativeProjectRunForeignHomeRelease, + NativeProjectRunScope, +} from "./native-project-run.ts"; import { + describeNativeForeignHomeMapping, inspectNativeProjectRunMapping, + nativeForeignHomeReleaseCommand, recoverNativeProjectRunFilesystem, + releaseNativeProjectRunForeignHome, } from "./native-project-run.ts"; import type { NativeRuntimeSelection } from "./native-runtime-client.ts"; @@ -17,9 +25,13 @@ const SELECTION = /^[a-f0-9]{64}$/; type Selection = | { readonly action: "inspect" } - | { readonly action: "repair"; readonly expectSelection: string }; + | { readonly action: "repair"; readonly expectSelection: string } + | { readonly action: "release"; readonly expectSelection: string }; -/** A device rebind is an explicit selected metadata migration, never an automatic Doctor fix. */ +/** + * A device rebind and a foreign-home release are explicit selected metadata changes, + * never automatic Doctor fixes; each requires the exact inspected selection. + */ export function parseNativeRunMappingRecoveryOptions(opts: { readonly action?: string; readonly expectSelection?: string; @@ -34,7 +46,7 @@ export function parseNativeRunMappingRecoveryOptions(opts: { opts.branch !== undefined ) { throw new CliUsageError( - "Run-mapping selection and branch options require --native-run-mapping inspect|repair." + "Run-mapping selection and branch options require --native-run-mapping inspect|repair|release." ); } return null; @@ -52,6 +64,18 @@ export function parseNativeRunMappingRecoveryOptions(opts: { } return { action: "inspect" }; } + if (opts.action === "release") { + if ( + opts.acceptLegacyDeviceRebind || + !opts.expectSelection || + !SELECTION.test(opts.expectSelection) + ) { + throw new CliUsageError( + "Run-mapping release requires --native-run-mapping release --expect-selection <64-hex> and no device-rebind acceptance." + ); + } + return { action: "release", expectSelection: opts.expectSelection }; + } if ( opts.action !== "repair" || !opts.acceptLegacyDeviceRebind || @@ -106,32 +130,92 @@ export async function runNativeProjectMappingCommand(opts: { nativeHome: opts.runtime.home, branch: selected.branch, }; - const data = - opts.selection.action === "inspect" - ? await inspectNativeProjectRunMapping({ - scope, - runtime: opts.runtime, - }) - : await recoverNativeProjectRunFilesystem({ - scope, - runtime: opts.runtime, - expectSelection: opts.selection.expectSelection, - acceptLegacyDeviceRebind: true, - }); + const data = await selectedMappingAction({ + selection: opts.selection, + scope, + runtime: opts.runtime, + }); if (opts.json) { emitCliResult({ result: okResult({ data }) }); - } else if ("state" in data) { - process.stdout.write( - `Native run mapping is current: branch=${selected.branch ?? "base"}, run=${data.run.run}\n` + - `graph=${data.graph ? `${data.graph.phase}, ownership ${data.graph.ownershipMatches ? "matches" : "differs"}` : "unavailable"}\n` + - "No repair is needed. Graph recovery, retained-data readback and application readiness require separate verification.\n" - ); } else { process.stdout.write( - `Native run mapping ${opts.selection.action === "repair" ? "repaired" : "inspected"}: branch=${selected.branch ?? "base"}, run=${data.run.run}\n` + - `selection=${data.selectionSha256}\nqualification=${data.qualification}\n` + - "Graph recovery, retained-data readback and application readiness require separate verification.\n" + mappingText({ + data, + action: opts.selection.action, + branch: selected.branch, + }) ); } return 0; } + +function selectedMappingAction(opts: { + readonly selection: Selection; + readonly scope: NativeProjectRunScope; + readonly runtime: NativeRuntimeSelection; +}) { + if (opts.selection.action === "inspect") { + return inspectNativeProjectRunMapping({ + scope: opts.scope, + runtime: opts.runtime, + }); + } + if (opts.selection.action === "release") { + return releaseNativeProjectRunForeignHome({ + scope: opts.scope, + expectSelection: opts.selection.expectSelection, + }); + } + return recoverNativeProjectRunFilesystem({ + scope: opts.scope, + runtime: opts.runtime, + expectSelection: opts.selection.expectSelection, + acceptLegacyDeviceRebind: true, + }); +} + +function mappingText(opts: { + readonly data: Awaited>; + readonly action: Selection["action"]; + readonly branch: string | null; +}): string { + const { data } = opts; + if ("state" in data && data.state === "foreign-home") { + return foreignHomeText(data, opts.branch); + } + if ("state" in data) { + return ( + `Native run mapping is current: branch=${opts.branch ?? "base"}, run=${data.run.run}\n` + + `graph=${data.graph ? `${data.graph.phase}, ownership ${data.graph.ownershipMatches ? "matches" : "differs"}` : "unavailable"}\n` + + "No repair is needed. Graph recovery, retained-data readback and application readiness require separate verification.\n" + ); + } + return ( + `Native run mapping ${opts.action === "repair" ? "repaired" : "inspected"}: branch=${opts.branch ?? "base"}, run=${data.run.run}\n` + + `selection=${data.selectionSha256}\nqualification=${data.qualification}\n` + + "Graph recovery, retained-data readback and application readiness require separate verification.\n" + ); +} + +function foreignHomeText( + data: + | NativeProjectRunForeignHomeInspection + | NativeProjectRunForeignHomeRelease, + branch: string | null +): string { + const head = `branch=${branch ?? "base"}, run=${data.run.run}\nowner home=${data.ownerHome}, owner=${data.owner}${data.ownerPid === null ? "" : ` (frontend pid ${data.ownerPid})`}\n`; + if ("released" in data) { + return ( + `Native run mapping released from another native home: ${head}` + + `audit=${data.auditPath}\n` + + "Its graph and retained data stay in that home. Run hack up to start this instance from this home.\n" + ); + } + if (data.selectionSha256 !== null) { + return ( + `Native run mapping belongs to another native home: ${head}selection=${data.selectionSha256}\n` + + `Release it from this home with ${nativeForeignHomeReleaseCommand({ selectionSha256: data.selectionSha256, branch })}\n` + ); + } + return `Native run mapping belongs to another native home: ${head}${describeNativeForeignHomeMapping({ inspection: data, branch })}\n`; +} diff --git a/src/backends/native-project-run.ts b/src/backends/native-project-run.ts index 480eae089..96b87fe7f 100644 --- a/src/backends/native-project-run.ts +++ b/src/backends/native-project-run.ts @@ -14,8 +14,10 @@ import { basename, isAbsolute, join, relative } from "node:path"; import { ClosedCliError, HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; import { + inspectNativeProjectFinalizationLifetime, isNativeProjectFinalizationToken, type NativeProjectFinalizationToken, + withCompletedNativeProjectFinalization, } from "./native-project-finalization.ts"; import { inspectNativeProjectGraph } from "./native-project-inspect.ts"; import { @@ -27,6 +29,9 @@ const HEX32 = /^[a-f0-9]{32}$/; const HEX64 = /^[a-f0-9]{64}$/; const LIMIT = 8192; const RECOVERY_SCHEMA = "hack.native-project-run-filesystem-recovery/v1"; +const FOREIGN_SCHEMA = "hack.native-project-run-foreign-home/v1"; +const SCOPE_KEYS = + "branch,dirIdentity,homeIdentity,nativeHome,projectDir,projectRoot,rootIdentity"; // Equivalent to normalizeEnvConfigName(value) === value, with a bounded length. const ENV_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; const AWS_PROFILE = /^[A-Za-z0-9_+=,.@-]{1,128}$/; @@ -779,8 +784,7 @@ function oldMapping( value.version !== 1 || !valid(value.run) || !isRecord(value.scope) || - Object.keys(value.scope).sort().join() !== - "branch,dirIdentity,homeIdentity,nativeHome,projectDir,projectRoot,rootIdentity" + Object.keys(value.scope).sort().join() !== SCOPE_KEYS ) { throw refused(); } @@ -1133,15 +1137,20 @@ async function currentGraph(opts: { * Read-only Doctor inspection of the saved run mapping. A mapping that still * matches this checkout is reported with its native graph phase and whether the * graph's ownership matches it; nothing needs repair. Only a mapping whose scope - * no longer matches falls through to the explicit legacy device-rebind - * inspection, which refuses every other difference. + * no longer matches falls through to the foreign-home inspection (only the native + * home differs) and then to the explicit legacy device-rebind inspection, which + * refuses every other difference. */ export async function inspectNativeProjectRunMapping(opts: { readonly scope: NativeProjectRunScope; readonly runtime: NativeRuntimeSelection; readonly invoke?: typeof invokeNativeRuntime; + /** Test seam for the owning home's frontend PID check. */ + readonly isAlive?: (pid: number) => boolean; }): Promise< - NativeProjectRunCurrentInspection | NativeProjectRunFilesystemInspection + | NativeProjectRunCurrentInspection + | NativeProjectRunFilesystemInspection + | NativeProjectRunForeignHomeInspection > { const p = await paths(opts.scope, false); await recoveryDirectories(p); @@ -1150,7 +1159,14 @@ export async function inspectNativeProjectRunMapping(opts: { try { run = record(value, p.identity); } catch { - return await inspectNativeProjectRunFilesystemRecovery(opts); + const foreign = await foreignHomeSelection({ + scope: opts.scope, + isAlive: opts.isAlive, + }).catch(() => undefined); + return ( + foreign?.inspection ?? + (await inspectNativeProjectRunFilesystemRecovery(opts)) + ); } return { schema: CURRENT_SCHEMA, @@ -1258,6 +1274,402 @@ export async function recoverNativeProjectRunFilesystem(opts: { await releaseHeldDirectory(restartLock, restartHeld); } } +/** + * A run mapping that another native home wrote for this exact checkout and + * instance. Each launcher version has its own native home, so an upgrade leaves the + * previous version's mapping behind. Its graph and retained data live only in that + * home; this home can neither restore nor stop it. + */ +export type NativeProjectRunForeignHomeInspection = { + readonly schema: typeof FOREIGN_SCHEMA; + readonly state: "foreign-home"; + /** The owning native home, as recorded and still at that path and identity. */ + readonly ownerHome: string; + /** + * The owning home's latest frontend lifetime for this run: `stopped` only when it + * acknowledged confirmed cleanup (`NativeProjectFinalizationLifetime`). + */ + readonly owner: "stopped" | "live" | "interrupted" | "unproven"; + /** The owning frontend's PID for a live or interrupted owner. */ + readonly ownerPid: number | null; + readonly run: NativeProjectRun; + readonly mappingSha256: string; + /** Present only for a cleanly stopped owner; `release` requires it exactly. */ + readonly selectionSha256: string | null; +}; + +export type NativeProjectRunForeignHomeRelease = + NativeProjectRunForeignHomeInspection & { + readonly released: true; + readonly auditPath: string; + }; + +const HOME_LIMIT = 4096; + +function directoryIdentity(value: unknown): DirectoryIdentity | undefined { + return isRecord(value) && + Object.keys(value).sort().join() === "dev,ino" && + typeof value.dev === "number" && + Number.isSafeInteger(value.dev) && + typeof value.ino === "number" && + Number.isSafeInteger(value.ino) + ? { dev: value.dev, ino: value.ino } + : undefined; +} + +function sameDirectory(value: unknown, expected: DirectoryIdentity): boolean { + const identity = directoryIdentity(value); + return identity?.dev === expected.dev && identity.ino === expected.ino; +} + +/** + * The mapping's owning home when the stored scope differs from this checkout in its + * native home only: the same canonical project root and directory, their exact + * identities, and the same instance. Any other difference is not a foreign home. + */ +function foreignHomeMapping( + value: unknown, + current: RunScopeIdentity +): + | { + readonly ownerHome: string; + readonly ownerIdentity: DirectoryIdentity; + readonly run: NativeProjectRun; + } + | undefined { + if ( + !isRecord(value) || + Object.keys(value).sort().join() !== "run,scope,version" || + value.version !== 1 || + !valid(value.run) || + !isRecord(value.scope) || + Object.keys(value.scope).sort().join() !== SCOPE_KEYS + ) { + return undefined; + } + const stored = value.scope; + const home = stored.nativeHome; + const ownerIdentity = directoryIdentity(stored.homeIdentity); + if ( + typeof home !== "string" || + !isAbsolute(home) || + home.length > HOME_LIMIT || + CONTROL.test(home) || + home === current.nativeHome || + ownerIdentity === undefined || + stored.projectRoot !== current.projectRoot || + stored.projectDir !== current.projectDir || + stored.branch !== current.branch || + !sameDirectory(stored.rootIdentity, current.rootIdentity) || + !sameDirectory(stored.dirIdentity, current.dirIdentity) + ) { + return undefined; + } + return { ownerHome: home, ownerIdentity, run: value.run }; +} + +/** The recorded home is still that exact canonical directory. */ +async function ownerHomeMatches( + home: string, + identity: DirectoryIdentity +): Promise { + try { + const info = await lstat(home); + return ( + info.isDirectory() && + !info.isSymbolicLink() && + info.dev === identity.dev && + info.ino === identity.ino && + (await realpath(home)) === home + ); + } catch { + return false; + } +} + +async function foreignHomeSelection(opts: { + readonly scope: NativeProjectRunScope; + readonly isAlive?: (pid: number) => boolean; +}): Promise< + | { + readonly inspection: NativeProjectRunForeignHomeInspection; + readonly p: RecoveryPaths; + readonly bytes: Buffer; + readonly fileIdentity: DirectoryIdentity; + readonly ownerScope: NativeProjectRunScope; + readonly attempt: string | null; + } + | undefined +> { + const p = await paths(opts.scope, false); + await recoveryDirectories(p); + const { bytes, value, identity } = await readRecoveryFile(p.file); + const foreign = foreignHomeMapping(value, p.identity); + if (foreign === undefined) { + return undefined; + } + const ownerScope = { ...opts.scope, nativeHome: foreign.ownerHome }; + const lifetime = (await ownerHomeMatches( + foreign.ownerHome, + foreign.ownerIdentity + )) + ? await inspectNativeProjectFinalizationLifetime({ + scope: ownerScope, + run: foreign.run, + isAlive: opts.isAlive, + }) + : ({ state: "unproven" } as const); + const mappingSha256 = createHash("sha256").update(bytes).digest("hex"); + const attempt = lifetime.state === "completed" ? lifetime.attempt : null; + const selectionSha256 = + attempt === null + ? null + : createHash("sha256") + .update( + JSON.stringify({ + schema: FOREIGN_SCHEMA, + mappingSha256, + fileIdentity: identity, + scope: p.identity, + ownerHome: foreign.ownerHome, + ownerIdentity: foreign.ownerIdentity, + run: foreign.run, + attempt, + }) + ) + .digest("hex"); + return { + inspection: { + schema: FOREIGN_SCHEMA, + state: "foreign-home", + ownerHome: foreign.ownerHome, + owner: lifetime.state === "completed" ? "stopped" : lifetime.state, + ownerPid: + lifetime.state === "live" || lifetime.state === "interrupted" + ? lifetime.pid + : null, + run: foreign.run, + mappingSha256, + selectionSha256, + }, + p, + bytes, + fileIdentity: identity, + ownerScope, + attempt, + }; +} + +/** The doctor command that releases a stopped foreign-home mapping from this home. */ +export function nativeForeignHomeReleaseCommand(opts: { + readonly selectionSha256: string; + readonly branch: string | null; +}): string { + const branch = opts.branch === null ? "" : ` --branch ${opts.branch}`; + return `hack doctor --native-run-mapping release --expect-selection ${opts.selectionSha256}${branch}`; +} + +/** The refusal sentence for a foreign-home mapping: names the owning home and the way out. */ +export function describeNativeForeignHomeMapping(opts: { + readonly inspection: NativeProjectRunForeignHomeInspection; + readonly branch: string | null; +}): string { + const { inspection } = opts; + const where = `run ${inspection.run.run} in another native home (${inspection.ownerHome})`; + const stopThere = "run hack down with the Hack version that owns that home"; + if (inspection.owner === "stopped" && inspection.selectionSha256 !== null) { + return ( + `Native project run mapping belongs to ${where}, which stopped it cleanly; its retained data stays in that home. ` + + `To start this instance from this native home, release the mapping with ${nativeForeignHomeReleaseCommand( + { + selectionSha256: inspection.selectionSha256, + branch: opts.branch, + } + )}, or ${stopThere}.` + ); + } + if (inspection.owner === "live") { + return ( + `Native project run mapping is owned by live ${where}, frontend pid ${inspection.ownerPid}. ` + + `Stop it first: press Ctrl-C in that hack up, or ${stopThere}; then retry here.` + ); + } + if (inspection.owner === "interrupted") { + return ( + `Native project run mapping belongs to ${where}, whose frontend (pid ${inspection.ownerPid}) exited without confirming cleanup. ` + + "With the Hack version that owns that home, inspect it with hack doctor --native-cleanup inspect, then run hack down; then retry here." + ); + } + return ( + `Native project run mapping belongs to ${where}, and its clean stop cannot be proven from this home. ` + + `Before using this home, ${stopThere}.` + ); +} + +async function foreignHomeRefusal( + scope: NativeProjectRunScope +): Promise { + try { + const selected = await foreignHomeSelection({ scope }); + return selected + ? new ClosedCliError( + describeNativeForeignHomeMapping({ + inspection: selected.inspection, + branch: scope.branch, + }) + ) + : undefined; + } catch { + return undefined; + } +} + +/** The kind tag of the value-free down-hook journal beside the run mapping. */ +const DOWN_HOOK_KIND = "hack.native-project-down-hooks"; + +function pendingDownHooks(): Error { + return new ClosedCliError( + "Native project run mapping from another native home has down hooks that are owed, unknown or not this run's; it was not released. Run hack down with the Hack version that owns that home first." + ); +} + +/** + * No down hook is owed for the selected run: its journal is absent, or it is a + * completed record of exactly this run (a successful stop keeps that terminal + * record rather than removing it). An owed, started (unknown), failed, unreadable or + * another run's record refuses. The journal's own writers hold its lock, which the + * caller checks separately. + */ +async function downHooksSettled(path: string, run: string): Promise { + let value: unknown; + try { + value = (await readRecoveryFile(path)).value; + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return; + } + throw pendingDownHooks(); + } + if ( + !( + isRecord(value) && + Object.keys(value).sort().join() === + "generation,kind,run,state,version" && + value.version === 1 && + value.kind === DOWN_HOOK_KIND && + value.run === run && + typeof value.generation === "string" && + HEX32.test(value.generation) && + value.state === "complete" + ) + ) { + throw pendingDownHooks(); + } +} + +/** + * Release a mapping that another native home left after a clean stop, so this home + * can start the instance afresh. Ownership is proved, never assumed: the stored scope + * differs only in its native home, that home is still the recorded directory, and + * its frontend records show the run's latest lifetime acknowledged confirmed cleanup. + * A down-hook journal must be absent or this run's completed record. + * The owning home's finalization lock is held throughout, so it cannot begin a new + * lifetime meanwhile. Pending restart or down-hook state refuses. The exact mapping + * bytes are kept in an audit file; the owning home's graph and data are untouched. + */ +export async function releaseNativeProjectRunForeignHome(opts: { + readonly scope: NativeProjectRunScope; + readonly expectSelection: string; +}): Promise { + if (!HEX64.test(opts.expectSelection)) { + throw refused(); + } + const initial = await foreignHomeSelection({ scope: opts.scope }); + if (initial === undefined) { + throw refused(); + } + if (initial.attempt === null) { + throw new ClosedCliError( + describeNativeForeignHomeMapping({ + inspection: initial.inspection, + branch: opts.scope.branch, + }) + ); + } + if (initial.inspection.selectionSha256 !== opts.expectSelection) { + throw refused(); + } + const attempt = initial.attempt; + await recoveryLocksAbsent(initial.p); + const base = initial.p.file.slice(0, -5); + const restartLock = `${initial.p.lock.slice(0, -5)}.restart.lock.operation`; + await mkdir(restartLock, { mode: 0o700 }); + const restartHeld = await heldDirectory(restartLock); + try { + await mkdir(initial.p.lock, { mode: 0o700 }); + const mappingHeld = await heldDirectory(initial.p.lock); + try { + return await withCompletedNativeProjectFinalization({ + scope: initial.ownerScope, + run: initial.inspection.run, + attempt, + action: async () => { + const selected = await foreignHomeSelection({ scope: opts.scope }); + if ( + selected === undefined || + selected.inspection.selectionSha256 !== opts.expectSelection || + selected.p.file !== initial.p.file || + !selected.bytes.equals(initial.bytes) || + selected.fileIdentity.dev !== initial.fileIdentity.dev || + selected.fileIdentity.ino !== initial.fileIdentity.ino + ) { + throw refused(); + } + await Promise.all([ + absentPath(`${base}.restart.json`), + downHooksSettled( + `${base}.down-hooks.json`, + selected.inspection.run.run + ), + absentPath(`${base}.down-hooks.lock`), + ]); + const audit = `${base}.foreign-home-release-${selected.inspection.mappingSha256}.json`; + await preserveRecoveryAudit( + audit, + JSON.stringify({ + version: 1, + schema: FOREIGN_SCHEMA, + selectionSha256: opts.expectSelection, + mappingSha256: selected.inspection.mappingSha256, + ownerHome: selected.inspection.ownerHome, + mapping: selected.bytes.toString("utf8"), + }) + ); + await sync(selected.p.root); + await Promise.all([restartHeld.verify(), mappingHeld.verify()]); + const closeMapping = await verifySelectedMappingPath( + selected.p.file, + selected.bytes, + selected.fileIdentity + ); + try { + await unlink(selected.p.file); + } finally { + await closeMapping(); + } + await sync(selected.p.root); + await absentPath(selected.p.file); + return { ...selected.inspection, released: true, auditPath: audit }; + }, + }); + } finally { + await releaseHeldDirectory(initial.p.lock, mappingHeld); + } + } finally { + await releaseHeldDirectory(restartLock, restartHeld); + } +} + /** * Private paths of the value-free down-hook journal beside the run mapping, * under the same scope, directory and ignore-file checks. Missing storage @@ -1289,15 +1701,22 @@ export async function loadNativeProjectRun( } catch { throw refused(); } + let stored: { readonly value: unknown; readonly identity: RunScopeIdentity }; try { const p = await paths(opts, false); - return record(await read(p.file), p.identity); + stored = { value: await read(p.file), identity: p.identity }; } catch (error) { if (isRecord(error) && error.code === "ENOENT") { return null; } throw refused(); } + try { + return record(stored.value, stored.identity); + } catch { + // Only a mapping that differs in nothing but its native home gets a named refusal. + throw (await foreignHomeRefusal(opts)) ?? refused(); + } } /** Publish a new owner once, or replace its exact stopped mapping after restore. */ export async function saveNativeProjectRun( diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index 1f022cab7..a4d6a5aab 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -61,6 +61,7 @@ import { } from "./native-project-down-hooks.ts"; import { discardUnpublishedNativeStart } from "./native-project-failed-start.ts"; import { + admitNativeProjectFinalization, beginNativeProjectFinalization, captureNativeProjectFinalization, waitNativeProjectFinalization, @@ -345,6 +346,7 @@ type Dependencies = { save: typeof saveNativeProjectRun; remove: typeof removeNativeProjectRun; finalization: typeof beginNativeProjectFinalization; + admitFinalization: typeof admitNativeProjectFinalization; captureFinalization: typeof captureNativeProjectFinalization; waitFinalization: typeof waitNativeProjectFinalization; discardFailedStart: typeof discardUnpublishedNativeStart; @@ -376,6 +378,7 @@ const DEFAULTS: Dependencies = { save: saveNativeProjectRun, remove: removeNativeProjectRun, finalization: beginNativeProjectFinalization, + admitFinalization: admitNativeProjectFinalization, captureFinalization: captureNativeProjectFinalization, waitFinalization: waitNativeProjectFinalization, discardFailedStart: discardUnpublishedNativeStart, @@ -1212,6 +1215,38 @@ async function confirmCleanupOrRecordOwner(opts: { } } +/** + * HACK-1952: admit a retained startup in its native home before its last mapping + * check, so another home's release of the mapping either finished first (the check + * refuses) or sees this admission and refuses. The caller retires it once it + * registers its own frontend lifetime; it is retired here if startup ends before any + * runtime effect. + */ +async function admitRetainedStartup(opts: { + readonly retained: NativeProjectRun | null; + readonly scope: NativeProjectRunScope; + readonly deps: Pick; + readonly signal: AbortSignal; +}): Promise<{ readonly retire: () => Promise } | undefined> { + const admission = opts.retained + ? await opts.deps.admitFinalization({ + scope: opts.scope, + run: opts.retained, + }) + : undefined; + try { + await verifyNativeRetainedMapping({ + run: opts.retained, + scope: opts.scope, + load: opts.deps.load, + }); + requireActiveStartup(opts.signal); + } catch (error) { + await admission?.retire(); + throw error; + } + return admission; +} async function refusePendingFreshStart( restore: NativeProjectRun | undefined, scope: NativeProjectRunScope, @@ -1883,12 +1918,12 @@ export async function startNativeProject(opts: { managedFiles: input.managedFiles, dependencyServices: hostDependencies.map((binding) => binding.service), }); - await verifyNativeRetainedMapping({ - run: retained, + const admission = await admitRetainedStartup({ + retained, scope: opts.scope, - load: deps.load, + deps, + signal: controller.signal, }); - requireActiveStartup(controller.signal); progress({ phase: "runtime" }); await deps.invoke({ runtime: opts.runtime, @@ -2232,6 +2267,8 @@ export async function startNativeProject(opts: { httpsPort: https ? opts.https?.httpsPort : null, httpsLease: https?.identity, }); + // The registered lifetime now describes this startup. + await admission?.retire(); requireActiveStartup(controller.signal); await verifyHttpsRoutes( https, diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 8ae48aea0..03a4a9559 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -383,9 +383,9 @@ const doctorOptions = [ name: "nativeRunMapping", type: "string", long: "--native-run-mapping", - valueHint: "inspect|repair", + valueHint: "inspect|repair|release", description: - "Inspect a native run mapping, or explicitly repair it after filesystem device renumbering", + "Inspect a native run mapping, explicitly repair it after filesystem device renumbering, or release one another native home left after a clean stop", } as const), defineOption({ name: "expectSelection", diff --git a/tests/native-project-mapping-command.test.ts b/tests/native-project-mapping-command.test.ts index 8e707d586..5dfec4f14 100644 --- a/tests/native-project-mapping-command.test.ts +++ b/tests/native-project-mapping-command.test.ts @@ -44,6 +44,22 @@ test("repair requires both an exact inspection selection and explicit legacy acc } }); +test("foreign-home release requires the exact selection and refuses device-rebind acceptance", () => { + expect( + parse({ action: "release", expectSelection: selected, otherOptions: false }) + ).toEqual({ action: "release", expectSelection: selected }); + for (const options of [ + {}, + { expectSelection: "a".repeat(63) }, + { expectSelection: `${selected}\n` }, + { expectSelection: selected, acceptLegacyDeviceRebind: true }, + ]) { + expect(() => + parse({ action: "release", ...options, otherOptions: false }) + ).toThrow("Run-mapping release requires"); + } +}); + test("unknown actions and orphaned selectors refuse before project lookup", () => { expect(() => parse({ action: "force", otherOptions: false })).toThrow( "requires" @@ -60,7 +76,7 @@ test("unknown actions and orphaned selectors refuse before project lookup", () = }); test("mapping migration cannot combine with ordinary repair, domain or browser flows", () => { - for (const action of ["inspect", "repair"]) { + for (const action of ["inspect", "repair", "release"]) { expect(() => parse({ action, diff --git a/tests/native-project-run-foreign-home.test.ts b/tests/native-project-run-foreign-home.test.ts new file mode 100644 index 000000000..326bf76a4 --- /dev/null +++ b/tests/native-project-run-foreign-home.test.ts @@ -0,0 +1,412 @@ +import { afterEach, expect, test } from "bun:test"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + realpath, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { replaceNativeDownHookRecord } from "../src/backends/native-project-down-hooks.ts"; +import { + admitNativeProjectFinalization, + beginNativeProjectFinalization, + recoverNativeProjectFinalization, +} from "../src/backends/native-project-finalization.ts"; +import { + inspectNativeProjectRunMapping, + loadNativeProjectRun as load, + type NativeProjectRun, + type NativeProjectRunForeignHomeInspection, + releaseNativeProjectRunForeignHome as release, + saveNativeProjectRun as save, +} from "../src/backends/native-project-run.ts"; +import type { invokeNativeRuntime } from "../src/backends/native-runtime-client.ts"; +import { expectClosedRefusal } from "./helpers/closed-refusal.ts"; + +/** + * HACK-1952: each launcher version has its own native home, and the project's run + * mapping names the home that wrote it. Two isolated homes share one checkout here; + * nothing starts a runtime. + */ + +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })) + ); +}); +const run: NativeProjectRun = { + run: "a".repeat(32), + owner: "b".repeat(32), + namespace: "c".repeat(64), + planId: "d".repeat(64), + effectiveEnvName: null, + profiles: [], + aws: null, +}; +/** The foreign-home path never asks a runtime; any request fails the test. */ +const noRuntime: typeof invokeNativeRuntime = () => { + throw new Error("foreign-home inspection must not invoke a native runtime"); +}; + +async function fixture(branch: string | null = null) { + const root = await realpath( + await mkdtemp(join(tmpdir(), "native-foreign-home-")) + ); + roots.push(root); + const projectRoot = join(root, "project"); + const projectDir = join(projectRoot, ".hack"); + const homeA = join(root, "versions", "one", "native-home"); + const homeB = join(root, "versions", "two", "native-home"); + await mkdir(projectDir, { recursive: true }); + await mkdir(homeA, { recursive: true, mode: 0o700 }); + await mkdir(homeB, { recursive: true, mode: 0o700 }); + const a = { projectRoot, projectDir, nativeHome: homeA, branch }; + const b = { projectRoot, projectDir, nativeHome: homeB, branch }; + const inspect = (isAlive?: (pid: number) => boolean) => + inspectNativeProjectRunMapping({ + scope: b, + runtime: { binary: join(root, "absent-runtime"), home: homeB }, + invoke: noRuntime, + isAlive, + }) as Promise; + const runs = join(projectDir, ".internal", "native-runs"); + return { root, homeA, homeB, a, b, inspect, runs }; +} + +/** Home A starts the run and, when `stop` is set, acknowledges its clean stop. */ +async function startInHomeA( + f: Awaited>, + stop: boolean +) { + await save({ ...f.a, run }); + const lifetime = await beginNativeProjectFinalization({ + scope: f.a, + run, + httpsPort: null, + }); + if (stop) { + await lifetime.complete(); + } + return lifetime; +} + +test("a cleanly stopped run from another home is named, inspectable and releasable", async () => { + const f = await fixture(); + await startInHomeA(f, true); + const mapping = (await readdir(f.runs)).find( + (name) => /^[a-f0-9]{64}\.json$/.test(name) // the run mapping itself + ); + const bytes = await readFile(join(f.runs, mapping ?? "absent"), "utf8"); + + const inspection = await f.inspect(); + expect(inspection).toMatchObject({ + schema: "hack.native-project-run-foreign-home/v1", + state: "foreign-home", + ownerHome: f.homeA, + owner: "stopped", + ownerPid: null, + run, + }); + const selection = inspection.selectionSha256 ?? ""; + expect(selection).toMatch(/^[a-f0-9]{64}$/); + await expectClosedRefusal( + load(f.b), + `Native project run mapping belongs to run ${run.run} in another native home (${f.homeA}), which stopped it cleanly; its retained data stays in that home. To start this instance from this native home, release the mapping with hack doctor --native-run-mapping release --expect-selection ${selection}, or run hack down with the Hack version that owns that home.` + ); + + await expectClosedRefusal( + release({ scope: f.b, expectSelection: "e".repeat(64) }), + "Native project run mapping is unsafe, changed, or owned by another run" + ); + expect(await load(f.a)).toEqual(run); + + const released = await release({ scope: f.b, expectSelection: selection }); + expect(released).toMatchObject({ + released: true, + owner: "stopped", + ownerHome: f.homeA, + }); + expect(await load(f.b)).toBeNull(); + const audit = JSON.parse(await readFile(released.auditPath, "utf8")); + expect(audit).toMatchObject({ + schema: "hack.native-project-run-foreign-home/v1", + selectionSha256: selection, + ownerHome: f.homeA, + mapping: bytes, + }); + // The owning home keeps its own frontend history; nothing there was removed. + expect( + ( + await readdir(join(f.homeA, ".hack-local", "frontend-finalization"), { + recursive: true, + }) + ).filter((name) => name.endsWith(".json")) + ).toHaveLength(2); + // This home can now publish its own fresh owner for the same instance. + await save({ ...f.b, run: { ...run, run: "f".repeat(32) } }); + expect((await load(f.b))?.run).toBe("f".repeat(32)); + await expectClosedRefusal(load(f.a), "another native home"); +}); + +test("a live owner in another home is refused by name and never released", async () => { + const f = await fixture("feature-api"); + await startInHomeA(f, false); + const inspection = await f.inspect(); + expect(inspection).toMatchObject({ + state: "foreign-home", + ownerHome: f.homeA, + owner: "live", + ownerPid: process.pid, + selectionSha256: null, + }); + const live = `Native project run mapping is owned by live run ${run.run} in another native home (${f.homeA}), frontend pid ${process.pid}. Stop it first: press Ctrl-C in that hack up, or run hack down with the Hack version that owns that home; then retry here.`; + await expectClosedRefusal(load(f.b), live); + await expectClosedRefusal( + release({ scope: f.b, expectSelection: "e".repeat(64) }), + live + ); + expect(await load(f.a)).toEqual(run); +}); + +test("an interrupted or unrecorded owner is named but cannot be released", async () => { + const f = await fixture(); + await startInHomeA(f, false); + expect(await f.inspect(() => false)).toMatchObject({ + owner: "interrupted", + ownerPid: process.pid, + selectionSha256: null, + }); + + const g = await fixture(); + await save({ ...g.a, run }); + expect(await g.inspect()).toMatchObject({ + owner: "unproven", + selectionSha256: null, + }); + await expectClosedRefusal( + load(g.b), + `Native project run mapping belongs to run ${run.run} in another native home (${g.homeA}), and its clean stop cannot be proven from this home. Before using this home, run hack down with the Hack version that owns that home.` + ); + await expectClosedRefusal( + release({ scope: g.b, expectSelection: "e".repeat(64) }), + "cannot be proven" + ); + expect(await load(g.a)).toEqual(run); +}); + +test("an explicitly recovered dead owner does not attest a clean stop", async () => { + const f = await fixture(); + const lifetime = await startInHomeA(f, false); + await recoverNativeProjectFinalization({ + scope: f.a, + run, + token: lifetime.token, + expectAttempt: lifetime.token.attempt, + verifyEffects: async () => undefined, + isDead: () => true, + }); + expect(await f.inspect(() => false)).toMatchObject({ + owner: "unproven", + selectionSha256: null, + }); +}); + +test("a retained startup admission in the owning home excludes release until retired", async () => { + const f = await fixture(); + await startInHomeA(f, true); + const selection = (await f.inspect()).selectionSha256 ?? ""; + const admission = await admitNativeProjectFinalization({ scope: f.a, run }); + expect(await f.inspect()).toMatchObject({ + owner: "live", + ownerPid: process.pid, + selectionSha256: null, + }); + await expectClosedRefusal( + release({ scope: f.b, expectSelection: selection }), + "is owned by live run" + ); + expect(await load(f.a)).toEqual(run); + // An admission whose startup exited without registering a lifetime is interrupted. + expect(await f.inspect(() => false)).toMatchObject({ + owner: "interrupted", + ownerPid: process.pid, + }); + await admission.retire(); + await admission.retire(); + expect((await f.inspect()).selectionSha256).toBe(selection); + expect( + (await release({ scope: f.b, expectSelection: selection })).released + ).toBe(true); + // A startup admitted after the release finds no mapping at its last check. + const late = await admitNativeProjectFinalization({ scope: f.a, run }); + expect(await load(f.a)).toBeNull(); + await late.retire(); +}); + +test("a selection goes stale when the owning home starts the run again", async () => { + const f = await fixture(); + await startInHomeA(f, true); + const stale = (await f.inspect()).selectionSha256 ?? ""; + // Home A resumes the retained run: a new lifetime begins after the completed one. + await beginNativeProjectFinalization({ scope: f.a, run, httpsPort: null }); + await expectClosedRefusal( + release({ scope: f.b, expectSelection: stale }), + "owned by live run" + ); + expect(await load(f.a)).toEqual(run); +}); + +test("a replaced owning home is unproven even with completed records copied in", async () => { + const f = await fixture(); + await startInHomeA(f, true); + await rename(f.homeA, `${f.homeA}-moved`); + await mkdir(f.homeA, { mode: 0o700 }); + await rename( + join(`${f.homeA}-moved`, ".hack-local"), + join(f.homeA, ".hack-local") + ); + expect(await f.inspect()).toMatchObject({ + owner: "unproven", + selectionSha256: null, + }); +}); + +test("pending down hooks, held locks and the owning home's lock refuse release", async () => { + const f = await fixture(); + await startInHomeA(f, true); + const selection = (await f.inspect()).selectionSha256 ?? ""; + const mapping = + (await readdir(f.runs)).find((name) => /^[a-f0-9]{64}\.json$/.test(name)) ?? + "absent.json"; + const base = join(f.runs, mapping.slice(0, -5)); + + await writeFile(`${base}.down-hooks.json`, "{}", { mode: 0o600 }); + await expectClosedRefusal( + release({ scope: f.b, expectSelection: selection }), + "down hooks that are owed, unknown or not this run's" + ); + await rm(`${base}.down-hooks.json`); + + await mkdir(`${base}.lock`); + await expect( + release({ scope: f.b, expectSelection: selection }) + ).rejects.toThrow(); + await rm(`${base}.lock`, { recursive: true }); + + const finalization = join(f.homeA, ".hack-local", "frontend-finalization"); + const [scopeHash] = await readdir(finalization); + const ownerLock = join( + finalization, + scopeHash ?? "absent", + run.run, + "operation.lock" + ); + await mkdir(ownerLock); + await expect( + release({ scope: f.b, expectSelection: selection }) + ).rejects.toThrow(); + await rm(ownerLock, { recursive: true }); + + expect(await load(f.a)).toEqual(run); + // No release lock or audit was left behind by the refusals. + expect( + (await readdir(f.runs)).filter( + (name) => name.endsWith(".lock") || name.includes("operation") + ) + ).toEqual([]); + expect( + (await release({ scope: f.b, expectSelection: selection })).released + ).toBe(true); +}); + +/** A down-hook journal record exactly as the journal writes it. */ +async function downHookJournal( + f: Awaited>, + record: { readonly run: string; readonly state: string } +) { + const mapping = + (await readdir(f.runs)).find((name) => /^[a-f0-9]{64}\.json$/.test(name)) ?? + "absent.json"; + const path = join(f.runs, `${mapping.slice(0, -5)}.down-hooks.json`); + await rm(path, { force: true }); + await writeFile( + path, + JSON.stringify({ + version: 1, + kind: "hack.native-project-down-hooks", + run: record.run, + generation: "9".repeat(32), + state: record.state, + }), + { mode: 0o600 } + ); + return path; +} + +test("a completed down-hook journal of the selected run does not block release", async () => { + const f = await fixture(); + await startInHomeA(f, true); + const selection = (await f.inspect()).selectionSha256 ?? ""; + // The journal's own writer, as a successful stop leaves it. + expect( + await replaceNativeDownHookRecord(f.a, null, { + run: run.run, + generation: "9".repeat(32), + state: "complete", + }) + ).toBe(true); + const journal = join( + f.runs, + (await readdir(f.runs)).find((name) => name.endsWith(".down-hooks.json")) ?? + "absent" + ); + const released = await release({ scope: f.b, expectSelection: selection }); + expect(released.released).toBe(true); + expect(await load(f.b)).toBeNull(); + // The journal is the owning home's history; release does not touch it. + expect(JSON.parse(await readFile(journal, "utf8")).state).toBe("complete"); +}); + +test("an owed, unknown or another run's down-hook journal refuses release", async () => { + for (const record of [ + { run: "f".repeat(32), state: "complete" }, + { run: run.run, state: "owed" }, + { run: run.run, state: "after-owed" }, + { run: run.run, state: "before-started" }, + { run: run.run, state: "after-started" }, + { run: run.run, state: "before-failed" }, + { run: run.run, state: "settled" }, + ]) { + const f = await fixture(); + await startInHomeA(f, true); + const selection = (await f.inspect()).selectionSha256 ?? ""; + await downHookJournal(f, record); + await expectClosedRefusal( + release({ scope: f.b, expectSelection: selection }), + "down hooks that are owed, unknown or not this run's" + ); + expect(await load(f.a)).toEqual(run); + } +}); + +test("a mapping that differs in more than its native home keeps the generic refusal", async () => { + const f = await fixture(); + await startInHomeA(f, true); + // Replace the project directory: its identity no longer matches the mapping. + await rename(f.a.projectDir, `${f.a.projectDir}-old`); + await mkdir(f.a.projectDir); + await rename( + join(`${f.a.projectDir}-old`, ".internal"), + join(f.a.projectDir, ".internal") + ); + await expectClosedRefusal( + load(f.b), + "Native project run mapping is unsafe, changed, or owned by another run; inspect native state before recovery." + ); +}); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index b99ece5de..75b415864 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -18,6 +18,7 @@ import { NativeHttpsStartupError } from "../src/backends/native-https-startup-fa import { NATIVE_IMAGE_ENSURE_TIMEOUT_MS } from "../src/backends/native-image-policy.ts"; import { discardUnpublishedNativeStart } from "../src/backends/native-project-failed-start.ts"; import { + admitNativeProjectFinalization, beginNativeProjectFinalization, captureNativeProjectFinalization, waitNativeProjectFinalization, @@ -33,8 +34,11 @@ import { } from "../src/backends/native-project-restart-preflight.ts"; import { selectNativeRetainedImages } from "../src/backends/native-project-retained-images.ts"; import { + inspectNativeProjectRunMapping, loadNativeProjectRun, type NativeProjectRun, + type NativeProjectRunForeignHomeInspection, + releaseNativeProjectRunForeignHome, saveNativeProjectRun, } from "../src/backends/native-project-run.ts"; import { @@ -3185,6 +3189,89 @@ test("retained startup admits only the acknowledged original frontend before eff expect(events).toContain("save"); }); +/** + * HACK-1952: a retained startup in the owning home and a release of its mapping from + * another native home must not interleave. The startup is admitted under the owning + * home's finalization lock before its last mapping check and any runtime effect. + */ +async function retainedStartWithAnotherHome() { + const fixture = await retainedFinalizationFixture(); + const { opts, saved } = fixture; + await saveNativeProjectRun({ ...opts.scope, run: saved }); + await acknowledgeRetainedFrontend(opts.scope, saved); + opts.dependencies.load = loadNativeProjectRun; + const otherHome = await mkdtemp(join(tmpdir(), "native-start-home-b-")); + roots.push(otherHome); + const other = { ...opts.scope, nativeHome: otherHome }; + const inspect = async () => + (await inspectNativeProjectRunMapping({ + scope: other, + runtime: { binary: join(otherHome, "absent-runtime"), home: otherHome }, + })) as NativeProjectRunForeignHomeInspection; + const stopped = await inspect(); + expect(stopped.owner).toBe("stopped"); + const release = () => + releaseNativeProjectRunForeignHome({ + scope: other, + expectSelection: stopped.selectionSha256 ?? "", + }); + const admissions = async () => + ( + await readdir(join(opts.scope.nativeHome, ".hack-local"), { + recursive: true, + }) + ).filter((name) => name.endsWith("admission.json")); + return { ...fixture, inspect, release, admissions }; +} + +test("a release from another home refuses while the owning home admits a retained startup", async () => { + const { opts, events, saved, inspect, release, admissions } = + await retainedStartWithAnotherHome(); + opts.dependencies.admitFinalization = async (selection) => { + const admitted = await admitNativeProjectFinalization(selection); + // Interleave: the other home tries to release right after admission. + expect((await inspect()).owner).toBe("live"); + await expectClosedRefusal(release(), "is owned by live run"); + events.push("release refused"); + return admitted; + }; + const invoke = opts.dependencies.invoke!; + opts.dependencies.invoke = async (request) => { + if (request.args[0] === "runtime" && request.args[1] === "up") { + // Runtime effects begin only while the admission still excludes release. + expect(await admissions()).toHaveLength(1); + await expectClosedRefusal(release(), "is owned by live run"); + } + return await invoke(request); + }; + expect(await startNativeProject(opts)).toBe(0); + expect(events.indexOf("release refused")).toBeLessThan( + events.indexOf("runtime up") + ); + // Readiness registered the new lifetime and retired the admission. + expect(await admissions()).toEqual([]); + expect(await loadNativeProjectRun(opts.scope)).toEqual(saved); +}); + +test("a release that wins before admission makes the retained startup refuse before runtime effects", async () => { + const { opts, events, release, admissions } = + await retainedStartWithAnotherHome(); + opts.dependencies.admitFinalization = async (selection) => { + // Interleave: the other home releases just before this startup is admitted. + expect((await release()).released).toBe(true); + events.push("released"); + return await admitNativeProjectFinalization(selection); + }; + await expect(startNativeProject(opts)).rejects.toThrow( + "Native retained run mapping changed before runtime start; no start was requested." + ); + expect(events).toContain("released"); + expect(events).not.toContain("runtime up"); + // A startup that ends before any runtime effect leaves no admission behind. + expect(await admissions()).toEqual([]); + expect(await loadNativeProjectRun(opts.scope)).toBeNull(); +}); + test("retained acknowledgement cannot authorize a newer or concurrently changed mapping", async () => { for (const fault of ["newer-run", "changed-mapping"] as const) { const { opts, events, saved } = await retainedFinalizationFixture(); diff --git a/tests/native-refusal-cli.test.ts b/tests/native-refusal-cli.test.ts index 8eafbc833..c91c2b66d 100644 --- a/tests/native-refusal-cli.test.ts +++ b/tests/native-refusal-cli.test.ts @@ -569,3 +569,87 @@ test( }, SPAWN_TIMEOUT_MS ); + +/** HACK-1952: home A stopped the run cleanly; home B is another launcher version's home. */ +async function stoppedInAnotherHome(f: Fixture, stop = true) { + const homeB = join(f.root, "native-home-b"); + await mkdir(homeB, { mode: 0o700 }); + await savedRun(f); + const lifetime = await beginNativeProjectFinalization({ + scope: f.scope, + run, + httpsPort: null, + }); + if (stop) { + await lifetime.complete(); + } + return { HACK_NATIVE_HOME: homeB, ...SHARED }; +} + +test( + "native-project-run: another home's stopped mapping names the release, which then frees this home", + async () => { + const f = await fixture(); + const homeB = await stoppedInAnotherHome(f); + const inspected = await cli({ + fixture: f, + args: ["doctor", "--native-run-mapping", "inspect", "--json"], + env: homeB, + }); + expect(inspected.code).toBe(0); + const inspection = JSON.parse(inspected.stdout).data; + expect(inspection).toMatchObject({ + state: "foreign-home", + owner: "stopped", + ownerHome: f.scope.nativeHome, + }); + const selection: string = inspection.selectionSha256; + expect(selection).toMatch(/^[a-f0-9]{64}$/); + for (const args of [ + ["ps"], + ["up"], + ["down"], + ["doctor", "--native-cleanup", "inspect"], + ]) { + expectOneClosedLine( + await cli({ fixture: f, args, env: homeB }), + `Native project run mapping belongs to run ${run.run} in another native home (${f.scope.nativeHome}), which stopped it cleanly; its retained data stays in that home. To start this instance from this native home, release the mapping with hack doctor --native-run-mapping release --expect-selection ${selection}, or run hack down with the Hack version that owns that home.` + ); + } + const released = await cli({ + fixture: f, + args: [ + "doctor", + "--native-run-mapping", + "release", + "--expect-selection", + selection, + ], + env: homeB, + }); + expect(released.code).toBe(0); + expect(released.stdout).toContain( + "Native run mapping released from another native home" + ); + const down = await cli({ fixture: f, args: ["down"], env: homeB }); + expect(down.output).not.toContain("run mapping"); + expect(down.code).toBe(0); + }, + SPAWN_TIMEOUT_MS * 2 +); + +test( + "native-project-run: another home's live owner is one closed line naming it", + async () => { + const f = await fixture(); + const homeB = await stoppedInAnotherHome(f, false); + for (const args of [["ps"], ["up"], ["down"]]) { + expectOneClosedLine( + await cli({ fixture: f, args, env: homeB }), + `Native project run mapping is owned by live run ${run.run} in another native home (${f.scope.nativeHome}), frontend pid ${process.pid}. Stop it first: press Ctrl-C in that hack up, or run hack down with the Hack version that owns that home; then retry here.` + ); + } + expect(await f.calls()).toEqual([]); + }, + SPAWN_TIMEOUT_MS * 2 +);