From 7e002128d50992cf7af28991e801747979fc43ee Mon Sep 17 00:00:00 2001 From: norvalbv Date: Fri, 20 Nov 2026 02:33:37 +0000 Subject: [PATCH 1/2] devkit's CLI exits by draining the event loop, so Node 24's process.exit deadlock no longer hangs a finished command (sc-8775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why `prove-regression.test.mts` cases sometimes failed with "testSpawnSync: 90s harness deadline fired". The story put it down to missing headroom, but each case takes 1–7s. The deadline was catching a real hang. I captured a stalled case under load with a `ps` snapshot and `/usr/bin/sample` of the CLI. The CLI had printed its result and reached its `process.exit` exit event 1.2s in. It then hung for 88s inside Node itself: - main thread: `Environment::Exit` → `DisposePlatform` → `WorkerThreadsTaskRunner::Shutdown` → `pthread_join` - V8Worker: concurrent Sparkplug `BaselineCompiler::Build` → `CollectionBarrier::AwaitCollectionBackground`, waiting for a GC that the exiting main thread never runs This is a known Node 24 bug, [nodejs/node#64274](https://github.com/nodejs/node/issues/64274). The fix is still pending upstream in nodejs/node#66171, and the repro is nodejs/node#67596. devkit's dispatcher ended every command with `process.exit`, so any `devkit ` could hang the same way, inside a consumer hook or a ship as well as in our own suite. ## What - `exitWhenDrained(code, graceMs = 1000)` in `gate-engine/deterministic/reason.mts` sets `process.exitCode` and lets the event loop drain. A normal exit disposes the isolate before the platform, so it never deadlocks. An unref'd timer forces `process.exit` only when a handle is still holding the loop, by which point the loop has serviced any pending GC request. - `cli/index.mts` ends both the returned path and the thrown path with it. `stdoutDrained` still runs first, to bound a slow pipe reader on the forced path. - New decision Target `cli-exit-drains-event-loop`. Its Ruling states the one carve-out: a throw during a gate run (`GATE_REASON_ENV`) keeps `crashExit`'s immediate exit, which other gates' crash preload shares. - `cli/lib/ship/run-packaged-script.mts`: a comment that said the CLI calls `process.exit()` now says `exitWhenDrained`. The behaviour it describes is unchanged: the signal handlers stay registered through the drain. The 90s deadline and the vitest project placement are unchanged. `test-deadlines-are-hang-detectors` holds, and this deadline did its job. ## Proof | Arm | `process.exit` (before) | drain (after) | |---|---|---| | prove-regression.test.mts, 20 file runs (5 rounds × 4 concurrent), ~560 CLI calls | 1 stall (90.4s, 124), 1 file failed | 0 stalls, 20/20 passed | | nodejs/node#67596 repro, 8-way parallel, Node v24.19.0 | 2/400 hung | 0/400 hung | - File duration is unchanged, 42–51s per run in the after arm. Spot checks of `base-status`, `guard-branch`, `doctor` and `ci-verdict` never reached the 1s forced path. - Revert check: with `process.exit` restored, the 3 new tests fail. Two `cli-exit.test.mts` cases fail on "process.exit called". The `reason.test.mts` case fails with stdout cut at 65536 of 5000000 bytes. All of them pass with the change. - `ship --dry-gates` passed, including related-tests (155 files, 4092 tests). ## Follow-ups (not in this PR) - The `process.exit` calls before dispatch in `cli/index.mts` (`--version`, `help`, unknown command, ` --help`): bug, sev 3, traced. - `crashExit` still calls `process.exit` on the gate crash path: bug, sev 3, traced. - About 220 other `process.exit(` calls in 79 gate and command files (`exitGate`, guard-* CLIs): bug, sev 2, claimed. - sc-6032 (existing): have gate-supervisor report its own deadline instead of inferring it. - Check every command for open handles that pay the 1s fallback: chore. - Async work or a late promise rejection running during the drain window: bug, sev 3, claimed. - `dist/` is rebuilt at the next `devkit release`: chore. Fixes [sc-8775] 🤖 Generated with [Claude Code](https://claude.com/claude-code) Complete against acceptance criteria: yes --- cli/__tests__/cli-exit.test.mts | 41 +++++++++++++++++++ cli/index.mts | 11 +++-- cli/lib/ship/run-packaged-script.mts | 2 +- docs/decisions/INDEX.md | 1 + docs/decisions/cli-exit-drains-event-loop.md | 21 ++++++++++ .../deterministic/__tests__/reason.test.mts | 27 +++++++++++- gate-engine/deterministic/reason.mts | 10 +++++ 7 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 cli/__tests__/cli-exit.test.mts create mode 100644 docs/decisions/cli-exit-drains-event-loop.md diff --git a/cli/__tests__/cli-exit.test.mts b/cli/__tests__/cli-exit.test.mts new file mode 100644 index 000000000..7e41ce85b --- /dev/null +++ b/cli/__tests__/cli-exit.test.mts @@ -0,0 +1,41 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { CLI, testSpawnSync } from './_helpers.mts'; + +// Never called: declares the spawned modules to vitest's graph, so `vitest related` selects this file. +void (() => import('../../gate-engine/deterministic/reason.mts')); +void (() => import('../commands/baseline/prove-regression.mts')); + +const dir = mkdtempSync(join(tmpdir(), 'cli-exit-')); +const spy = join(dir, 'exit-spy.mjs'); +writeFileSync( + spy, + "const exit = process.exit; process.exit = (code) => { process.stderr.write('process.exit called\\n'); return exit(code); };\n", +); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +const run = (args: string[], env: NodeJS.ProcessEnv = process.env) => + testSpawnSync(process.execPath, ['--import', spy, CLI, ...args], { encoding: 'utf8', env }); + +// Node 24's process.exit can deadlock in platform shutdown (nodejs/node#64274), so a returning +// command must end by draining its event loop instead. +describe('devkit exit', () => { + it('ends a returning command with its code without calling process.exit', () => { + const result = run(['prove-regression']); + + expect(result.status, result.stderr).toBe(1); + expect(result.stderr).toContain('prove-regression usage'); + expect(result.stderr).not.toContain('process.exit called'); + }); + + it('ends a command that throws with 1 without calling process.exit', () => { + // An empty PATH leaves git unresolvable, so the dispatcher's git check throws out of main. + const result = run(['prove-regression'], { ...process.env, PATH: '/var/empty' }); + + expect(result.status, result.stderr).toBe(1); + expect(result.stderr).toMatch(/git is not installed or not on PATH/); + expect(result.stderr).not.toContain('process.exit called'); + }); +}); diff --git a/cli/index.mts b/cli/index.mts index 039aad0f9..c195d323f 100755 --- a/cli/index.mts +++ b/cli/index.mts @@ -17,7 +17,12 @@ * Consumers install prebuilt .mjs — no build on their side. */ import { readFileSync } from 'node:fs'; -import { crashExit, GATE_REASON_ENV, stdoutDrained } from '../gate-engine/deterministic/reason.mts'; +import { + crashExit, + exitWhenDrained, + GATE_REASON_ENV, + stdoutDrained, +} from '../gate-engine/deterministic/reason.mts'; import { assertGit } from './lib/guard/require-git.mts'; import { type CommandMeta, renderCommandHelp, renderTopLevelHelp } from './lib/help/render.mts'; import { wantsCommandHelp } from './lib/help/wants-help.mts'; @@ -183,12 +188,12 @@ async function main() { : process.cwd(); const code = await mod.default(cmdArgs, cwd); await stdoutDrained(); - process.exit(code ?? 0); + exitWhenDrained(code ?? 0); } main().catch((e) => { // Run as a gate (anti-slop), a throw is a crash, never the exit 1 that means a finding. if (process.env[GATE_REASON_ENV] !== undefined) crashExit(e); console.error(`devkit: ${e.message}`); - process.exit(1); + exitWhenDrained(1); }); diff --git a/cli/lib/ship/run-packaged-script.mts b/cli/lib/ship/run-packaged-script.mts index ed92c3613..ca552c697 100644 --- a/cli/lib/ship/run-packaged-script.mts +++ b/cli/lib/ship/run-packaged-script.mts @@ -100,7 +100,7 @@ export function runManagedPackagedScript( finished = true; resolve(status); }; - // The CLI calls process.exit() immediately after awaiting this promise. Keep the handlers and + // The CLI ends through exitWhenDrained after awaiting this promise. Keep the handlers and // finalization lock alive through that boundary so a last-moment signal cannot change only the // process status after Bash has emitted terminal telemetry. // Do not unregister the signal handlers in the exit listener: native termination happens only diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 3a8d70e07..2fa3e296b 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -20,6 +20,7 @@ timeline. New rationale lives in the per-axis file. | [ci-emits-per-file-test-results](ci-emits-per-file-test-results.md) | Fix the PRODUCER, not the reader. 'devkit test-report-run' runs vitest with default + github-actions + json reporters and emits two artifacts: the full report and a few-KB per-file summary, uploaded separately with if: always(). 'devkit baseline-status' answers from the summary and NEVER parses log text. It reports runStatus (the whole run) separately from testsStatus (the test step), because a red run whose tests passed is devkit's actual steady state. Per-file answers split into passed/failed/skipped/excluded/absent/unknown; gh is a SOFT dependency degrading to a named unknown per the detectMerged precedent in cli/lib/reconcile.mts. The cache keys on runId+attempt, never on the commit sha. lastPassed is forward-facing: it reports no-artifact-history rather than implying a file never passed. | sc-2245. devkit's gate has been red on main continuously (sc-1896, … | 2026-08-29 | | [ci-merge-gate-one-aggregate-check](ci-merge-gate-one-aggregate-check.md) | main's merge gate is one aggregate job, required, in gate.yml. It needs every other job in the workflow, runs if: always(), and passes only when its needs are non-empty and every one concluded success. It is the only required status check in main's ruleset, switched on by hand once main has 3 consecutive green push runs and the first of them shows required: success. A job added to gate.yml joins its needs list; a contract test fails otherwise. | main had no required status check: ruleset 19675205 carried only de… | 2026-10-10 | | [cli-cross-process-lock](cli-cross-process-lock.md) | gate-engine/judge/process/process-lock.mts (withProcessLock, reapIfDead, LockHeldError) is devkit's one cross-process mutex. The lock is a directory renamed into place already holding one file NAMED by the holder's nonce (pid + OS start identity), so it is never ownerless. A holder is reaped only when processOwnerIsProvablyGone proves it dead, by renaming exactly / away then rmdir (atomic compare-and-delete); there is NO age-based reap, and release removes only its own nonce. The critical section is synchronous; a nested acquire of a lock the same process holds fails at once with LockHeldError; a contended acquire waits (5 s default, liveness re-checked every 250 ms) then throws LockHeldError naming the holder pid. Staged and grave siblings end in .lock so a consumer's .devkit/*.lock ignore hides crash leftovers. The ship queue (slot.lock, seq.lock) and the correctness-overrides store lock (.mutex.lock) run on it; a new cross-process lock uses it or records here why it cannot. | devkit carried several hand-rolled cross-process locks with differe… | 2026-10-18 | +| [cli-exit-drains-event-loop](cli-exit-drains-event-loop.md) | cli/index.mts ends a dispatched command, returned or thrown, through exitWhenDrained (gate-engine/deterministic/reason.mts): it sets process.exitCode and lets the event loop drain, which disposes the isolate before the platform, and an unref'd 1s timer calls process.exit only when a handle still holds the loop. stdoutDrained stays ahead of it to bound a slow pipe reader on that forced path. A throw under a gate run (GATE_REASON_ENV set) keeps crashExit's immediate exit, which the crash preload of other gates shares. | On Node 24, process.exit() can deadlock forever: platform shutdown … | 2026-11-20 | | [clone-approvals-expire-on-content-not-calendar](clone-approvals-expire-on-content-not-calendar.md) | guard-dup-allowlist add-clone writes an explicit 3650-day lifetime, the same value baseline-clones uses to freeze pre-existing clones, unless the approver passes --decay-days N. A clone entry is keyed on a hash of the duplicated fragment, so an edit to that code already ends the approval; the calendar is therefore an opt-in temporary waiver for clones and not the default. Symbol-pair approvals written by add keep the 7-day default, because a pair is keyed on symbol and file names and nothing else ends it when the code drifts. The shared default in the decay module is unchanged, so clone entries written before this ruling with no lifetime stay expired and are never silently revived. The gate's block output and the CLI usage text state the lifetime each verb buys. | A reasoned approval of deliberate duplication lapsed 7 days after i… | 2026-10-14 | | [clone-gate-head-novelty](clone-gate-head-novelty.md) | The clone gate runs jscpd at an exact pinned version (5.4.1), once, with --similarity at thresholds.similarity (default 0.9) and --min-lines 3 beside token mode. With --changed it passes --baseline-from-ref HEAD and blocks only clones jscpd marks new that touch a staged file; on an unborn HEAD every clone is new. An allowlist entry approves the clone a commit introduces, keyed on the file pair plus the fragment. Pre-existing clones are not tracked in the allowlist: guard-clone scan lists clone debt, and refactoring retires it. A jscpd other than the pinned version is a named could-not-run. | A 3-7 line helper copied under a new name passed every deterministi… | 2026-10-16 | | [clone-gate-non-import-code](clone-gate-non-import-code.md) | Clone detection runs jscpd --mode weak, so comments, like import/re-export declarations (excluded via four bounded, comma-free --ignore-pattern regexes; dynamic import(...) matches none), never enter the token stream: the min-tokens floor counts non-import, non-comment code. Post-hoc classification of reported fragments stays banned on this axis | jscpd's mild mode keeps comment tokens in the hashed stream, and ev… | 2026-10-16 | diff --git a/docs/decisions/cli-exit-drains-event-loop.md b/docs/decisions/cli-exit-drains-event-loop.md new file mode 100644 index 000000000..bfb87f456 --- /dev/null +++ b/docs/decisions/cli-exit-drains-event-loop.md @@ -0,0 +1,21 @@ +--- +slug: cli-exit-drains-event-loop +created: 2026-11-20 +--- + +# cli-exit-drains-event-loop + +## Target · 2026-11-20 — devkit's CLI ends by draining the event loop, not process.exit + +**Context:** On Node 24, process.exit() can deadlock forever: platform shutdown joins V8's background compile threads while the isolate is alive, and a Sparkplug or Maglev job parked waiting for a main-thread GC never wakes (nodejs/node#64274, fix pending in nodejs/node#66171). devkit's dispatcher ended every command with process.exit, so a finished command could hang at 0% CPU. prove-regression.test.mts saw this as a 90s harness deadline about once per 500 CLI calls under load, the CLI having printed its result 1s in; the sampled stack was main thread in WorkerThreadsTaskRunner::Shutdown, V8Worker in CollectionBarrier::AwaitCollectionBackground. A consumer hook or ship that runs devkit can hang the same way. +**Ruling:** cli/index.mts ends a dispatched command, returned or thrown, through exitWhenDrained (gate-engine/deterministic/reason.mts): it sets process.exitCode and lets the event loop drain, which disposes the isolate before the platform, and an unref'd 1s timer calls process.exit only when a handle still holds the loop. stdoutDrained stays ahead of it to bound a slow pipe reader on that forced path. A throw under a gate run (GATE_REASON_ENV set) keeps crashExit's immediate exit, which the crash preload of other gates shares. +**Consequences:** +- Positive: A command that has finished exits with its own code instead of wedging until a supervisor's deadline, so devkit stops producing load-only 124s in its own suite and in consumer hooks. +- Negative: A command that leaves a ref'd handle (a timer, an unreaped child, an open socket) now runs up to 1s longer before the forced exit, and in-flight async work keeps running during that second instead of being cut. The forced path still calls process.exit, but only after the loop has run idle for 1s and serviced any pending GC request. Not migrated yet: the pre-dispatch exits in cli/index.mts (--version, help, unknown command, --help), crashExit, and about 220 other process.exit calls in gate and command code. +**Vision-fit:** n/a - internal tooling; devkit runs inside consumer commits, where a hung exit blocks the commit. +**Researched:** nodejs/node#64274, #67596 (repro and stacks), #54918, #66171 (proposed fix); downstream mitigations in cloudflare/nimbus#382 and nrwl/nx#36726. Local repro of #67596 under 8-way load on Node 24.19.0: 2/400 hung with process.exit, 0/400 with exitCode plus unref'd fallback. +**Rejected:** (a) Raise the 90s test deadline or move the file to another vitest project - REJECTED: the deadline caught a real hang (test-deadlines-are-hang-detectors), and both projects share one pool and deadline. (b) Disable concurrent Sparkplug/Maglev with V8 flags - REJECTED: devkit does not own the node command line in a consumer, and v8.setFlagsFromString after startup is unsupported. (c) Pin Node 22 - REJECTED: consumers choose their Node. +**Anchored-bet:** [VALIDATED] +**Revisit-when:** Every Node major devkit supports carries the nodejs/node#66171 fix, at which point process.exit is safe again. +**Scope:** cli/index.mts,gate-engine/deterministic/reason.mts +**Source:** web · https://github.com/nodejs/node/issues/64274 diff --git a/gate-engine/deterministic/__tests__/reason.test.mts b/gate-engine/deterministic/__tests__/reason.test.mts index 6a83b75a6..c55d846d9 100644 --- a/gate-engine/deterministic/__tests__/reason.test.mts +++ b/gate-engine/deterministic/__tests__/reason.test.mts @@ -1,4 +1,4 @@ -import { execFileSync, spawn } from 'node:child_process'; +import { execFileSync, spawn, spawnSync } from 'node:child_process'; import { once } from 'node:events'; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -297,3 +297,28 @@ describe('stdoutDrained', () => { expect(code).toBe(0); }); }); + +describe('exitWhenDrained', () => { + const REASON = join(import.meta.dirname, '..', 'reason.mts'); + const child = (body: string) => + spawnSync( + process.execPath, + [ + '--input-type=module', + '-e', + `import { exitWhenDrained } from ${JSON.stringify(REASON)}; ${body}`, + ], + { maxBuffer: 1e8 }, + ); + + it('ends with its code once every queued stdout byte reached the pipe', () => { + const result = child("process.stdout.write('x'.repeat(5e6)); exitWhenDrained(3);"); + expect(result.status).toBe(3); + expect(result.stdout.length).toBe(5e6); + }); + + it('forces the exit with its code when a handle keeps the event loop alive', () => { + const result = child('setInterval(() => {}, 1000); exitWhenDrained(4, 50);'); + expect(result.status).toBe(4); + }); +}); diff --git a/gate-engine/deterministic/reason.mts b/gate-engine/deterministic/reason.mts index 50e09f7e8..cc638e7fe 100644 --- a/gate-engine/deterministic/reason.mts +++ b/gate-engine/deterministic/reason.mts @@ -97,6 +97,16 @@ export function exitGate(code: number, lines: readonly string[]): never { /** Bound on waiting for stdout: a reader that never drains the pipe must not hang the exit. */ const DRAIN_MS = 10_000; +/** How long a CLI whose event loop still holds a handle may run before it is forced out. */ +const EXIT_GRACE_MS = 1_000; + +/** Ends with `code` when the event loop drains, forcing process.exit only after `graceMs`. Node 24's + * process.exit can deadlock joining a V8 compile thread parked on a GC (nodejs/node#64274). */ +export function exitWhenDrained(code: number, graceMs = EXIT_GRACE_MS): void { + process.exitCode = code; + setTimeout(() => process.exit(code), graceMs).unref(); +} + /** Resolves once stdout has handed every queued write to the OS, or after `ms`. process.exit drops * a pipe's unflushed tail (about 64 KB survives on macOS), so a piped --json answer would be cut. */ export function stdoutDrained(ms = DRAIN_MS): Promise { From 720d019bb42e377891f3805a8111abcf5cc9b7fd Mon Sep 17 00:00:00 2001 From: norvalbv Date: Fri, 20 Nov 2026 02:47:14 +0000 Subject: [PATCH 2/2] devkit's CLI exits by draining the event loop, so Node 24's process.exit deadlock no longer hangs a finished command (sc-8775) The forced exit uses the current process.exitCode, as a natural drain does (sc-8775) exitWhenDrained's 1s fallback now calls process.exit() with no argument, so a process.exitCode written after the command returned wins on both the natural-drain and the forced path. A reason.test.mts case holds a handle open, writes exitCode 5 after exitWhenDrained(0), and expects status 5. Co-Authored-By: Claude Opus 5.5 --- gate-engine/deterministic/__tests__/reason.test.mts | 7 +++++++ gate-engine/deterministic/reason.mts | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/gate-engine/deterministic/__tests__/reason.test.mts b/gate-engine/deterministic/__tests__/reason.test.mts index c55d846d9..593285deb 100644 --- a/gate-engine/deterministic/__tests__/reason.test.mts +++ b/gate-engine/deterministic/__tests__/reason.test.mts @@ -321,4 +321,11 @@ describe('exitWhenDrained', () => { const result = child('setInterval(() => {}, 1000); exitWhenDrained(4, 50);'); expect(result.status).toBe(4); }); + + it('honours a later exitCode write on the forced path, as a natural drain does', () => { + const result = child( + 'setInterval(() => {}, 1000); exitWhenDrained(0, 200); setTimeout(() => { process.exitCode = 5; }, 20);', + ); + expect(result.status).toBe(5); + }); }); diff --git a/gate-engine/deterministic/reason.mts b/gate-engine/deterministic/reason.mts index cc638e7fe..f6eac1668 100644 --- a/gate-engine/deterministic/reason.mts +++ b/gate-engine/deterministic/reason.mts @@ -104,7 +104,7 @@ const EXIT_GRACE_MS = 1_000; * process.exit can deadlock joining a V8 compile thread parked on a GC (nodejs/node#64274). */ export function exitWhenDrained(code: number, graceMs = EXIT_GRACE_MS): void { process.exitCode = code; - setTimeout(() => process.exit(code), graceMs).unref(); + setTimeout(() => process.exit(), graceMs).unref(); } /** Resolves once stdout has handed every queued write to the OS, or after `ms`. process.exit drops