Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .devkit/baselines/size-lines.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"gate-engine/decisions/detect.mts": 527,
"gate-engine/decisions/eval/bench.mts": 806,
"gate-engine/edge-cases/eval/lib/sources.mts": 570,
"gate-engine/review/__tests__/run-review.test.mts": 2276,
"gate-engine/review/__tests__/run-review.test.mts": 2272,
"gate-engine/review/eval/bench.mts": 657,
"gate-engine/review/eval/conventions/bench.mts": 887,
"gate-engine/review/eval/reviewers/bench.mts": 710,
Expand Down
1 change: 1 addition & 0 deletions docs/decisions/review-gate-in-chain.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,4 @@ created: 2026-07-02
- 2026-10-25 — Supersedes the whitespace-only quoting in the previous note: a checklist script path holding any shell metacharacter (space, $, backtick, either quote) is now single-quoted through shellWord, in the grant and in every printed command alike. Probed on Claude Code 2.1.302: each such path ran verbatim with no denial, and $HOME stayed literal. Double quotes would have let the shell expand $ and backticks, so the judge ran a different path from the one it was granted.
- 2026-11-17 — When status, check-item or finalize finds no correctness checklist, the script names the path it read and lists the non-empty lens checklists present as --lens values (a lensed call sees only its own group, so the hint cannot steer one parallel judge into another's file), telling the judge to repeat the command with the flags its prompt shows. It never auto-selects a group, since parallel lens judges share one worktree, and never lists chunk indices or the un-lensed file: chunks share item names, so a chunk menu could steer one judge into another's artifact, and in split mode the gate never verifies the un-lensed file. The hint lives in the correctness script, not the shared checklist store, because the store is hashed into every reviewer's bench and review-mode identity.
- 2026-11-19 — sc-8539 — A correctness lens judge's checklist scope is pinned by the gate, not typed by the judge. withStagedFiles exports DEVKIT_CORRECTNESS_LENS (the sorted group) and DEVKIT_CORRECTNESS_CHUNK into each derived lens judge's spawn env, cleared first for every other checklist reviewer. checklist.mjs takes its scope from that env when set: a bare generate/check-item/finalize works the lens-scoped state file the gate verifies, a flag repeating the scope runs, and a flag naming another group or chunk exits 1 leaving every checklist untouched (also sc-6705). Cause: since gateBrief rewrote the brief's bare commands into the granted literal, haiku lens judges ran them without --lens, wrote the un-scoped file, and 7 of 8 'checklist artifact missing' ships on 2026-11-17..19 failed closed on PASS verdicts. This qualifies the 2026-11-17 'never auto-select a group' clause: that held because the script could not tell which judge called it; the env now tells it. Interactive and monolith runs carry no env and behave as before. A judge that never calls the script stays a contract miss.
- 2026-11-19 — The zero-scope notice now leads with SKIPPED (`guard-review: SKIPPED — no domain reviewer ran; …`), the token the ship's end-of-run banner tells a reader to scan for, and says 'domain' so it does not contradict completeness's own PASS line. Each spawned deterministic gate closes its `▶ deterministic:` header with one runner-printed verdict (passed / failed / could not run / opted out / BYPASSED / DEGRADED), derived from the same facts as its gate_step row, where a gate that verified nothing reads SKIPPED, BYPASSED or DEGRADED. The reviewer gate names the reviewers it selected on one roster line, per-reviewer skips print SKIPPED, and a replayed completeness PASS says whether this ship run or an earlier one judged it (ship_id stored with the PASS); the ship digest stays silent on not_selected under gate-verdict-attribution's silence clause.
105 changes: 105 additions & 0 deletions gate-engine/deterministic/__tests__/gate-start-lines.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { GATE_REASON_ENV } from '../reason.mts';
import { runDeterministic } from '../run.mts';

const dirs: string[] = [];
afterEach(() => {
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
delete process.env.DEVKIT_SHIP;
for (const f of ['GUARD_HOOK_PARITY_OK', 'GUARD_DECISION_CHECKS_OK', 'GUARD_COVERAGE_OK'])
delete process.env[f];
delete process.env.GUARD_DETERMINISTIC_STRICT;
delete process.env.FRINK_DETERMINISTIC_STRICT;
vi.restoreAllMocks();
Expand Down Expand Up @@ -61,12 +64,16 @@ describe('runDeterministic — a line names each gate as it starts', () => {
expect(events).toEqual([
'▶ deterministic: guard-size',
'exec node',
' ✓ guard-size passed',
'▶ deterministic: hook-parity',
'exec true',
' ✓ hook-parity passed',
'▶ deterministic: knip',
'exec knip',
' ✓ knip passed',
'▶ deterministic: structure-lint',
'exec eslint',
' ✓ structure-lint passed',
]);
});

Expand Down Expand Up @@ -102,3 +109,101 @@ describe('runDeterministic — a line names each gate as it starts', () => {
expect(err.mock.calls.flat().join('\n')).toContain('empty(unrunnable: empty command)');
});
});

describe('runDeterministic — a verdict line closes each gate header', () => {
// Throws what execFileSync throws for a gate exiting non-zero or timing out, keyed on its argv.
const failing = (byArg: Record<string, { status?: number; code?: string }>) => {
const t = timeline();
t.exec.mockImplementation((cmd: string, args: string[] = []) => {
t.events.push(`exec ${cmd}`);
const hit = Object.entries(byArg).find(([k]) => [cmd, ...args].some((a) => a.includes(k)));
if (hit) throw Object.assign(new Error(cmd), hit[1]);
});
return t;
};

it('names a failure, an opt-out and a timeout, never as passed', () => {
const { events, exec } = failing({
false: { status: 1 },
matcher: { status: 2 },
slow: { code: 'ETIMEDOUT' },
eslint: { status: 2 },
});
const d = repo(['dup'], { hang: 'slow' });
const extra = [{ label: 'hook-parity', cmd: 'false' }];

expect(runDeterministic(d, { exec, extra, structure: 'eslint src' })).toBe(1);

expect(events.filter((l) => l.startsWith(' '))).toEqual([
' ⚠ guard-dup SKIPPED — opted out, did NOT run',
' ✗ hook-parity failed',
' ✗ hang could not run',
' ✗ structure-lint could not run',
]);
});

it('names a declared extraGates entry on a passing run, and drops it once undeclared', () => {
const declared = timeline();
expect(runDeterministic(repo([], { 'git-pathspec': 'node check.mjs' }), declared)).toBe(0);
expect(declared.events).toContain(' ✓ git-pathspec passed');

vi.restoreAllMocks();
const undeclared = timeline();
expect(runDeterministic(repo([]), undeclared)).toBe(0);
expect(undeclared.events.some((l) => l.includes('git-pathspec'))).toBe(false);
});

it('marks only the gate that left a withheld note as DEGRADED, not the gates after it', () => {
const { events, exec } = timeline();
exec.mockImplementation((_cmd: string, args: string[], opts: { env: NodeJS.ProcessEnv }) => {
events.push(`exec ${args.join(' ')}`);
if (args.some((a) => a.includes('size')))
writeFileSync(String(opts.env[GATE_REASON_ENV]), 'semantic index unavailable\n');
});

expect(runDeterministic(repo(['size', 'fanout']), { exec })).toBe(0);

expect(events.filter((l) => l.startsWith(' '))).toEqual([
' ⚠ guard-size passed (DEGRADED)',
' ✓ guard-fanout passed',
]);
});

it('reports an opt-out that strict mode refuses as could not run, never as SKIPPED', () => {
process.env.GUARD_DETERMINISTIC_STRICT = '1';
const { events, exec } = failing({ matcher: { status: 2 } });

expect(runDeterministic(repo(['dup']), { exec })).toBe(1);

expect(events).toContain(' ✗ guard-dup could not run');
});

// An --extra label is bare; a registry gate's label carries the guard- prefix its flag omits.
it.each([
[
'an --extra gate',
'GUARD_HOOK_PARITY_OK',
[],
[{ label: 'hook-parity', cmd: 'true' }],
'hook-parity',
],
[
'a registry gate',
'GUARD_DECISION_CHECKS_OK',
['decision-checks'],
[],
'guard-decision-checks',
],
['the coverage gate', 'GUARD_COVERAGE_OK', ['coverage'], [], 'guard-coverage'],
])(
'marks %s that passed through its own bypass flag as BYPASSED',
(_, flag, guards, extra, label) => {
process.env[flag] = '1';
const { events, exec } = timeline();

expect(runDeterministic(repo(guards), { exec, extra })).toBe(0);

expect(events).toContain(` ⚠ ${label} BYPASSED — verified nothing`);
},
);
});
14 changes: 14 additions & 0 deletions gate-engine/deterministic/command-gates.mts
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,20 @@ export function emitGateStep(
});
}

/** The line closing a gate's `▶ deterministic:` header, from the same facts as its gate_step row. */
export function gateVerdictLine(
label: string,
failure: { verdict: boolean } | undefined,
optedOut: boolean,
degraded: boolean,
): string {
if (failure) return ` ✗ ${label} ${failure.verdict ? 'failed' : 'could not run'}`;
if (optedOut) return ` ⚠ ${label} SKIPPED — opted out, did NOT run`;
if (selfBypassed(label.replace(GUARD_PREFIX_RE, '')))
return ` ⚠ ${label} BYPASSED — verified nothing`;
return degraded ? ` ⚠ ${label} passed (DEGRADED)` : ` ✓ ${label} passed`;
}

/** What runArgv returns for a gate that produced no exit within its deadline. */
export const TIMED_OUT = 'timed-out';
// A hang detector, not a performance budget: a slow gate on a loaded machine is still legitimate.
Expand Down
18 changes: 11 additions & 7 deletions gate-engine/deterministic/run.mts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ import {
commandGate,
emitGateStep,
extraGatesFor,
gateVerdictLine,
gateTimeoutMs,
GUARD_PREFIX_RE,
reportCachedBypasses,
Expand Down Expand Up @@ -273,7 +274,11 @@ function runGates(
const startedAt = Date.now();
console.log(`▶ deterministic: ${gate.label}`);
const rc = runArgv(cwd, gate.argv, exec, reasonFile, timeoutMs, gate.label);
const [nFailed, nSkipped] = [out.failed.length, out.skipped.length];
const [nFailed, nSkipped, nDegraded] = [
out.failed.length,
out.skipped.length,
out.degraded.length,
];
const code = exitCodeOrBlock(out, gate, rc, timeoutMs, startedAt, reasonFile);
if (code === null) return;
const failRc = (suffix = '') =>
Expand All @@ -290,12 +295,10 @@ function runGates(
} else if (code !== 0) failRc(`(unexpected:${code})`);
else if (gate.id) noteDegraded(out, gate.id, reasonFile);
} finally {
emitGateStep(
gate.label,
code,
startedAt,
out.failed[nFailed],
out.skipped.length > nSkipped,
const [failure, optedOut] = [out.failed[nFailed], out.skipped.length > nSkipped];
emitGateStep(gate.label, code, startedAt, failure, optedOut);
console.log(
gateVerdictLine(gate.label, failure, optedOut, out.degraded.length > nDegraded),
);
}
});
Expand Down Expand Up @@ -343,6 +346,7 @@ function exitCodeOrBlock(
);
}
emitGateStep(gate.label, failure, startedAt, out.failed[nFailed], false);
console.log(gateVerdictLine(gate.label, out.failed[nFailed], false, false));
return null;
}

Expand Down
16 changes: 16 additions & 0 deletions gate-engine/review/__tests__/completeness-pass-findings.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,22 @@ describe('a completeness PASS with findings', () => {
expect(advisoryRows()).toHaveLength(6); // three live, three replayed
});

// The ship's parallel pre-commit judge and its commit-msg replay share one DEVKIT_SHIP_ID.
it.each([
[SHIP, 'judged earlier in this ship run'],
['a-later-ship', 'judged in an earlier ship run'],
])('names which ship run judged a replayed PASS (replayed under %s)', async (run, source) => {
const repo = consumerRepo({ backend: true });
const msg = messageFile(repo, 'fix(ship): pin the --base');
await runCompleteness(msg, repo, { exec: judge(PASSED) });
vi.stubEnv('DEVKIT_SHIP_ID', run);
vi.mocked(console.error).mockClear();

expect(await runCompleteness(msg, repo, { exec: judge('VERDICT: FAIL — not asked') })).toBe(0);

expect(printed().find((l) => l.includes('cached PASS'))).toMatch(new RegExp(` · ${source}$`));
});

it('stays at one line with the ref, and emits nothing advisory, when it found nothing', async () => {
const repo = consumerRepo({ backend: true });
const clean = 'ISSUES: 0 critical, 0 important, 0 low\nVERDICT: PASS — complete';
Expand Down
43 changes: 39 additions & 4 deletions gate-engine/review/__tests__/empty-roots.test.mts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { execFileSync } from 'node:child_process';
import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it, vi } from 'vitest';
Expand All @@ -17,6 +17,8 @@ import {
SYNCED_ASSET_EXCLUDES,
} from '../evidence/scope.mts';
import { selectReviewers } from '../reviewers.mts';
import { runReviewGate } from '../run-review.mts';
import { cleanupReviewFixtures, consumerRepo, mkExec } from './run-review-fixtures.mts';

const REPO_ROOT = join(import.meta.dirname, '../../..');

Expand All @@ -40,6 +42,7 @@ afterEach(() => {
vi.unstubAllEnvs();
if (sinkDir) rmSync(sinkDir, { recursive: true, force: true });
sinkDir = '';
cleanupReviewFixtures();
});

describe('domainsDisabledByEmptyRoots', () => {
Expand Down Expand Up @@ -123,16 +126,47 @@ describe('reportNonRuns', () => {
const notices = (alreadyReported: Set<string>) => {
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
reportNonRuns(['relay/index.ts', 'README.md'], inverted, [], alreadyReported);
return spy.mock.calls.map((c) => String(c[0])).filter((l) => l.includes('no reviewer ran'));
return spy.mock.calls
.map((c) => String(c[0]))
.filter((l) => l.includes('no domain reviewer ran'));
};

it('says so when path scope leaves the commit with no reviewer at all', () => {
expect(notices(new Set())).toEqual([
'guard-review: no reviewer ran — none of the 2 staged file(s) is in review scope ' +
'guard-review: SKIPPED — no domain reviewer ran; none of the 2 staged file(s) is in review scope ' +
'(scanRoots, review roots and review.paths in guard.config.json)',
]);
});

it('names every reviewer it selected, so a reader can tell which domains judged the commit', () => {
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
const staged = ['src/server/db.ts'];
reportNonRuns(staged, inverted, selectReviewers(staged, inverted), new Set());

const roster = spy.mock.calls.map((c) => String(c[0])).filter((l) => l.includes('selected:'));
expect(roster).toHaveLength(1);
expect(roster[0]).toMatch(/^guard-review: \d+ reviewer\(s\) selected: [a-z-]+(, [a-z-]+)*$/);
expect(roster[0]).toContain('correctness-reviewer');
});

it("ends a docs-only commit's reviewer gate on one SKIPPED line, spawning no judge", async () => {
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
const repo = consumerRepo();
mkdirSync(join(repo, 'docs'));
writeFileSync(join(repo, 'docs', 'guide.md'), '# Guide\n');
execFileSync('git', ['add', 'docs/guide.md'], { cwd: repo });
const exec = mkExec(async () => 'VERDICT: FAIL — no judge should run');

expect(await runReviewGate(repo, { exec })).toBe(0);

const lines = spy.mock.calls.map((c) => String(c[0]));
expect(exec).not.toHaveBeenCalled();
expect(
lines.filter((l) => l.startsWith('guard-review: SKIPPED — no domain reviewer ran')),
).toHaveLength(1);
expect(lines.some((l) => l.includes(' selected: '))).toBe(false);
});

it('stays quiet when another line already named why a reviewer did not run', () => {
expect(notices(new Set(['commit-guard']))).toEqual([]);
});
Expand Down Expand Up @@ -447,7 +481,8 @@ describe('filesUnseenByCorrectness', () => {
const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
const selection = selectReviewers(staged, cfg);
reportNonRuns(staged, cfg, selection, new Set(), new Set(), selection);
return spy.mock.calls.map((c) => String(c[0]));
// The routing lines only: the roster names what DID run, and has its own test.
return spy.mock.calls.map((c) => String(c[0])).filter((l) => !l.includes(' selected: '));
};

it('names code that reached no reviewer at all, on one line', () => {
Expand Down
2 changes: 1 addition & 1 deletion gate-engine/review/__tests__/recheck.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,7 @@ describe('unrouted-files notice wiring', () => {
const repo = consumerRepo({ backend: true });
captureEvents(repo);
expect(await runReviewGate(repo, { exec: lensExec(repo) })).toBe(0);
expect(stderr()).toContain('api-security-reviewer skipped (GUARD_REVIEW_SKIP)');
expect(stderr()).toContain('api-security-reviewer SKIPPED (GUARD_REVIEW_SKIP)');
expect(stderr()).not.toContain(NOTICE);
expect(routingRows(repo)).toEqual([]);
});
Expand Down
6 changes: 3 additions & 3 deletions gate-engine/review/__tests__/run-review.test.mts
Original file line number Diff line number Diff line change
Expand Up @@ -618,8 +618,8 @@ describe('runReviewGate — cascade + exit contract', () => {
expect(exec).not.toHaveBeenCalled(); // proves the early return was taken
expect(lines(spy)).toEqual(
expect.arrayContaining([
'guard-review: frontend-security-reviewer skipped (review.frontendRoots is empty in guard.config.json)',
'guard-review: frontend-performance-reviewer skipped (review.frontendRoots is empty in guard.config.json)',
'guard-review: frontend-security-reviewer SKIPPED (review.frontendRoots is empty in guard.config.json)',
'guard-review: frontend-performance-reviewer SKIPPED (review.frontendRoots is empty in guard.config.json)',
]),
);
expect(lines(spy).some((l) => l.includes('1 staged frontend file(s) went unreviewed'))).toBe(
Expand Down Expand Up @@ -1057,7 +1057,7 @@ describe('runReviewGate — cascade + exit contract', () => {
expect(exec.mock.calls.map(([o]) => o.label)).not.toContain('review:api-security-reviewer');
expect(exec).toHaveBeenCalledTimes(4); // backend-performance + commit-guard + correctness + conventions still ran
expect(err.mock.calls.flat().join('\n')).toContain(
'api-security-reviewer skipped (GUARD_REVIEW_SKIP)',
'api-security-reviewer SKIPPED (GUARD_REVIEW_SKIP)',
);
});

Expand Down
1 change: 1 addition & 0 deletions gate-engine/review/completeness.mts
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,7 @@ export async function runCompleteness(
};
// Names the snapshot the judge was shown; absent (no snapshot formed) never vouches.
if (stagedIdentity) meta.diff_sha = stagedIdentity;
meta.ship_id = process.env.DEVKIT_SHIP_ID; // undefined off a ship, and dropped when stored
if (mcpCause) meta.mcp_degraded_cause = mcpCause;
if (findings.length) meta.advisories = findings;
// Both identities: the exact byte key (any caller, any order) and the branch+message sticky
Expand Down
12 changes: 11 additions & 1 deletion gate-engine/review/evidence/base-context.mts
Original file line number Diff line number Diff line change
Expand Up @@ -324,11 +324,21 @@ export function reportVerdictTranscript(name: string, status: string, ref?: stri
console.error(`guard-review: ${name} — reviewed diff: guard-review transcript ${ref}`);
}

/** Which ship run judged a cached PASS, when the entry recorded one and this run is a ship. */
function cachedShipRun(meta: VerdictMeta): string {
const run = process.env.DEVKIT_SHIP_ID;
if (!run || meta.ship_id === undefined) return '';
return meta.ship_id === run
? ' · judged earlier in this ship run'
: ' · judged in an earlier ship run';
}

/** A cached PASS line (`tail` = its provenance), marked and reported when it replays MCP-degraded,
* followed by the findings the PASS stored. */
export function printCachedPass(name: string, meta: VerdictMeta, tail: string): void {
const cause = cachedMcpDegradation(meta);
console.error(`guard-review: ${name} — cached PASS${cause ? ' (DEGRADED)' : ''} ${tail}`);
const from = cachedShipRun(meta);
console.error(`guard-review: ${name} — cached PASS${cause ? ' (DEGRADED)' : ''} ${tail}${from}`);
if (cause) reportMcpDegraded(name, cause, true);
reportAdvisories(name, cachedAdvisories(meta));
}
Expand Down
Loading