From f69baeeeca5211754e9a04e4c3a0f5ba8d5567d8 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 23 Oct 2026 02:44:17 +0000 Subject: [PATCH 01/22] feat(evals): add coding-task evaluation framework (#2084) --- package.json | 2 + scripts/agent-evals/README.md | 68 +- scripts/agent-evals/coding-compare.mts | 112 +++ scripts/agent-evals/coding-compare.ts | 61 ++ scripts/agent-evals/coding-digest.test.ts | 24 + scripts/agent-evals/coding-digest.ts | 28 + scripts/agent-evals/coding-fixtures.test.ts | 80 +++ scripts/agent-evals/coding-fixtures.ts | 669 ++++++++++++++++++ scripts/agent-evals/coding-grader.test.ts | 182 +++++ scripts/agent-evals/coding-grader.ts | 132 ++++ scripts/agent-evals/coding-metrics.test.ts | 305 ++++++++ scripts/agent-evals/coding-metrics.ts | 194 +++++ scripts/agent-evals/coding-outcomes.test.ts | 60 ++ scripts/agent-evals/coding-outcomes.ts | 54 ++ scripts/agent-evals/coding-pins.test.ts | 46 ++ scripts/agent-evals/coding-pins.ts | 34 + scripts/agent-evals/coding-pricing.test.ts | 53 ++ scripts/agent-evals/coding-pricing.ts | 74 ++ .../coding-reference-solutions.test.ts | 72 ++ scripts/agent-evals/coding-report.test.ts | 181 +++++ scripts/agent-evals/coding-report.ts | 211 ++++++ scripts/agent-evals/coding-run.test.ts | 163 +++++ scripts/agent-evals/coding-run.ts | 144 ++++ scripts/agent-evals/coding-side.mts | 135 ++++ scripts/agent-evals/coding-stats.test.ts | 62 ++ scripts/agent-evals/coding-stats.ts | 118 +++ scripts/agent-evals/coding-suite.ts | 53 ++ scripts/agent-evals/coding-workspace.test.ts | 102 +++ scripts/agent-evals/coding-workspace.ts | 49 ++ scripts/agent-evals/container-runtime.test.ts | 62 ++ scripts/agent-evals/container-runtime.ts | 141 ++++ scripts/agent-evals/grader-driver.mjs | 139 ++++ scripts/agent-evals/launcher.ts | 52 +- scripts/agent-evals/runner.ts | 9 + 34 files changed, 3862 insertions(+), 9 deletions(-) create mode 100644 scripts/agent-evals/coding-compare.mts create mode 100644 scripts/agent-evals/coding-compare.ts create mode 100644 scripts/agent-evals/coding-digest.test.ts create mode 100644 scripts/agent-evals/coding-digest.ts create mode 100644 scripts/agent-evals/coding-fixtures.test.ts create mode 100644 scripts/agent-evals/coding-fixtures.ts create mode 100644 scripts/agent-evals/coding-grader.test.ts create mode 100644 scripts/agent-evals/coding-grader.ts create mode 100644 scripts/agent-evals/coding-metrics.test.ts create mode 100644 scripts/agent-evals/coding-metrics.ts create mode 100644 scripts/agent-evals/coding-outcomes.test.ts create mode 100644 scripts/agent-evals/coding-outcomes.ts create mode 100644 scripts/agent-evals/coding-pins.test.ts create mode 100644 scripts/agent-evals/coding-pins.ts create mode 100644 scripts/agent-evals/coding-pricing.test.ts create mode 100644 scripts/agent-evals/coding-pricing.ts create mode 100644 scripts/agent-evals/coding-reference-solutions.test.ts create mode 100644 scripts/agent-evals/coding-report.test.ts create mode 100644 scripts/agent-evals/coding-report.ts create mode 100644 scripts/agent-evals/coding-run.test.ts create mode 100644 scripts/agent-evals/coding-run.ts create mode 100644 scripts/agent-evals/coding-side.mts create mode 100644 scripts/agent-evals/coding-stats.test.ts create mode 100644 scripts/agent-evals/coding-stats.ts create mode 100644 scripts/agent-evals/coding-suite.ts create mode 100644 scripts/agent-evals/coding-workspace.test.ts create mode 100644 scripts/agent-evals/coding-workspace.ts create mode 100644 scripts/agent-evals/container-runtime.test.ts create mode 100644 scripts/agent-evals/container-runtime.ts create mode 100644 scripts/agent-evals/grader-driver.mjs diff --git a/package.json b/package.json index b0c290b97c..4118c12fc3 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", "test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts", "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", + "coding-evals:side": "node --import tsx scripts/agent-evals/coding-side.mts", + "coding-evals:compare": "node --import tsx scripts/agent-evals/coding-compare.mts", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", "webview-recovery:model-check": "node --import tsx scripts/check-webview-recovery.ts", diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 6eef1d6b0b..48db29e872 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -97,13 +97,65 @@ pnpm agent-evals:check-types Both run offline. No check spawns VS Code or touches a provider. -## Follow-up milestones +## Coding evaluation framework + +The coding framework layers an evaluation on the substrate. It grades runs, +repeats them, and compares two exact commits. It stays independent from any +release process: a maintainer runs it locally or on demand. + +| File | Purpose | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `coding-fixtures.ts` | Five frozen coding fixtures. Each ships a known-good workspace, one prompt, a tool policy, limits, hidden acceptance tests, and two reference solutions. | +| `coding-suite.ts` | Suite identity `coding-basic-v1`, registration, and the suite content digest. | +| `coding-workspace.ts` | The immutable initial-state digest, fresh workspace builds, and fresh grading copies. | +| `coding-grader.ts` | The grader contract, version, driver invocation, and the local subprocess runtime. | +| `grader-driver.mjs` | The acceptance-test driver. Runs the workspace tests and writes one result JSON file. | +| `container-runtime.ts` | The default grading runtime. Builds and runs the locked-down container invocation. | +| `coding-outcomes.ts` | Outcome rules: `correct`, `incorrect`, `timed out`, `grading failure`, `infrastructure failure`. | +| `coding-pricing.ts` | The operator-maintained price table and the run cost estimate. | +| `coding-metrics.ts` | Per-run metrics: requests, usage, tool calls, cost, unavailable telemetry, cost per successful task. | +| `coding-run.ts` | One repetition: substrate run, final-workspace capture, grading, outcome, metrics. | +| `coding-pins.ts` | Pinned comparison inputs and the incompatibility check. | +| `coding-stats.ts` | Three repetitions, success rates with Wilson intervals, paired differences. | +| `coding-report.ts` | The self-contained JSON report and the Markdown renderer. | +| `coding-compare.ts` | The comparison orchestration and the side result shape. | +| `coding-side.mts` | One comparison side: all fixtures times N repetitions on this commit, graded, with pinned inputs. | +| `coding-compare.mts` | Two-commit comparison driver. Writes `comparison-report.json` and `comparison-report.md`. | + +### Grading + +- The grader is model-free and credential-free. It judges the final workspace state only. +- The final workspace is captured as bounded file data and rebuilt as a fresh copy. Hidden acceptance tests enter only that copy, so the agent can never read or change them, and one run can never inherit another run's files. +- The default runtime executes the driver in a disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. +- The grader has its own wall-clock limit. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. +- Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. +- `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. + +### Comparisons + +A paired comparison pins both commit SHAs, the suite digest, the settings digest, the model identifier, the grader version, and the runner environment on both sides. Any difference in a pinned value marks the pair incompatible. The report records the values and flags the pair instead of dropping it. + +The default is three repetitions per fixture per commit. Reports show success rates with Wilson 95% intervals and paired differences. They make no significance claim: three repetitions can show large regressions, not small improvements. + +Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct or a correct run lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. + +### Commands -The substrate supports these planned steps. None of them exist in this -change: +```sh +# Unit tests and type checks. Offline. No VS Code, no provider, no containers. +pnpm test:agent-evals:unit +pnpm agent-evals:check-types + +# One comparison side against the real provider. Requires the credential +# environment variable and the container runtime for grading. +pnpm coding-evals:side --reps 3 --out /tmp/side-out + +# Full paired comparison of two commits. +pnpm coding-evals:compare --baseline --candidate --reps 3 --out /tmp/compare-out +``` + +### Follow-up milestones -1. Five frozen, real coding task fixtures with known-good workspaces. -2. Independent credential-free graders that judge workspace state only. -3. Paired exact trusted commit comparisons on the same fixtures. -4. Repeated runs with uncertainty reporting. -5. Advisory release integration aligned to the actual release workflows. +1. Release integration: trigger runs, store baselines, and promote them. See issue #2085. +2. Later suites: repository navigation, debugging, and cost stress, toward the 21-task set. +3. Optimization experiments: per-request context instrumentation, non-progress loop detection, tool-output filtering, compaction strategies, and dynamic tool-schema loading. diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts new file mode 100644 index 0000000000..771c5c48b6 --- /dev/null +++ b/scripts/agent-evals/coding-compare.mts @@ -0,0 +1,112 @@ +// Paired comparison driver. Checks out both commits into temporary worktrees, +// runs one coding-evals side per commit with identical repetition counts, +// then writes the JSON and Markdown comparison reports. Both commits must +// contain this framework. The driver is trusted repo code and only runs +// trusted repo commits with the credential. +import { execFile, spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { sideResultFileName, toSideResult, type SideResultFile } from "./coding-compare.ts" +import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" + +const execFileAsync = promisify(execFile) +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +const readArgs = (): { + baseline: string + candidate: string + reps: number + out: string + credentialEnv: string + allowUnisolatedGrading: boolean +} => { + const args = process.argv.slice(2) + const read = (name: string): string | undefined => { + const index = args.indexOf(`--${name}`) + return index >= 0 ? args[index + 1] : undefined + } + const baseline = read("baseline") + const candidate = read("candidate") + const out = read("out") + if (!baseline || !candidate) throw new Error("Set --baseline and --candidate to commit SHAs") + if (!out) throw new Error("Set --out to a scratch directory") + const reps = Number(read("reps") ?? "3") + if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") + return { + baseline, + candidate, + reps, + out: path.resolve(out), + credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + } +} + +const runSide = async ( + side: "baseline" | "candidate", + commit: string, + inputs: ReturnType, + scratchDir: string, +): Promise => { + const worktree = path.join(scratchDir, side) + await execFileAsync("git", ["worktree", "add", "--detach", worktree, commit], { cwd: root, timeout: 120_000 }) + const sideOut = path.join(scratchDir, `${side}-out`) + await fs.mkdir(sideOut, { recursive: true }) + const exit = await new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + "scripts/agent-evals/coding-side.mts", + "--reps", + String(inputs.reps), + "--out", + sideOut, + "--credential-env", + inputs.credentialEnv, + ...(inputs.allowUnisolatedGrading ? ["--allow-unisolated-grading"] : []), + ], + { + cwd: worktree, + env: { ...process.env, CODING_EVALS_SIDE: side }, + stdio: "inherit", + }, + ) + child.once("error", reject) + child.once("exit", (code) => resolve(code ?? 1)) + }) + if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) + return JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) as SideResultFile +} + +const inputs = readArgs() +const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-compare-")) +await fs.mkdir(inputs.out, { recursive: true }) +try { + const report = buildComparisonReport({ + baseline: toSideResult(await runSide("baseline", inputs.baseline, inputs, scratchDir)), + candidate: toSideResult(await runSide("candidate", inputs.candidate, inputs, scratchDir)), + repetitions: inputs.reps, + }) + await fs.writeFile(path.join(inputs.out, "comparison-report.json"), JSON.stringify(report, null, 2)) + await fs.writeFile(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) + console.log( + report.compatibility.compatible + ? `comparison complete: compatible pair, report in ${inputs.out}` + : `comparison complete: INCOMPATIBLE pair (${report.compatibility.differences.join(", ")}), report in ${inputs.out}`, + ) +} finally { + await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "baseline")], { + cwd: root, + timeout: 30_000, + }).catch(() => {}) + await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "candidate")], { + cwd: root, + timeout: 30_000, + }).catch(() => {}) + await fs.rm(scratchDir, { recursive: true, force: true }).catch(() => {}) +} diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts new file mode 100644 index 0000000000..56357f419d --- /dev/null +++ b/scripts/agent-evals/coding-compare.ts @@ -0,0 +1,61 @@ +import type { SidePins } from "./coding-pins.ts" +import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" +import { DEFAULT_REPETITIONS } from "./coding-stats.ts" +import type { CodingRunRecord } from "./coding-run.ts" + +export { buildComparisonReport, renderMarkdownReport } +export type { ComparisonReport, SideResult } + +export interface SideExecutionRequest { + side: "baseline" | "candidate" + commit: string + repetitions: number +} + +// Executes one side of the comparison. The live executor runs the side's own +// in-repo suite inside a worktree of that commit. Tests replace it with a +// synthetic executor. +export interface SideExecutor { + runSide(request: SideExecutionRequest): Promise +} + +export interface ComparisonRequest { + baselineCommit: string + candidateCommit: string + repetitions?: number + executor: SideExecutor +} + +// Paired comparison. Each side runs the same repetition count over its own +// frozen suite copy with pinned inputs. Pinned value differences surface in +// the report as an incompatible pair; they never abort the run, because the +// recorded values are the evidence a reviewer needs. +export async function runComparison(request: ComparisonRequest): Promise { + const repetitions = request.repetitions ?? DEFAULT_REPETITIONS + const baseline = await request.executor.runSide({ + side: "baseline", + commit: request.baselineCommit, + repetitions, + }) + const candidate = await request.executor.runSide({ + side: "candidate", + commit: request.candidateCommit, + repetitions, + }) + return buildComparisonReport({ baseline, candidate, repetitions }) +} + +// Serializes a side result for artifact storage. Full run records travel so +// each side's traces and grading verdicts stay inspectable. +export interface SideResultFile { + pins: SidePins + runs: CodingRunRecord[] +} + +// Reduces a side result file to the shape the report builder consumes. +export const toSideResult = (file: SideResultFile): SideResult => ({ + pins: file.pins, + runs: file.runs.map((record) => record.metrics), +}) + +export const sideResultFileName = (side: "baseline" | "candidate"): string => `${side}-results.json` diff --git a/scripts/agent-evals/coding-digest.test.ts b/scripts/agent-evals/coding-digest.test.ts new file mode 100644 index 0000000000..daedc1b1c3 --- /dev/null +++ b/scripts/agent-evals/coding-digest.test.ts @@ -0,0 +1,24 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { canonicalJson, digestValue, sha256Hex } from "./coding-digest.ts" + +describe("coding digest", () => { + it("produces stable digests independent of key order", () => { + const a = { b: 1, a: { d: 2, c: [1, 2] } } + const b = { a: { c: [1, 2], d: 2 }, b: 1 } + assert.equal(canonicalJson(a), canonicalJson(b)) + assert.equal(digestValue(a), digestValue(b)) + }) + + it("changes the digest when content changes", () => { + assert.notEqual(digestValue({ a: 1 }), digestValue({ a: 2 })) + }) + + it("keeps array order significant", () => { + assert.notEqual(digestValue([1, 2]), digestValue([2, 1])) + }) + + it("matches sha256 over the canonical form", () => { + assert.equal(digestValue({ x: "y" }), sha256Hex(canonicalJson({ x: "y" }))) + }) +}) diff --git a/scripts/agent-evals/coding-digest.ts b/scripts/agent-evals/coding-digest.ts new file mode 100644 index 0000000000..d66bbd19e2 --- /dev/null +++ b/scripts/agent-evals/coding-digest.ts @@ -0,0 +1,28 @@ +import { createHash } from "node:crypto" + +// Canonical JSON for digests. Object keys are sorted, array order is kept, and +// the output is stable across processes. Any content change changes the digest, +// so a digest pins content, not formatting. +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortValue(value)) +} + +function sortValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortValue) + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sortValue(entry)]), + ) + } + return value +} + +export function sha256Hex(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex") +} + +export function digestValue(value: unknown): string { + return sha256Hex(canonicalJson(value)) +} diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts new file mode 100644 index 0000000000..d91d63b474 --- /dev/null +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { configurationMatchesToolPolicy, validateFixture } from "./contract.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { CODING_SUITE_ID, codingBundle, codingSuiteDigest } from "./coding-suite.ts" +import { GRADER_VERSION } from "./coding-grader.ts" + +const CREDENTIAL = "sk-test-credential-for-projection-check" + +describe("coding fixture suite", () => { + it("exposes exactly five fixtures", () => { + assert.equal(CODING_FIXTURE_BUNDLES.length, 5) + assert.equal(new Set(CODING_FIXTURE_BUNDLES.map((bundle) => bundle.fixture.identity.name)).size, 5) + }) + + it("passes the substrate contract for every fixture", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) validateFixture(bundle.fixture) + }) + + it("builds configurations that match the declared tool policy", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + const config = bundle.fixture.buildConfiguration(CREDENTIAL) as Record + assert.equal(configurationMatchesToolPolicy(config, bundle.fixture.toolPolicy), true) + } + }) + + it("keeps the credential out of the settings projection", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + const projectionJson = JSON.stringify(bundle.fixture.settingsProjection) + assert.equal(projectionJson.includes(CREDENTIAL), false) + const configJson = JSON.stringify(bundle.fixture.buildConfiguration(CREDENTIAL)) + assert.equal(configJson.includes(CREDENTIAL), true) + } + }) + + it("ships hidden tests that are absent from the run workspace", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + assert.ok(bundle.grading.hiddenTests.length > 0) + for (const hidden of bundle.grading.hiddenTests) + assert.equal( + bundle.fixture.workspace.files.some((file) => file.path === hidden.path), + false, + `hidden test leaked into workspace: ${hidden.path}`, + ) + } + }) + + it("ships two distinct reference solutions over existing workspace paths", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + assert.ok(bundle.grading.correctSolution.length > 0) + const workspacePaths = new Set(bundle.fixture.workspace.files.map((file) => file.path)) + for (const file of [...bundle.grading.correctSolution, ...bundle.grading.incorrectSolution]) + assert.equal(workspacePaths.has(file.path), true, `solution path missing from workspace: ${file.path}`) + const sameFiles = bundle.grading.correctSolution.every((file) => { + const incorrect = bundle.grading.incorrectSolution.find((candidate) => candidate.path === file.path) + return incorrect === undefined || incorrect.content !== file.content + }) + assert.equal(sameFiles, true) + } + }) + + it("pins the grader identity with resources and an image", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + assert.ok(bundle.grading.resources.timeoutMs > 0) + assert.ok(bundle.grading.resources.memoryMb > 0) + assert.ok(bundle.grading.resources.cpus > 0) + assert.ok(bundle.grading.resources.pidsLimit > 0) + assert.match(bundle.grading.resources.image, /^node:/) + } + }) + + it("produces a stable suite digest that tracks the grader version", () => { + assert.equal(codingSuiteDigest(), codingSuiteDigest()) + assert.equal(codingBundle("clamp-range").fixture.identity.name, "clamp-range") + assert.throws(() => codingBundle("unknown-fixture")) + assert.ok(codingSuiteDigest().length === 64) + assert.equal(CODING_SUITE_ID, "coding-basic-v1") + assert.equal(GRADER_VERSION, "coding-grader-v1") + }) +}) diff --git a/scripts/agent-evals/coding-fixtures.ts b/scripts/agent-evals/coding-fixtures.ts new file mode 100644 index 0000000000..f5990e9816 --- /dev/null +++ b/scripts/agent-evals/coding-fixtures.ts @@ -0,0 +1,669 @@ +import type { RooCodeSettings } from "@roo-code/types" +import type { AgentFixture, AgentFixtureWorkspaceFile } from "./contract.ts" +import type { GraderResources } from "./coding-grader.ts" +import { GRADER_IMAGE } from "./container-runtime.ts" + +// Suite provider. Mirrors the smoke fixture provider endpoint so a coding run +// uses the same trusted, preconfigured provider as the smoke run. +export const CODING_BASE_URL = "https://api.solheim.ai/v1" +export const CODING_MODEL_ID = "qwen3.8-27b" + +const CODING_ROLE_DEFINITION = "Complete one bounded coding task in this workspace. Work only inside the workspace." +const CODING_DISABLED_TOOLS = [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", +] as const + +export const CODING_GRADER_RESOURCES: GraderResources = { + timeoutMs: 30_000, + memoryMb: 512, + cpus: 1, + pidsLimit: 128, + image: GRADER_IMAGE, +} + +function buildCodingConfiguration(apiKey: string): RooCodeSettings { + return { + apiProvider: "openai", + openAiBaseUrl: CODING_BASE_URL, + openAiApiKey: apiKey, + openAiModelId: CODING_MODEL_ID, + openAiR1FormatEnabled: true, + openAiExtraBody: JSON.stringify({ chat_template_kwargs: { enable_thinking: false } }), + modelMaxTokens: 8192, + autoCondenseContext: false, + mode: "coding-eval", + customModes: [ + { + slug: "coding-eval", + name: "Coding eval", + roleDefinition: CODING_ROLE_DEFINITION, + groups: ["read", "edit", "command"], + }, + ], + disabledTools: [...CODING_DISABLED_TOOLS], + autoApprovalEnabled: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: true, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: true, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + } +} + +// Explicit nonsecret mirror of the built configuration. Must stay in step +// with buildCodingConfiguration; the registry checks the builder against the +// tool policy and the trace digests the projection. +const codingSettingsProjection: Record = { + apiProvider: "openai", + openAiBaseUrl: CODING_BASE_URL, + openAiModelId: CODING_MODEL_ID, + openAiR1FormatEnabled: true, + openAiExtraBody: { chat_template_kwargs: { enable_thinking: false } }, + modelMaxTokens: 8192, + autoCondenseContext: false, + mode: "coding-eval", + customModes: [ + { + slug: "coding-eval", + name: "Coding eval", + roleDefinition: CODING_ROLE_DEFINITION, + groups: ["read", "edit", "command"], + }, + ], + disabledTools: [...CODING_DISABLED_TOOLS], + autoApprovalEnabled: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: true, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: true, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, +} + +export interface CodingFixtureBundle { + fixture: AgentFixture + grading: { + // Hidden acceptance tests. Copied into a fresh grading copy after the + // run. Never part of the workspace the agent sees. + hiddenTests: readonly AgentFixtureWorkspaceFile[] + // Reference solutions, each an overlay over the workspace files. The + // known-correct solution must pass the grader; the deliberately + // incorrect solution must fail it while passing the visible tests. + correctSolution: readonly AgentFixtureWorkspaceFile[] + incorrectSolution: readonly AgentFixtureWorkspaceFile[] + resources: GraderResources + } +} + +const workspacePackageJson = (name: string): AgentFixtureWorkspaceFile => ({ + path: "package.json", + content: `${JSON.stringify( + { + name: `${name}-workspace`, + private: true, + version: "1.0.0", + scripts: { test: "node --test test/" }, + }, + null, + 2, + )}\n`, +}) + +const codingLimits = { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 300_000, + usageDrainMs: 10_000, + teardownStepMs: 10_000, +} + +const bundle = ( + name: string, + version: string, + description: string, + prompt: string, + files: readonly AgentFixtureWorkspaceFile[], + hiddenTests: readonly AgentFixtureWorkspaceFile[], + correctSolution: readonly AgentFixtureWorkspaceFile[], + incorrectSolution: readonly AgentFixtureWorkspaceFile[], +): CodingFixtureBundle => ({ + fixture: { + identity: { name, version, description }, + prompt, + workspace: { files }, + toolPolicy: { + mode: "coding-eval", + roleDefinition: CODING_ROLE_DEFINITION, + groups: ["read", "edit", "command"], + disabledTools: [...CODING_DISABLED_TOOLS], + allow: { + readOnly: true, + readOnlyOutsideWorkspace: false, + write: true, + writeOutsideWorkspace: false, + writeProtected: false, + execute: true, + mcp: false, + modeSwitch: false, + subtasks: false, + followupQuestions: false, + }, + }, + limits: codingLimits, + settingsProjection: codingSettingsProjection, + buildConfiguration: buildCodingConfiguration, + }, + grading: { hiddenTests, correctSolution, incorrectSolution, resources: CODING_GRADER_RESOURCES }, +}) + +// --------------------------------------------------------------------------- +// Fixture 1: clamp-range +// --------------------------------------------------------------------------- + +const clampStub = `// Clamp a number into a closed range. +// +// clamp(value, min, max) returns value when min <= value <= max. A value +// below min returns min. A value above max returns max. When min is greater +// than max the range is empty and clamp returns min for every value. A NaN +// value returns NaN. + +function clamp(value, min, max) { + throw new Error("TODO: implement clamp") +} + +module.exports = { clamp } +` + +const clampVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { clamp } = require("../src/clamp.js") + +test("keeps in-range values unchanged", () => { + assert.strictEqual(clamp(5, 0, 10), 5) +}) + +test("pulls values below the range down to min", () => { + assert.strictEqual(clamp(-1, 0, 10), 0) +}) + +test("pulls values above the range up to max", () => { + assert.strictEqual(clamp(11, 0, 10), 10) +}) +` + +const clampHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { clamp } = require("../src/clamp.js") + +test("returns min for an empty range", () => { + assert.strictEqual(clamp(5, 10, 2), 10) +}) + +test("returns min for an empty range at the min boundary", () => { + assert.strictEqual(clamp(10, 10, 2), 10) +}) + +test("returns max for an empty range at the max boundary", () => { + assert.strictEqual(clamp(2, 10, 2), 10) +}) +` + +const clampCorrect = `// Clamp a number into a closed range. +function clamp(value, min, max) { + if (min > max) return min + if (value < min) return min + if (value > max) return max + return value +} + +module.exports = { clamp } +` + +const clampIncorrect = `function clamp(value, min, max) { + return Math.min(Math.max(value, min), max) +} + +module.exports = { clamp } +` + +// --------------------------------------------------------------------------- +// Fixture 2: slugify-text +// --------------------------------------------------------------------------- + +const slugifyStub = `// Convert text into a URL slug. +// +// slugify(text) lowercases the text, keeps latin letters and digits, and +// replaces every run of all other characters with exactly one dash. Leading +// and trailing runs produce no dashes. An empty result for empty input is a +// valid slug. + +function slugify(text) { + throw new Error("TODO: implement slugify") +} + +module.exports = { slugify } +` + +const slugifyVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { slugify } = require("../src/slugify.js") + +test("joins words with one dash", () => { + assert.strictEqual(slugify("Hello World"), "hello-world") +}) + +test("collapses runs of separators", () => { + assert.strictEqual(slugify("A B"), "a-b") +}) + +test("keeps letters and digits", () => { + assert.strictEqual(slugify("abc123"), "abc123") +}) +` + +const slugifyHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { slugify } = require("../src/slugify.js") + +test("treats punctuation as separators", () => { + assert.strictEqual(slugify("Hello, World!!"), "hello-world") +}) + +test("trims leading and trailing separators", () => { + assert.strictEqual(slugify(" --A-- "), "a") +}) + +test("returns an empty slug for empty input", () => { + assert.strictEqual(slugify(""), "") +}) +` + +const slugifyCorrect = `// Convert text into a URL slug. +function slugify(text) { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") +} + +module.exports = { slugify } +` + +const slugifyIncorrect = `function slugify(text) { + return text.toLowerCase().trim().replace(/\\s+/g, "-") +} + +module.exports = { slugify } +` + +// --------------------------------------------------------------------------- +// Fixture 3: parse-csv-line +// --------------------------------------------------------------------------- + +const csvStub = `// Parse one single-line CSV record. +// +// parseCsvLine(line) splits the line on commas. A field that starts with a +// double quote is quoted: it may contain commas and ends at the next double +// quote. Inside a quoted field, a doubled double-quote (\"\") stands for one +// literal double quote. Unquoted fields stay exactly as written, with no +// trimming. + +function parseCsvLine(line) { + throw new Error("TODO: implement parseCsvLine") +} + +module.exports = { parseCsvLine } +` + +const csvVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { parseCsvLine } = require("../src/parse-csv-line.js") + +test("splits plain fields on commas", () => { + assert.deepStrictEqual(parseCsvLine("a,b,c"), ["a", "b", "c"]) +}) + +test("keeps commas inside quoted fields", () => { + assert.deepStrictEqual(parseCsvLine('"x,y",z'), ["x,y", "z"]) +}) + +test("returns a single field for a bare line", () => { + assert.deepStrictEqual(parseCsvLine("plain"), ["plain"]) +}) +` + +const csvHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { parseCsvLine } = require("../src/parse-csv-line.js") + +test("unescapes doubled quotes inside quoted fields", () => { + assert.deepStrictEqual(parseCsvLine('"he said ""hi""",ok'), ['he said "hi"', "ok"]) +}) + +test("keeps a trailing empty field", () => { + assert.deepStrictEqual(parseCsvLine("a,"), ["a", ""]) +}) + +test("parses a quoted empty string", () => { + assert.deepStrictEqual(parseCsvLine('""'), [""]) +}) +` + +const csvCorrect = `function parseCsvLine(line) { + const fields = [] + let current = "" + let inQuotes = false + for (let index = 0; index < line.length; index++) { + const ch = line[index] + if (inQuotes) { + if (ch === '"') { + if (line[index + 1] === '"') { + current += '"' + index++ + } else { + inQuotes = false + } + } else { + current += ch + } + continue + } + if (ch === '"' && current.length === 0) { + inQuotes = true + continue + } + if (ch === ",") { + fields.push(current) + current = "" + continue + } + current += ch + } + fields.push(current) + return fields +} + +module.exports = { parseCsvLine } +` + +// Plausible but wrong: quote-aware, without doubled-quote unescaping. +const csvIncorrect = `function parseCsvLine(line) { + const fields = [] + let current = "" + let inQuotes = false + for (const ch of line) { + if (inQuotes) { + if (ch === '"') { + inQuotes = false + continue + } + current += ch + continue + } + if (ch === '"') { + inQuotes = true + continue + } + if (ch === ",") { + fields.push(current) + current = "" + continue + } + current += ch + } + fields.push(current) + return fields +} + +module.exports = { parseCsvLine } +` + +// --------------------------------------------------------------------------- +// Fixture 4: format-bytes +// --------------------------------------------------------------------------- + +const bytesStub = `// Format a byte count with SI decimal units. +// +// formatBytes(bytes) returns "0 B" for zero. Counts below 1000 return the +// integer with the unit "B". From 1000 upward it uses the units kB, MB, GB, +// and TB, dividing by 1000 at each step, always with exactly one decimal. + +function formatBytes(bytes) { + throw new Error("TODO: implement formatBytes") +} + +module.exports = { formatBytes } +` + +const bytesVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { formatBytes } = require("../src/format-bytes.js") + +test("formats small counts in bytes", () => { + assert.strictEqual(formatBytes(500), "500 B") +}) + +test("formats one decimal kilobytes", () => { + assert.strictEqual(formatBytes(1500), "1.5 kB") +}) + +test("formats exact byte counts without decimals", () => { + assert.strictEqual(formatBytes(123), "123 B") +}) +` + +const bytesHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { formatBytes } = require("../src/format-bytes.js") + +test("formats zero bytes", () => { + assert.strictEqual(formatBytes(0), "0 B") +}) + +test("uses decimal megabytes, not binary", () => { + assert.strictEqual(formatBytes(2500000), "2.5 MB") +}) + +test("uses decimal gigabytes, not binary", () => { + assert.strictEqual(formatBytes(1000000000), "1.0 GB") +}) + +test("keeps the last byte below the kilobyte boundary", () => { + assert.strictEqual(formatBytes(999), "999 B") +}) +` + +const bytesCorrect = `function formatBytes(bytes) { + if (bytes === 0) return "0 B" + if (bytes < 1000) return \`\${bytes} B\` + const units = ["kB", "MB", "GB", "TB"] + let value = bytes + let unit = -1 + do { + value /= 1000 + unit++ + } while (value >= 1000 && unit < units.length - 1) + return \`\${value.toFixed(1)} \${units[unit]}\` +} + +module.exports = { formatBytes } +` + +// Plausible but wrong: binary divisors and boundaries. +const bytesIncorrect = `function formatBytes(bytes) { + if (bytes === 0) return "0 B" + const units = ["B", "kB", "MB", "GB", "TB"] + let value = bytes + let unit = 0 + while (value >= 1024 && unit < units.length - 1) { + value /= 1024 + unit++ + } + return unit === 0 ? \`\${value} B\` : \`\${value.toFixed(1)} \${units[unit]}\` +} + +module.exports = { formatBytes } +` + +// --------------------------------------------------------------------------- +// Fixture 5: deep-get +// --------------------------------------------------------------------------- + +const deepGetStub = `// Read a value from an object by dot path. +// +// get(object, path) splits the path on dots and walks the object. Numeric +// segments index arrays. A missing key, a missing intermediate object, or a +// null intermediate returns undefined. get never throws on missing data. + +function get(object, path) { + throw new Error("TODO: implement get") +} + +module.exports = { get } +` + +const deepGetVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { get } = require("../src/deep-get.js") + +test("reads nested objects", () => { + assert.strictEqual(get({ a: { b: 1 } }, "a.b"), 1) +}) + +test("indexes arrays with numeric segments", () => { + assert.strictEqual(get({ a: [10, 20] }, "a.1"), 20) +}) +` + +const deepGetHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { get } = require("../src/deep-get.js") + +test("returns undefined for a missing path on an empty object", () => { + assert.strictEqual(get({}, "a.b"), undefined) +}) + +test("returns undefined past a non-object intermediate", () => { + assert.strictEqual(get({ a: 1 }, "a.b.c"), undefined) +}) + +test("walks through arrays and objects", () => { + assert.strictEqual(get({ a: [{ b: 3 }] }, "a.0.b"), 3) +}) + +test("returns undefined for an absent root key", () => { + assert.strictEqual(get({ a: { b: 2 } }, "x.y"), undefined) +}) +` + +const deepGetCorrect = `function get(object, path) { + let current = object + for (const segment of path.split(".")) { + if (current === null || current === undefined) return undefined + current = current[segment] + } + return current +} + +module.exports = { get } +` + +// Plausible but wrong: throws on missing intermediates. +const deepGetIncorrect = `function get(object, path) { + return path.split(".").reduce((value, key) => value[key], object) +} + +module.exports = { get } +` + +// --------------------------------------------------------------------------- +// Frozen suite definition. Fixture data is frozen: editing any prompt, file, +// test, or solution changes the suite digest and invalidates comparisons. +// --------------------------------------------------------------------------- + +export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ + bundle( + "clamp-range", + "1.0.0", + "Implement clamp(value, min, max) with empty-range semantics; visible tests miss the empty-range case.", + "Implement the clamp function in src/clamp.js so the test suite passes. The header comment in src/clamp.js defines the exact behavior, including the empty range rule. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("clamp-range"), + { path: "src/clamp.js", content: clampStub }, + { path: "test/clamp.test.js", content: clampVisibleTest }, + ], + [{ path: "test/clamp.acceptance.test.js", content: clampHiddenTest }], + [{ path: "src/clamp.js", content: clampCorrect }], + [{ path: "src/clamp.js", content: clampIncorrect }], + ), + bundle( + "slugify-text", + "1.0.0", + "Implement slugify(text) with punctuation-as-separator semantics; visible tests miss punctuation runs.", + "Implement the slugify function in src/slugify.js so the test suite passes. The header comment in src/slugify.js defines the exact behavior. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("slugify-text"), + { path: "src/slugify.js", content: slugifyStub }, + { path: "test/slugify.test.js", content: slugifyVisibleTest }, + ], + [{ path: "test/slugify.acceptance.test.js", content: slugifyHiddenTest }], + [{ path: "src/slugify.js", content: slugifyCorrect }], + [{ path: "src/slugify.js", content: slugifyIncorrect }], + ), + bundle( + "parse-csv-line", + "1.0.0", + "Implement parseCsvLine(line) with doubled-quote unescaping; visible tests miss escaped quotes.", + "Implement the parseCsvLine function in src/parse-csv-line.js so the test suite passes. The header comment in src/parse-csv-line.js defines the quoting rules, including doubled quotes. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("parse-csv-line"), + { path: "src/parse-csv-line.js", content: csvStub }, + { path: "test/parse-csv-line.test.js", content: csvVisibleTest }, + ], + [{ path: "test/parse-csv-line.acceptance.test.js", content: csvHiddenTest }], + [{ path: "src/parse-csv-line.js", content: csvCorrect }], + [{ path: "src/parse-csv-line.js", content: csvIncorrect }], + ), + bundle( + "format-bytes", + "1.0.0", + "Implement formatBytes(bytes) with SI decimal units; visible tests miss zero and the MB and GB boundaries.", + "Implement the formatBytes function in src/format-bytes.js so the test suite passes. The header comment in src/format-bytes.js defines the exact units and rounding. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("format-bytes"), + { path: "src/format-bytes.js", content: bytesStub }, + { path: "test/format-bytes.test.js", content: bytesVisibleTest }, + ], + [{ path: "test/format-bytes.acceptance.test.js", content: bytesHiddenTest }], + [{ path: "src/format-bytes.js", content: bytesCorrect }], + [{ path: "src/format-bytes.js", content: bytesIncorrect }], + ), + bundle( + "deep-get", + "1.0.0", + "Implement get(object, path) that never throws on missing data; visible tests miss the missing-data paths.", + "Implement the get function in src/deep-get.js so the test suite passes. The header comment in src/deep-get.js defines the missing-data rules. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("deep-get"), + { path: "src/deep-get.js", content: deepGetStub }, + { path: "test/deep-get.test.js", content: deepGetVisibleTest }, + ], + [{ path: "test/deep-get.acceptance.test.js", content: deepGetHiddenTest }], + [{ path: "src/deep-get.js", content: deepGetCorrect }], + [{ path: "src/deep-get.js", content: deepGetIncorrect }], + ), +] diff --git a/scripts/agent-evals/coding-grader.test.ts b/scripts/agent-evals/coding-grader.test.ts new file mode 100644 index 0000000000..dd60644366 --- /dev/null +++ b/scripts/agent-evals/coding-grader.test.ts @@ -0,0 +1,182 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { + createLocalSubprocessGradingRuntime, + runGrader, + type DriverCompleted, + type GraderJob, + type GraderVerdict, + type GradingRuntime, +} from "./coding-grader.ts" +import { verdictFromDriver } from "./container-runtime.ts" + +const tmpdir = (): Promise => fs.mkdtemp(path.join(os.tmpdir(), "coding-grader-test-")) + +const writeDriver = async (dir: string, script: string): Promise => { + const file = path.join(dir, "driver.mjs") + await fs.writeFile(file, script) + return file +} + +const driverCompleted = (overrides: Partial = {}): DriverCompleted => ({ + kind: "completed", + passed: true, + noTestsDiscovered: false, + testFiles: [], + passedCount: 3, + failedCount: 0, + failures: [], + durationMs: 1, + ...overrides, +}) + +const staticRuntime = (verdict: GraderVerdict): GradingRuntime => ({ + isolation: "container", + run: async () => verdict, +}) + +const job = async (scratch: string): Promise => { + const gradingDir = path.join(scratch, "grading-copy") + await fs.mkdir(gradingDir, { recursive: true }) + await fs.writeFile(path.join(gradingDir, "marker.txt"), "x") + return { + gradingDir, + driverPath: await writeDriver(scratch, "process.exit(0)"), + resources: { timeoutMs: 5_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + } +} + +describe("grader verdict rules", () => { + it("maps driver results to verdicts", () => { + assert.equal(verdictFromDriver(driverCompleted(), "container").kind, "passed") + assert.equal( + verdictFromDriver( + driverCompleted({ passed: false, failedCount: 2, failures: [{ name: "x", file: "y" }] }), + "container", + ).kind, + "failed", + ) + assert.deepEqual(verdictFromDriver(driverCompleted({ noTestsDiscovered: true }), "container"), { + kind: "grading-failure", + isolation: "container", + reason: "noTestsDiscovered", + }) + assert.deepEqual(verdictFromDriver({ kind: "driver-error", message: "boom" }, "container"), { + kind: "grading-failure", + isolation: "container", + reason: "driverError", + }) + }) +}) + +describe("runGrader", () => { + it("builds a grading copy and removes it afterwards", async () => { + const scratch = await tmpdir() + try { + let sawGradingDir = false + const runtime: GradingRuntime = { + isolation: "container", + run: async (input) => { + sawGradingDir = await fs + .access(path.join(input.gradingDir, "marker.txt")) + .then(() => true) + .catch(() => false) + return { kind: "passed", isolation: "container", result: driverCompleted() } + }, + } + const verdict = await runGrader({ + finalFiles: [{ path: "marker.txt", content: "x" }], + hiddenTests: [], + resources: { timeoutMs: 1_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + runtime, + scratchDir: scratch, + }) + assert.equal(verdict.kind, "passed") + assert.equal(sawGradingDir, true) + const gradingCopyGone = await fs + .access(path.join(scratch, "grading-copy")) + .then(() => false) + .catch(() => true) + assert.equal(gradingCopyGone, true) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a failed grading copy as grading failure", async () => { + const scratch = await tmpdir() + try { + const verdict = await runGrader({ + finalFiles: [{ path: "../escape.txt", content: "x" }], + hiddenTests: [], + resources: { timeoutMs: 1_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + runtime: staticRuntime({ kind: "passed", isolation: "container", result: driverCompleted() }), + scratchDir: scratch, + }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "gradingCopyFailed", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) + +describe("local subprocess runtime failure paths", () => { + it("reports a hung grader as graderTimeout", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + const driverPath = await writeDriver(scratch, "setInterval(() => {}, 1000)") + const verdict = await createLocalSubprocessGradingRuntime().run({ + ...base, + driverPath, + resources: { ...base.resources, timeoutMs: 300 }, + }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "local-subprocess", + reason: "graderTimeout", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a crashed grader as graderCrashed", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + const driverPath = await writeDriver(scratch, "process.exit(3)") + const verdict = await createLocalSubprocessGradingRuntime().run({ ...base, driverPath }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "local-subprocess", + reason: "graderCrashed", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a silent grader as graderResultMissing", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + const driverPath = await writeDriver(scratch, "process.exit(0)") + const verdict = await createLocalSubprocessGradingRuntime().run({ ...base, driverPath }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "local-subprocess", + reason: "graderResultMissing", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts new file mode 100644 index 0000000000..3950bc9168 --- /dev/null +++ b/scripts/agent-evals/coding-grader.ts @@ -0,0 +1,132 @@ +import { spawn } from "node:child_process" +import { fileURLToPath } from "node:url" +import fs from "node:fs/promises" +import path from "node:path" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" +import { buildGradingCopy } from "./coding-workspace.ts" +import { readDriverResult, verdictFromDriver } from "./container-runtime.ts" + +// The grader version is a pinned comparison input. Any change to the driver, +// the image, or the verdict rules must bump it. +export const GRADER_VERSION = "coding-grader-v1" + +export interface GraderResources { + timeoutMs: number + memoryMb: number + cpus: number + pidsLimit: number + image: string +} + +export interface DriverCompleted { + kind: "completed" + passed: boolean + noTestsDiscovered: boolean + testFiles: string[] + passedCount: number + failedCount: number + failures: { name: string; file: string }[] + durationMs: number +} + +export interface DriverError { + kind: "driver-error" + message: string +} + +export type DriverResult = DriverCompleted | DriverError + +export interface GraderJob { + // Fresh copy of the final workspace plus the hidden acceptance tests. + gradingDir: string + driverPath: string + resources: GraderResources +} + +export type GradingIsolation = "container" | "local-subprocess" + +export type GraderVerdict = + | { kind: "passed"; isolation: GradingIsolation; result: DriverCompleted } + | { kind: "failed"; isolation: GradingIsolation; result: DriverCompleted } + | { kind: "grading-failure"; isolation: GradingIsolation; reason: string } + +export interface GradingRuntime { + readonly isolation: GradingIsolation + run(job: GraderJob): Promise +} + +export const driverFilePath = (): string => fileURLToPath(new URL("./grader-driver.mjs", import.meta.url)) + +// Runs one grading job. The grader: +// - works on a fresh copy of the final workspace, never the original, +// - adds hidden acceptance tests only to that copy, +// - executes through the given runtime (disposable container by default), +// - has no credential and no network access in either runtime, +// - reports a hung, crashed, or over-limit grader as a grading failure. +export async function runGrader(inputs: { + finalFiles: readonly AgentFixtureWorkspaceFile[] + hiddenTests: readonly AgentFixtureWorkspaceFile[] + resources: GraderResources + runtime: GradingRuntime + scratchDir: string +}): Promise { + const gradingDir = path.join(inputs.scratchDir, "grading-copy") + try { + await buildGradingCopy(inputs.finalFiles, inputs.hiddenTests, gradingDir) + return await inputs.runtime.run({ + gradingDir, + driverPath: driverFilePath(), + resources: inputs.resources, + }) + } catch { + return { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "gradingCopyFailed" } + } finally { + await fs.rm(gradingDir, { recursive: true, force: true }) + } +} + +const INPROCESS_ENV_ALLOWLIST = ["PATH", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"] as const + +// Local subprocess runtime. It runs the same driver directly on the host with +// a minimal environment and the same wall-clock kill. It has NO container +// isolation: agent-written code runs with the host filesystem. It exists for +// unit tests and explicit local checks only. It does not give isolation equal +// to the container runtime and must never back a recorded comparison. +export function createLocalSubprocessGradingRuntime(nodeExecutable = process.execPath): GradingRuntime { + return { + isolation: "local-subprocess", + async run(job: GraderJob): Promise { + const outDir = path.join(path.dirname(job.gradingDir), "grading-out") + const resultPath = path.join(outDir, "result.json") + await fs.mkdir(outDir, { recursive: true }) + try { + const exit = await new Promise((resolve) => { + const env: NodeJS.ProcessEnv = {} + for (const name of INPROCESS_ENV_ALLOWLIST) { + const value = process.env[name] + if (value !== undefined) env[name] = value + } + const child = spawn(nodeExecutable, [job.driverPath, resultPath, "test"], { + cwd: job.gradingDir, + env, + stdio: "ignore", + timeout: job.resources.timeoutMs, + killSignal: "SIGKILL", + }) + child.once("error", () => resolve("error")) + child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) + }) + if (exit === "timeout") + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderTimeout" } + if (exit === "error" || exit !== 0) + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderCrashed" } + const result = await readDriverResult(resultPath) + if (!result) + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderResultMissing" } + return verdictFromDriver(result, "local-subprocess") + } finally { + await fs.rm(outDir, { recursive: true, force: true }) + } + }, + } +} diff --git a/scripts/agent-evals/coding-metrics.test.ts b/scripts/agent-evals/coding-metrics.test.ts new file mode 100644 index 0000000000..9b20ff52dd --- /dev/null +++ b/scripts/agent-evals/coding-metrics.test.ts @@ -0,0 +1,305 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { buildRunMetrics, costPerSuccessfulTask, type CodingRunMetrics } from "./coding-metrics.ts" +import type { PriceTable } from "./coding-pricing.ts" +import type { AgentTrace, TraceAccounting } from "./trace.ts" +import { TRACE_KIND, TRACE_SCHEMA_VERSION, identityUnavailable } from "./trace.ts" +import type { ClassifiedOutcome } from "./coding-outcomes.ts" + +const correctOutcome: ClassifiedOutcome = { outcome: "correct", reason: "hiddenTestsPassed" } + +const accounting = (overrides: Partial = {}): TraceAccounting => ({ + requests: [], + requestCount: 0, + requestsMissingUsage: 0, + apiReqRetriedMarkers: 0, + usage: { available: false, snapshotsObserved: 0 }, + toolSnapshot: {}, + toolFailureEvents: 0, + toolFailuresByTool: {}, + completeness: { status: "complete", reasons: [] }, + ...overrides, +}) + +const trace = (overrides: Partial = {}): AgentTrace => ({ + traceSchemaVersion: TRACE_SCHEMA_VERSION, + kind: TRACE_KIND, + run: { + runId: "run-1", + fixture: { name: "clamp-range", version: "1.0.0" }, + startedAt: "2026-10-22T00:00:00.000Z", + finishedAt: "2026-10-22T00:01:00.000Z", + durationMs: 60_000, + identity: identityUnavailable(), + }, + settings: { available: true }, + execution: { + code: "COMPLETED", + stopReason: "taskCompleted", + activated: true, + taskAccepted: true, + sessionLost: false, + completionMessageSeen: true, + timings: { totalMs: 60_000 }, + }, + eventFlow: { + observed: 0, + malformed: 0, + foreignTaskEvents: 0, + bufferedBeforeAck: 0, + replayedAfterAck: 0, + eventBufferLimit: 256, + bufferOverflow: false, + droppedByOverflow: 0, + }, + accounting: accounting(), + teardown: { + status: "ok", + hostExited: true, + closeTaskSent: true, + escalatedSignals: [], + cleanup: "done", + failures: [], + }, + notes: [], + ...overrides, +}) + +const fullUsage = { + tokensIn: 1_000_000, + tokensOut: 100_000, + cacheWrites: { available: true, value: 200_000 } as const, + cacheReads: { available: true, value: 400_000 } as const, + cost: 0.42, + contextTokens: 50_000, + source: "taskCompleted" as const, + observedAtMs: 1, +} + +const testTable: PriceTable = { + id: "test-price-table", + date: "2026-01-01", + currency: "USD", + cachePricing: "test rates", + models: { + "qwen3.8-27b": { + inputPerMillion: 0.5, + outputPerMillion: 1.5, + cacheReadPerMillion: 0.125, + cacheWritePerMillion: 0.625, + }, + }, +} + +describe("run metrics", () => { + it("keeps missing token values missing and never writes zero", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace(), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.equal(metrics.usage.tokensIn, undefined) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "noUsageEventsObserved") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("estimates cost from reported usage with an explicitly configured price table", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage } }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, true) + if (metrics.cost.estimated.available) { + // 1.0 * 0.5 + 0.1 * 1.5 + 0.4 * 0.125 + 0.2 * 0.625 = 0.825 + assert.ok(Math.abs(metrics.cost.estimated.usd - 0.825) < 1e-9) + assert.equal(metrics.cost.estimated.priceTableId, "test-price-table") + assert.ok(metrics.cost.estimated.cachePricing.length > 0) + } + assert.deepEqual(metrics.cost.providerReported, { available: true, value: 0.42 }) + assert.equal(metrics.cost.accountingComplete, true) + assert.equal(metrics.usage.cacheReads, 400_000) + assert.equal(metrics.usage.contextTokensEstimate, 50_000) + assert.equal(metrics.usage.labels.context, "extensionContextEstimate") + }) + + it("keeps cost unavailable while the shipped table has no rates", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage } }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "modelMissingFromPriceTable") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("marks accounting incomplete when a cache count is missing", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + usage: { + available: true, + snapshotsObserved: 1, + lastSnapshot: { ...fullUsage, cacheWrites: { available: false, reason: "notReported" } }, + }, + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "cacheWritesNotReported") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("summarizes tool calls and failures from the tool snapshot", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + toolSnapshot: { + snapshot: { read_file: { attempts: 4, failures: 1 }, apply_diff: { attempts: 2, failures: 0 } }, + }, + toolFailureEvents: 1, + toolFailuresByTool: { read_file: 1 }, + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.deepEqual(metrics.toolCalls.totalAttempts, { available: true, value: 6 }) + assert.equal(metrics.toolCalls.totalFailures, 1) + assert.equal(metrics.toolCalls.failureEvents, 1) + assert.equal(metrics.toolCalls.byTool["apply_diff"]?.attempts, 2) + }) + + it("reports unsupported telemetry as unavailable", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace(), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.deepEqual(metrics.unavailableTelemetry, [ + { name: "duplicateCallDetection", reason: "notInstrumented" }, + { name: "httpRetryCount", reason: "apiReqRetriedMarkersCountsMessagesNotHttpRetries" }, + ]) + }) + + it("labels per-request prompt size rows", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + requestCount: 2, + requests: [ + { + index: 0, + ts: 1, + apiProtocol: { available: false, reason: "notReported" }, + tokensIn: { available: true, value: 1_234 }, + tokensOut: { available: false, reason: "notReported" }, + cacheWrites: { available: false, reason: "notReported" }, + cacheReads: { available: false, reason: "notReported" }, + cost: { available: false, reason: "notReported" }, + streamingFailed: false, + firstObservedMs: 0, + lastUpdatedMs: 1, + messageCount: 1, + }, + ], + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.deepEqual(metrics.requests.promptSize, [ + { + index: 0, + reportedInputTokens: { available: true, value: 1_234 }, + label: "providerReportedInputTokens", + }, + ]) + }) +}) + +const metricsRun = (overrides: Partial & { repIndex?: number }): CodingRunMetrics => ({ + fixtureName: "clamp-range", + repIndex: overrides.repIndex ?? 0, + outcome: correctOutcome, + requests: { count: 0, promptSize: [], requestsMissingUsage: 0 }, + usage: { labels: { context: "extensionContextEstimate" } }, + toolCalls: { byTool: {}, totalAttempts: { available: false, reason: "none" }, totalFailures: 0, failureEvents: 0 }, + cost: { + estimated: { available: false, reason: "noUsageEventsObserved" }, + providerReported: { available: false, reason: "noUsageEventsObserved" }, + accountingComplete: false, + }, + accounting: { status: "complete", reasons: [] }, + unavailableTelemetry: [], + ...overrides, +}) + +describe("cost per successful task", () => { + const completeCost = (usd: number) => ({ + estimated: { available: true, usd, priceTableId: "t", priceTableDate: "d", cachePricing: "c" } as const, + providerReported: { available: false, reason: "none" } as const, + accountingComplete: true, + }) + + it("divides the total attempt cost by the correct run count", () => { + const runs: CodingRunMetrics[] = [ + metricsRun({ repIndex: 0, cost: completeCost(0.5) }), + metricsRun({ + repIndex: 1, + outcome: { outcome: "incorrect", reason: "hiddenTestsFailed" }, + cost: completeCost(0.3), + }), + metricsRun({ repIndex: 2, cost: completeCost(0.4) }), + ] + const result = costPerSuccessfulTask(runs) + assert.equal(result.available, true) + assert.ok(Math.abs((result.usd ?? 0) - 0.6) < 1e-9) + assert.equal(result.correctRuns, 2) + assert.equal(result.includedRuns, 3) + assert.equal(result.excludedRuns, 0) + }) + + it("is unavailable with no correct runs", () => { + const result = costPerSuccessfulTask([ + metricsRun({ outcome: { outcome: "incorrect", reason: "x" }, cost: completeCost(1) }), + ]) + assert.equal(result.available, false) + assert.equal(result.reason, "noCorrectRuns") + }) + + it("is unavailable when a correct run lacks complete cost and flags the exclusion", () => { + const result = costPerSuccessfulTask([ + metricsRun({ repIndex: 0, cost: completeCost(1) }), + metricsRun({ repIndex: 1 }), + ]) + assert.equal(result.available, false) + assert.equal(result.reason, "correctRunCostIncomplete") + assert.equal(result.excludedRuns, 1) + assert.equal(result.policy, "excludeIncompleteFromAggregates-v1") + }) +}) diff --git a/scripts/agent-evals/coding-metrics.ts b/scripts/agent-evals/coding-metrics.ts new file mode 100644 index 0000000000..3b8898075e --- /dev/null +++ b/scripts/agent-evals/coding-metrics.ts @@ -0,0 +1,194 @@ +import type { AgentTrace, Measured } from "./trace.ts" +import type { GraderVerdict } from "./coding-grader.ts" +import type { ClassifiedOutcome } from "./coding-outcomes.ts" +import { estimateRunCost, type EstimatedCost, type PriceTable } from "./coding-pricing.ts" + +export const PARTIAL_DATA_POLICY = "excludeIncompleteFromAggregates-v1" + +export interface PromptSizeRow { + index: number + // Provider-reported input tokens for the request. + reportedInputTokens: Measured + label: "providerReportedInputTokens" +} + +export interface ToolCallSummary { + byTool: Record + totalAttempts: Measured + totalFailures: number + failureEvents: number +} + +export interface RunCostMetric { + estimated: EstimatedCost + // Provider-reported cumulative cost from extension events. Independent of + // the estimate and never used as a fallback estimate. + providerReported: Measured + accountingComplete: boolean +} + +export interface CodingRunMetrics { + fixtureName: string + repIndex: number + outcome: ClassifiedOutcome + executionCode?: string + stopReason?: string + durationMs?: number + requests: { count: number; promptSize: PromptSizeRow[]; requestsMissingUsage: number } + usage: { + tokensIn?: number + tokensOut?: number + cacheReads?: number + cacheWrites?: number + // Extension context estimate. Not a provider measurement. + contextTokensEstimate?: number + labels: { context: "extensionContextEstimate" } + } + toolCalls: ToolCallSummary + cost: RunCostMetric + accounting: { status: "complete" | "partial"; reasons: string[] } + grading?: { + kind: GraderVerdict["kind"] + isolation?: string + failedCount?: number + failures?: { name: string; file: string }[] + } + unavailableTelemetry: { name: string; reason: string }[] +} + +// Telemetry the substrate does not instrument. Reported as unavailable, never +// inferred from unrelated events. +export const UNSUPPORTED_TELEMETRY = [ + { name: "duplicateCallDetection", reason: "notInstrumented" }, + { name: "httpRetryCount", reason: "apiReqRetriedMarkersCountsMessagesNotHttpRetries" }, +] as const + +const measuredOrUndefined = (value: Measured): T | undefined => (value.available ? value.value : undefined) + +export function buildRunMetrics(input: { + fixtureName: string + repIndex: number + trace?: AgentTrace + grading?: GraderVerdict + outcome: ClassifiedOutcome + modelIdentifier: Measured + priceTable?: PriceTable +}): CodingRunMetrics { + const trace = input.trace + const accounting = trace?.accounting + const last = accounting?.usage.lastSnapshot + const estimated: EstimatedCost = last + ? estimateRunCost( + input.modelIdentifier.available ? input.modelIdentifier.value : "", + { available: true, value: last.tokensIn }, + { available: true, value: last.tokensOut }, + last.cacheReads, + last.cacheWrites, + input.priceTable, + ) + : { available: false, reason: last ? "modelIdentifierMissing" : "noUsageEventsObserved" } + const snapshot = accounting?.toolSnapshot.snapshot + const byTool: Record = {} + let totalAttempts = 0 + let totalFailures = 0 + for (const [tool, counts] of Object.entries(snapshot ?? {})) { + byTool[tool] = { attempts: counts.attempts, failures: counts.failures } + totalAttempts += counts.attempts + totalFailures += counts.failures + } + const cost: RunCostMetric = { + estimated, + providerReported: last + ? { available: true, value: last.cost } + : { available: false, reason: "noUsageEventsObserved" }, + accountingComplete: estimated.available, + } + return { + fixtureName: input.fixtureName, + repIndex: input.repIndex, + outcome: input.outcome, + executionCode: trace?.execution.code, + stopReason: trace?.execution.stopReason, + durationMs: trace?.run.durationMs, + requests: { + count: accounting?.requestCount ?? 0, + promptSize: (accounting?.requests ?? []).map((row) => ({ + index: row.index, + reportedInputTokens: row.tokensIn, + label: "providerReportedInputTokens", + })), + requestsMissingUsage: accounting?.requestsMissingUsage ?? 0, + }, + usage: { + tokensIn: last?.tokensIn, + tokensOut: last?.tokensOut, + cacheReads: measuredOrUndefined(last?.cacheReads ?? { available: false, reason: "noUsageEventsObserved" }), + cacheWrites: measuredOrUndefined( + last?.cacheWrites ?? { available: false, reason: "noUsageEventsObserved" }, + ), + contextTokensEstimate: last?.contextTokens, + labels: { context: "extensionContextEstimate" }, + }, + toolCalls: { + byTool, + totalAttempts: totalAttempts + ? { available: true, value: totalAttempts } + : { available: false, reason: "noToolSnapshotObserved" }, + totalFailures, + failureEvents: accounting?.toolFailureEvents ?? 0, + }, + cost, + accounting: { + status: accounting?.completeness.status ?? "partial", + reasons: accounting?.completeness.reasons ?? ["runTraceMissing"], + }, + grading: input.grading + ? { + kind: input.grading.kind, + isolation: input.grading.isolation, + failedCount: + input.grading.kind === "passed" || input.grading.kind === "failed" + ? input.grading.result.failedCount + : undefined, + failures: input.grading.kind === "failed" ? input.grading.result.failures.slice(0, 10) : undefined, + } + : undefined, + unavailableTelemetry: UNSUPPORTED_TELEMETRY.map((entry) => ({ ...entry })), + } +} + +export interface CostPerSuccessfulTask { + policy: typeof PARTIAL_DATA_POLICY + includedRuns: number + excludedRuns: number + correctRuns: number + available: boolean + reason?: string + usd?: number +} + +// Total estimated cost of all attempts with complete cost accounting, divided +// by the number of correct runs. Runs with incomplete cost accounting are +// flagged and excluded from aggregates. If no run is correct, or if any +// correct run lacks complete cost accounting, the value is unavailable. +export function costPerSuccessfulTask(runs: readonly CodingRunMetrics[]): CostPerSuccessfulTask { + const included = runs.filter((run) => run.cost.accountingComplete) + const excluded = runs.length - included.length + const correctRuns = runs.filter((run) => run.outcome.outcome === "correct").length + const base: Pick = { + policy: PARTIAL_DATA_POLICY, + includedRuns: included.length, + excludedRuns: excluded, + correctRuns, + } + if (correctRuns === 0) return { ...base, available: false, reason: "noCorrectRuns" } + const correctIncomplete = runs.filter( + (run) => run.outcome.outcome === "correct" && !run.cost.accountingComplete, + ).length + if (correctIncomplete > 0) return { ...base, available: false, reason: "correctRunCostIncomplete" } + const total = included.reduce((sum, run) => { + if (run.cost.estimated.available) return sum + run.cost.estimated.usd + return sum + }, 0) + return { ...base, available: true, usd: total / correctRuns } +} diff --git a/scripts/agent-evals/coding-outcomes.test.ts b/scripts/agent-evals/coding-outcomes.test.ts new file mode 100644 index 0000000000..6b6dbf57fa --- /dev/null +++ b/scripts/agent-evals/coding-outcomes.test.ts @@ -0,0 +1,60 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { classifyRunOutcome } from "./coding-outcomes.ts" + +const base = { + hasTrace: true, + executionCode: "COMPLETED" as const, + finalWorkspaceAvailable: true, +} + +describe("run outcome classification", () => { + it("grades a completed run by the hidden tests", () => { + assert.deepEqual(classifyRunOutcome({ ...base, grading: { kind: "passed" } }), { + outcome: "correct", + reason: "hiddenTestsPassed", + }) + // A completion message from the agent never makes a failing workspace correct. + assert.deepEqual(classifyRunOutcome({ ...base, grading: { kind: "failed" } }), { + outcome: "incorrect", + reason: "hiddenTestsFailed", + }) + }) + + it("grades a stopped run by the hidden tests", () => { + assert.equal( + classifyRunOutcome({ ...base, executionCode: "STOPPED", grading: { kind: "passed" } }).outcome, + "correct", + ) + assert.equal( + classifyRunOutcome({ ...base, executionCode: "STOPPED", grading: { kind: "failed" } }).outcome, + "incorrect", + ) + }) + + it("keeps grading failures out of incorrect", () => { + assert.deepEqual( + classifyRunOutcome({ ...base, grading: { kind: "grading-failure", reason: "graderTimeout" } }), + { outcome: "grading-failure", reason: "graderTimeout" }, + ) + }) + + it("reports timed out for the run-window expiry", () => { + assert.equal(classifyRunOutcome({ ...base, executionCode: "TASK_RUN_TIMEOUT" }).outcome, "timed-out") + assert.equal( + classifyRunOutcome({ ...base, executionCode: "TASK_RUN_TIMEOUT", grading: { kind: "passed" } }).outcome, + "timed-out", + ) + }) + + it("reports infrastructure failures for host, IPC, and provider faults", () => { + for (const code of ["SETUP_FAILED", "ACTIVATION_FAILED", "TASK_START_FAILED", "SESSION_LOST"] as const) + assert.equal(classifyRunOutcome({ ...base, executionCode: code }).outcome, "infrastructure-failure") + assert.equal(classifyRunOutcome({ ...base, hasTrace: false }).outcome, "infrastructure-failure") + assert.equal( + classifyRunOutcome({ ...base, grading: undefined, finalWorkspaceAvailable: false }).outcome, + "infrastructure-failure", + ) + assert.equal(classifyRunOutcome({ ...base, grading: undefined }).outcome, "infrastructure-failure") + }) +}) diff --git a/scripts/agent-evals/coding-outcomes.ts b/scripts/agent-evals/coding-outcomes.ts new file mode 100644 index 0000000000..74738edc02 --- /dev/null +++ b/scripts/agent-evals/coding-outcomes.ts @@ -0,0 +1,54 @@ +import type { AgentExecutionCode } from "./trace.ts" +import type { GraderVerdict } from "./coding-grader.ts" + +export type CodingRunOutcome = "correct" | "incorrect" | "timed-out" | "grading-failure" | "infrastructure-failure" + +export interface OutcomeInput { + hasTrace: boolean + executionCode?: AgentExecutionCode + finalWorkspaceAvailable: boolean + grading?: Pick & { reason?: string } +} + +export interface ClassifiedOutcome { + outcome: CodingRunOutcome + reason: string +} + +const INFRASTRUCTURE_CODES: readonly AgentExecutionCode[] = [ + "SETUP_FAILED", + "ACTIVATION_FAILED", + "TASK_START_FAILED", + "SESSION_LOST", +] + +// Outcome rules, in order: +// - A missing trace is infrastructure failure. Nothing about the task ran. +// - Setup, activation, start, and session-loss codes are infrastructure +// failure. VS Code, IPC, or the provider failed; the agent did not. +// - A run-window expiry is timed out. The agent hit an execution limit. +// - COMPLETED and STOPPED runs are graded. The hidden tests decide: +// passed is correct, failed is incorrect, including when the agent +// reported completion. +// - A grader that hung, crashed, or hit its limits is grading failure and is +// never counted as incorrect. +export function classifyRunOutcome(input: OutcomeInput): ClassifiedOutcome { + if (!input.hasTrace) return { outcome: "infrastructure-failure", reason: "runTraceMissing" } + const code = input.executionCode + if (code && INFRASTRUCTURE_CODES.includes(code)) + return { outcome: "infrastructure-failure", reason: `execution-${code}` } + if (code === "TASK_RUN_TIMEOUT") return { outcome: "timed-out", reason: "agentHitRunLimit" } + if (code !== "COMPLETED" && code !== "STOPPED") + return { outcome: "infrastructure-failure", reason: `execution-${code ?? "unknown"}` } + if (!input.finalWorkspaceAvailable) + return { outcome: "infrastructure-failure", reason: "finalWorkspaceUnavailable" } + if (!input.grading) return { outcome: "infrastructure-failure", reason: "graderNotRun" } + switch (input.grading.kind) { + case "passed": + return { outcome: "correct", reason: "hiddenTestsPassed" } + case "failed": + return { outcome: "incorrect", reason: "hiddenTestsFailed" } + case "grading-failure": + return { outcome: "grading-failure", reason: input.grading.reason ?? "gradingFailure" } + } +} diff --git a/scripts/agent-evals/coding-pins.test.ts b/scripts/agent-evals/coding-pins.test.ts new file mode 100644 index 0000000000..10a8bd2179 --- /dev/null +++ b/scripts/agent-evals/coding-pins.test.ts @@ -0,0 +1,46 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { comparePins, type SidePins } from "./coding-pins.ts" + +const pins = (side: "baseline" | "candidate", overrides: Partial = {}): SidePins => ({ + side, + commit: side === "baseline" ? "a".repeat(40) : "b".repeat(40), + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "qwen3.8-27b", + graderVersion: "coding-grader-v1", + runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, + ...overrides, +}) + +describe("pinned comparison inputs", () => { + it("accepts identical pins with different commits", () => { + const result = comparePins(pins("baseline"), pins("candidate")) + assert.equal(result.compatible, true) + assert.deepEqual(result.differences, []) + }) + + it("flags each differing pinned value", () => { + const cases: Partial[] = [ + { suiteId: "coding-basic-v2" }, + { suiteDigest: "zz".repeat(32) }, + { settingsDigest: "zz".repeat(32) }, + { modelIdentifier: "other-model" }, + { graderVersion: "coding-grader-v2" }, + { runnerEnvironment: { os: "darwin", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" } }, + { runnerEnvironment: { os: "linux", nodeVersion: "v24.0.0", vscodeTarget: "1.104.0" } }, + { runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.100.0" } }, + ] + for (const overrides of cases) { + const result = comparePins(pins("baseline"), pins("candidate", overrides)) + assert.equal(result.compatible, false, `expected incompatibility for ${JSON.stringify(overrides)}`) + assert.ok(result.differences.length >= 1) + } + }) + + it("never compares the commit SHAs themselves", () => { + const result = comparePins(pins("baseline"), pins("candidate", { commit: "c".repeat(40) })) + assert.equal(result.compatible, true) + }) +}) diff --git a/scripts/agent-evals/coding-pins.ts b/scripts/agent-evals/coding-pins.ts new file mode 100644 index 0000000000..3433d11bfd --- /dev/null +++ b/scripts/agent-evals/coding-pins.ts @@ -0,0 +1,34 @@ +export interface SidePins { + side: "baseline" | "candidate" + commit: string + suiteId: string + suiteDigest: string + settingsDigest: string + modelIdentifier: string + graderVersion: string + runnerEnvironment: { + os: string + nodeVersion: string + vscodeTarget: string + } +} + +export interface PinComparison { + compatible: boolean + differences: string[] +} + +// Pinned comparison inputs. The two commit SHAs differ by design and are +// recorded, never compared. Every other pinned value must match on both +// sides; any difference marks the pair incompatible. +const COMPARED_FIELDS = ["suiteId", "suiteDigest", "settingsDigest", "modelIdentifier", "graderVersion"] as const + +export function comparePins(baseline: SidePins, candidate: SidePins): PinComparison { + const differences: string[] = [] + for (const field of COMPARED_FIELDS) if (baseline[field] !== candidate[field]) differences.push(field) + const envFields = ["os", "nodeVersion", "vscodeTarget"] as const + for (const field of envFields) + if (baseline.runnerEnvironment[field] !== candidate.runnerEnvironment[field]) + differences.push(`runnerEnvironment.${field}`) + return { compatible: differences.length === 0, differences } +} diff --git a/scripts/agent-evals/coding-pricing.test.ts b/scripts/agent-evals/coding-pricing.test.ts new file mode 100644 index 0000000000..d5fabe09d8 --- /dev/null +++ b/scripts/agent-evals/coding-pricing.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { PRICE_TABLE, estimateRunCost } from "./coding-pricing.ts" +import { CODING_MODEL_ID } from "./coding-fixtures.ts" + +const reported = (value: number) => ({ available: true as const, value }) + +describe("shipped price table", () => { + it("contains no rates, so no placeholder price can look measured", () => { + assert.deepEqual(Object.keys(PRICE_TABLE.models), []) + }) + + it("keeps the suite model cost unavailable until real rates are configured", () => { + const cost = estimateRunCost( + CODING_MODEL_ID, + reported(1_000_000), + reported(100_000), + reported(400_000), + reported(200_000), + ) + assert.equal(cost.available, false) + if (!cost.available) assert.equal(cost.reason, "modelMissingFromPriceTable") + }) + + it("estimates from an explicitly configured table", () => { + const cost = estimateRunCost( + CODING_MODEL_ID, + reported(1_000_000), + reported(100_000), + reported(400_000), + reported(200_000), + { + id: "configured", + date: "2026-01-01", + currency: "USD", + cachePricing: "separate cache rates", + models: { + [CODING_MODEL_ID]: { + inputPerMillion: 1, + outputPerMillion: 2, + cacheReadPerMillion: 0.5, + cacheWritePerMillion: 0.25, + }, + }, + }, + ) + assert.equal(cost.available, true) + if (cost.available) { + assert.ok(Math.abs(cost.usd - (1 + 0.2 + 0.2 + 0.05)) < 1e-9) + assert.equal(cost.priceTableId, "configured") + } + }) +}) diff --git a/scripts/agent-evals/coding-pricing.ts b/scripts/agent-evals/coding-pricing.ts new file mode 100644 index 0000000000..74d81e001b --- /dev/null +++ b/scripts/agent-evals/coding-pricing.ts @@ -0,0 +1,74 @@ +import type { Measured } from "./trace.ts" + +export interface PriceTableEntry { + inputPerMillion: number + outputPerMillion: number + cacheReadPerMillion: number + cacheWritePerMillion: number +} + +export interface PriceTable { + id: string + date: string + currency: string + cachePricing: string + models: Record +} + +// Operator-maintained price table. The identifier and date travel with every +// estimated cost. Cache reads and writes are priced separately from input and +// output tokens. Update the entries; never reuse the identifier for new rates. +export const PRICE_TABLE: PriceTable = { + id: "agent-evals-prices-2026-10", + date: "2026-10-01", + currency: "USD", + cachePricing: + "cache reads and cache writes are priced per million tokens at their own rates; missing cache counts make the estimate unavailable", + // No rates are configured. Cost estimates stay unavailable until a + // maintainer records real rates here. Placeholder or guessed rates must + // never enter this table, because reports present estimates as measured + // cost. A rate change needs a new table identifier. + models: {}, +} + +export type EstimatedCost = + | { + available: true + usd: number + priceTableId: string + priceTableDate: string + cachePricing: string + } + | { available: false; reason: string } + +// Estimates one run's cost from the final cumulative usage snapshot. All four +// token counts must be reported. A missing count is never treated as zero. +export function estimateRunCost( + modelId: string, + tokensIn: Measured, + tokensOut: Measured, + cacheReads: Measured, + cacheWrites: Measured, + table: PriceTable = PRICE_TABLE, +): EstimatedCost { + const entry = table.models[modelId] + if (!entry) return { available: false, reason: "modelMissingFromPriceTable" } + // Every count must be reported. A missing count is never treated as zero. + if (!tokensIn.available) return { available: false, reason: "tokensInNotReported" } + if (!tokensOut.available) return { available: false, reason: "tokensOutNotReported" } + if (!cacheReads.available) return { available: false, reason: "cacheReadsNotReported" } + if (!cacheWrites.available) return { available: false, reason: "cacheWritesNotReported" } + const perMillion = (tokens: number, rate: number) => (tokens / 1_000_000) * rate + const usd = + perMillion(tokensIn.value, entry.inputPerMillion) + + perMillion(tokensOut.value, entry.outputPerMillion) + + perMillion(cacheReads.value, entry.cacheReadPerMillion) + + perMillion(cacheWrites.value, entry.cacheWritePerMillion) + return { + available: true, + usd, + priceTableId: table.id, + priceTableDate: table.date, + cachePricing: table.cachePricing, + } +} diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts new file mode 100644 index 0000000000..a4034affa7 --- /dev/null +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -0,0 +1,72 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { createLocalSubprocessGradingRuntime, runGrader, GRADER_VERSION } from "./coding-grader.ts" +import { CODING_GRADER_RESOURCES } from "./coding-fixtures.ts" + +// Validates every grader against its two reference solutions. Runs the real +// driver and the real Node test runner. No VS Code, no provider, no network. +describe("coding graders against reference solutions", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + const name = bundle.fixture.identity.name + + const gradeSolution = async (overlay: typeof bundle.grading.correctSolution) => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), `coding-grade-${name}-`)) + try { + const files = bundle.fixture.workspace.files.map((file) => ({ ...file })) + for (const patch of overlay) { + const index = files.findIndex((file) => file.path === patch.path) + if (index >= 0) files[index] = { ...patch } + else files.push({ ...patch }) + } + return await runGrader({ + finalFiles: files, + hiddenTests: bundle.grading.hiddenTests, + resources: bundle.grading.resources, + runtime: createLocalSubprocessGradingRuntime(), + scratchDir: scratch, + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + } + + it(`[${name}] passes the known-correct solution`, async () => { + const verdict = await gradeSolution(bundle.grading.correctSolution) + assert.equal(verdict.kind, "passed") + if (verdict.kind === "passed") assert.ok(verdict.result.failedCount === 0) + }) + + it(`[${name}] fails the plausible incorrect solution on hidden tests only`, async () => { + const verdict = await gradeSolution(bundle.grading.incorrectSolution) + assert.equal(verdict.kind, "failed") + if (verdict.kind === "failed") { + assert.ok(verdict.result.failedCount > 0) + for (const failure of verdict.result.failures) + assert.match( + path.basename(failure.file), + /\.acceptance\.test\.js$/, + `unexpected visible-test failure: ${failure.name}`, + ) + } + }) + + it(`[${name}] passes visible tests even for the incorrect solution`, async () => { + const verdict = await gradeSolution(bundle.grading.incorrectSolution) + if (verdict.kind === "failed") { + const visibleFailures = verdict.result.failures.filter( + (failure) => !path.basename(failure.file).includes(".acceptance."), + ) + assert.equal(visibleFailures.length, 0) + } + }) + } + + it("pins one grader version across the suite", () => { + assert.equal(GRADER_VERSION, "coding-grader-v1") + assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) + }) +}) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts new file mode 100644 index 0000000000..e92aa27aa7 --- /dev/null +++ b/scripts/agent-evals/coding-report.test.ts @@ -0,0 +1,181 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { buildComparisonReport, renderMarkdownReport, type SideResult } from "./coding-report.ts" +import { runComparison, type SideExecutor } from "./coding-compare.ts" +import type { CodingRunMetrics } from "./coding-metrics.ts" +import type { SidePins } from "./coding-pins.ts" + +const pins = (side: "baseline" | "candidate", commit: string, overrides: Partial = {}): SidePins => ({ + side, + commit, + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "qwen3.8-27b", + graderVersion: "coding-grader-v1", + runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, + ...overrides, +}) + +const completeCost = (usd: number): CodingRunMetrics["cost"] => ({ + estimated: { available: true, usd, priceTableId: "t", priceTableDate: "d", cachePricing: "c" }, + providerReported: { available: false, reason: "none" }, + accountingComplete: true, +}) + +const metrics = ( + fixtureName: string, + repIndex: number, + outcome: CodingRunMetrics["outcome"]["outcome"], + usd: number | undefined, +): CodingRunMetrics => ({ + fixtureName, + repIndex, + outcome: { outcome, reason: outcome }, + durationMs: 1_000 + repIndex, + executionCode: "COMPLETED", + stopReason: "taskCompleted", + requests: { count: 0, promptSize: [], requestsMissingUsage: 0 }, + usage: { labels: { context: "extensionContextEstimate" } }, + toolCalls: { byTool: {}, totalAttempts: { available: false, reason: "none" }, totalFailures: 0, failureEvents: 0 }, + cost: + usd === undefined + ? { + estimated: { available: false, reason: "tokensInNotReported" }, + providerReported: { available: false, reason: "none" }, + accountingComplete: false, + } + : completeCost(usd), + accounting: { status: "complete", reasons: [] }, + unavailableTelemetry: [], +}) + +const sideResult = ( + side: "baseline" | "candidate", + commit: string, + runs: CodingRunMetrics[], + overrides?: Partial, +): SideResult => ({ + pins: pins(side, commit, overrides), + runs, +}) + +describe("comparison report", () => { + it("builds a self-contained JSON report over all suite fixtures", () => { + const baseline = sideResult("baseline", "a".repeat(40), [ + metrics("clamp-range", 0, "correct", 0.1), + metrics("clamp-range", 1, "correct", 0.1), + metrics("clamp-range", 2, "incorrect", 0.1), + ]) + const candidate = sideResult("candidate", "b".repeat(40), [ + metrics("clamp-range", 0, "correct", 0.05), + metrics("clamp-range", 1, "timed-out", undefined), + metrics("clamp-range", 2, "grading-failure", 0.05), + ]) + const report = buildComparisonReport({ baseline, candidate }) + assert.equal(report.reportVersion, 1) + assert.equal(report.fixtures.length, 5) + const row = report.fixtures.find((fixture) => fixture.fixtureName === "clamp-range")! + assert.equal(row.baseline.successRate.successes, 2) + assert.equal(row.candidate.successRate.successes, 1) + assert.deepEqual([...row.candidate.outcomes].sort(), ["correct", "grading-failure", "timed-out"].sort()) + assert.equal(row.pairedOutcomes.pairs, 3) + assert.ok(row.pairedDurations.pairs.length === 3) + // Missing candidate cost on rep 1 is a missing pair, never zero. + assert.equal(row.pairedCosts.missingPairs, 1) + // The report serializes cleanly: a self-contained result. + const parsed = JSON.parse(JSON.stringify(report)) as typeof report + assert.equal(parsed.fixtures.length, report.fixtures.length) + }) + + it("flags incompatible pairs when pinned inputs differ", () => { + const baseline = sideResult("baseline", "a".repeat(40), []) + const candidate = sideResult("candidate", "b".repeat(40), [], { modelIdentifier: "other-model" }) + const report = buildComparisonReport({ baseline, candidate }) + assert.equal(report.compatibility.compatible, false) + assert.deepEqual(report.compatibility.differences, ["modelIdentifier"]) + }) + + it("keeps grading failures out of the incorrect count and excludes incomplete cost runs", () => { + const baseline = sideResult("baseline", "a".repeat(40), [ + metrics("clamp-range", 0, "correct", 0.2), + metrics("clamp-range", 1, "grading-failure", undefined), + metrics("clamp-range", 2, "infrastructure-failure", undefined), + ]) + const report = buildComparisonReport({ baseline, candidate: sideResult("candidate", "b".repeat(40), []) }) + const summary = report.costSummary.baseline + assert.equal(summary.correctRuns, 1) + assert.equal(summary.excludedRuns, 2) + assert.equal(summary.available, true) + assert.equal(summary.usd, 0.2) + assert.equal(report.partialDataPolicy, "excludeIncompleteFromAggregates-v1") + }) + + it("renders the Markdown report with required sections and no significance claim", () => { + const baseline = sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]) + const candidate = sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "incorrect", 0.3)]) + const report = buildComparisonReport({ baseline, candidate }) + const markdown = renderMarkdownReport(report) + assert.match(markdown, /# Coding eval comparison: coding-basic-v1/) + assert.match(markdown, /## Pinned inputs and compatibility/) + assert.match(markdown, /Baseline commit/) + assert.match(markdown, /## Method/) + assert.match(markdown, /cannot show small improvements/) + assert.match(markdown, /## Results per fixture/) + assert.match(markdown, /## Cost per successful task/) + assert.match(markdown, /unavailable|over 1 correct run/) + assert.match(markdown, /## Per-run appendix/) + assert.match(markdown, /baseline \| clamp-range \| 0 \| correct /) + }) + + it("marks an incompatible pair in the Markdown report", () => { + const baseline = sideResult("baseline", "a".repeat(40), []) + const candidate = sideResult("candidate", "b".repeat(40), [], { graderVersion: "coding-grader-v9" }) + const markdown = renderMarkdownReport(buildComparisonReport({ baseline, candidate })) + assert.match(markdown, /Incompatible pair/) + assert.match(markdown, /graderVersion/) + }) +}) + +describe("comparison orchestration", () => { + it("runs both sides with the same repetition count and merges results", async () => { + const requests: { side: string; repetitions: number }[] = [] + const executor: SideExecutor = { + runSide: async (request) => { + requests.push({ side: request.side, repetitions: request.repetitions }) + return request.side === "baseline" + ? sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.1)]) + : sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]) + }, + } + const report = await runComparison({ + baselineCommit: "a".repeat(40), + candidateCommit: "b".repeat(40), + repetitions: 2, + executor, + }) + assert.deepEqual(requests, [ + { side: "baseline", repetitions: 2 }, + { side: "candidate", repetitions: 2 }, + ]) + assert.equal(report.compatibility.compatible, true) + assert.equal(report.repetitions, 2) + assert.equal(report.runs.filter((run) => run.side === "baseline").length, 1) + assert.equal(report.runs.filter((run) => run.side === "candidate").length, 1) + }) + + it("defaults to three repetitions", async () => { + const executor: SideExecutor = { + runSide: async (request) => { + assert.equal(request.repetitions, 3) + return sideResult(request.side, "c".repeat(40), []) + }, + } + const report = await runComparison({ + baselineCommit: "a".repeat(40), + candidateCommit: "b".repeat(40), + executor, + }) + assert.equal(report.repetitions, 3) + }) +}) diff --git a/scripts/agent-evals/coding-report.ts b/scripts/agent-evals/coding-report.ts new file mode 100644 index 0000000000..d4a8f80948 --- /dev/null +++ b/scripts/agent-evals/coding-report.ts @@ -0,0 +1,211 @@ +import type { CodingRunOutcome } from "./coding-outcomes.ts" +import type { CodingRunMetrics, CostPerSuccessfulTask, PARTIAL_DATA_POLICY } from "./coding-metrics.ts" +import { costPerSuccessfulTask } from "./coding-metrics.ts" +import type { SidePins, PinComparison } from "./coding-pins.ts" +import { comparePins } from "./coding-pins.ts" +import { + STATISTICAL_POLICY, + successRate, + pairedOutcomeDiffs, + pairedNumericDiffs, + type SuccessRate, + type PairedOutcomeDiff, + type PairedNumericDiff, +} from "./coding-stats.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { initialStateDigest } from "./coding-workspace.ts" +import { CODING_SUITE_ID } from "./coding-suite.ts" + +export interface SideResult { + pins: SidePins + runs: CodingRunMetrics[] +} + +export interface FixtureComparisonRow { + fixtureName: string + fixtureVersion: string + initialStateDigest: string + baseline: SideFixtureStats + candidate: SideFixtureStats + pairedOutcomes: PairedOutcomeDiff + pairedDurations: PairedNumericDiff + pairedCosts: PairedNumericDiff +} + +export interface SideFixtureStats { + outcomes: CodingRunOutcome[] + successRate: SuccessRate +} + +export interface ComparisonReport { + reportVersion: 1 + generatedAt: string + suiteId: string + repetitions: number + statisticalPolicy: typeof STATISTICAL_POLICY + partialDataPolicy: typeof PARTIAL_DATA_POLICY + pins: { baseline: SidePins; candidate: SidePins } + compatibility: PinComparison + fixtures: FixtureComparisonRow[] + costSummary: { baseline: CostPerSuccessfulTask; candidate: CostPerSuccessfulTask } + runs: (CodingRunMetrics & { side: "baseline" | "candidate" })[] +} + +export interface ComparisonInputs { + baseline: SideResult + candidate: SideResult + repetitions?: number +} + +const byRepIndex = (a: CodingRunMetrics, b: CodingRunMetrics) => a.repIndex - b.repIndex + +export function buildComparisonReport(inputs: ComparisonInputs): ComparisonReport { + const repetitions = inputs.repetitions ?? STATISTICAL_POLICY.repetitionsDefault + const compatibility = comparePins(inputs.baseline.pins, inputs.candidate.pins) + const rows: FixtureComparisonRow[] = [] + for (const bundle of CODING_FIXTURE_BUNDLES) { + const name = bundle.fixture.identity.name + const baselineRuns = inputs.baseline.runs.filter((run) => run.fixtureName === name).sort(byRepIndex) + const candidateRuns = inputs.candidate.runs.filter((run) => run.fixtureName === name).sort(byRepIndex) + rows.push({ + fixtureName: name, + fixtureVersion: bundle.fixture.identity.version, + initialStateDigest: initialStateDigest(bundle.fixture), + baseline: { + outcomes: baselineRuns.map((run) => run.outcome.outcome), + successRate: successRate(baselineRuns.map((run) => run.outcome.outcome)), + }, + candidate: { + outcomes: candidateRuns.map((run) => run.outcome.outcome), + successRate: successRate(candidateRuns.map((run) => run.outcome.outcome)), + }, + pairedOutcomes: pairedOutcomeDiffs( + baselineRuns.map((run) => run.outcome.outcome), + candidateRuns.map((run) => run.outcome.outcome), + ), + pairedDurations: pairedNumericDiffs( + baselineRuns.map((run) => run.durationMs), + candidateRuns.map((run) => run.durationMs), + ), + pairedCosts: pairedNumericDiffs( + baselineRuns.map((run) => (run.cost.estimated.available ? run.cost.estimated.usd : undefined)), + candidateRuns.map((run) => (run.cost.estimated.available ? run.cost.estimated.usd : undefined)), + ), + }) + } + return { + reportVersion: 1, + generatedAt: new Date().toISOString(), + suiteId: CODING_SUITE_ID, + repetitions, + statisticalPolicy: STATISTICAL_POLICY, + partialDataPolicy: costPerSuccessfulTask([]).policy, + pins: { baseline: inputs.baseline.pins, candidate: inputs.candidate.pins }, + compatibility, + fixtures: rows, + costSummary: { + baseline: costPerSuccessfulTask(inputs.baseline.runs), + candidate: costPerSuccessfulTask(inputs.candidate.runs), + }, + runs: [ + ...inputs.baseline.runs.map((run) => ({ ...run, side: "baseline" as const })), + ...inputs.candidate.runs.map((run) => ({ ...run, side: "candidate" as const })), + ], + } +} + +const percent = (value: number): string => `${(value * 100).toFixed(0)}%` + +const rateCell = (stats: SideFixtureStats): string => { + const rate = stats.successRate + const counts = stats.outcomes.reduce>((acc, outcome) => { + acc[outcome] = (acc[outcome] ?? 0) + 1 + return acc + }, {}) + const breakdown = [ + `correct ${counts["correct"] ?? 0}`, + `incorrect ${counts["incorrect"] ?? 0}`, + `timed-out ${counts["timed-out"] ?? 0}`, + `grading-failure ${counts["grading-failure"] ?? 0}`, + `infrastructure-failure ${counts["infrastructure-failure"] ?? 0}`, + ].join(", ") + const interval = rate.interval ? ` [${percent(rate.interval.lower)}, ${percent(rate.interval.upper)}]` : "" + return `${rate.successes}/${rate.total}${interval} (${breakdown})` +} + +const costCell = (cost: CostPerSuccessfulTask): string => + cost.available + ? `$${cost.usd?.toFixed(4)} over ${cost.correctRuns} correct run(s), ${cost.excludedRuns} excluded` + : `unavailable (${cost.reason ?? "unavailable"}; ${cost.excludedRuns} excluded)` + +// Self-contained Markdown rendering of the JSON report. +export function renderMarkdownReport(report: ComparisonReport): string { + const lines: string[] = [] + lines.push(`# Coding eval comparison: ${report.suiteId}`) + lines.push("") + lines.push(`Generated: ${report.generatedAt}`) + lines.push("") + lines.push("## Pinned inputs and compatibility") + lines.push("") + lines.push(`- Baseline commit: \`${report.pins.baseline.commit}\``) + lines.push(`- Candidate commit: \`${report.pins.candidate.commit}\``) + lines.push(`- Suite: ${report.pins.baseline.suiteId} / ${report.pins.candidate.suiteId}`) + lines.push(`- Suite digest: \`${report.pins.baseline.suiteDigest}\` / \`${report.pins.candidate.suiteDigest}\``) + lines.push( + `- Settings digest: \`${report.pins.baseline.settingsDigest}\` / \`${report.pins.candidate.settingsDigest}\``, + ) + lines.push(`- Model: ${report.pins.baseline.modelIdentifier} / ${report.pins.candidate.modelIdentifier}`) + lines.push(`- Grader version: ${report.pins.baseline.graderVersion} / ${report.pins.candidate.graderVersion}`) + lines.push( + `- Runner environment: ${report.pins.baseline.runnerEnvironment.os}, node ${report.pins.baseline.runnerEnvironment.nodeVersion}, vscode ${report.pins.baseline.runnerEnvironment.vscodeTarget} / ${report.pins.candidate.runnerEnvironment.os}, node ${report.pins.candidate.runnerEnvironment.nodeVersion}, vscode ${report.pins.candidate.runnerEnvironment.vscodeTarget}`, + ) + lines.push("") + if (report.compatibility.compatible) lines.push("**Compatible pair.** All pinned inputs match.") + else + lines.push( + `**Incompatible pair.** Differing pinned inputs: ${report.compatibility.differences.join(", ")}. Treat the comparison as invalid.`, + ) + lines.push("") + lines.push("## Method") + lines.push("") + lines.push( + `- ${report.repetitions} repetitions per fixture per commit (default ${STATISTICAL_POLICY.repetitionsDefault}).`, + ) + lines.push(`- ${report.statisticalPolicy.method}.`) + lines.push(`- ${report.statisticalPolicy.significanceClaim}`) + lines.push( + `- Partial-data policy: ${report.partialDataPolicy}. Runs with incomplete cost accounting are flagged and excluded from cost aggregates.`, + ) + lines.push("") + lines.push("## Results per fixture") + lines.push("") + lines.push("| Fixture | Baseline | Candidate | Paired (improved/regressed/unchanged) |") + lines.push("| --- | --- | --- | --- |") + for (const row of report.fixtures) + lines.push( + `| ${row.fixtureName} | ${rateCell(row.baseline)} | ${rateCell(row.candidate)} | ${row.pairedOutcomes.improved}/${row.pairedOutcomes.regressed}/${row.pairedOutcomes.unchanged} |`, + ) + lines.push("") + lines.push("## Cost per successful task") + lines.push("") + lines.push(`- Baseline: ${costCell(report.costSummary.baseline)}`) + lines.push(`- Candidate: ${costCell(report.costSummary.candidate)}`) + lines.push(`- Price table: recorded per run with identifier and date; cache reads and writes priced separately.`) + lines.push("") + lines.push("## Unavailable telemetry") + lines.push("") + lines.push( + "Duplicate-call detection and HTTP retry counts are unsupported and reported as unavailable. They are never inferred from unrelated events.", + ) + lines.push("") + lines.push("## Per-run appendix") + lines.push("") + lines.push("| Side | Fixture | Rep | Outcome | Reason | Duration ms |") + lines.push("| --- | --- | --- | --- | --- | --- |") + for (const run of report.runs) + lines.push( + `| ${run.side} | ${run.fixtureName} | ${run.repIndex} | ${run.outcome.outcome} | ${run.outcome.reason} | ${run.durationMs ?? "unavailable"} |`, + ) + lines.push("") + return lines.join("\n") +} diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts new file mode 100644 index 0000000000..dce16722b6 --- /dev/null +++ b/scripts/agent-evals/coding-run.test.ts @@ -0,0 +1,163 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" +import { runCodingFixtureOnce, withWorkspaceCapture } from "./coding-run.ts" +import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" +import type { AgentClient, AgentHost, AgentRuntime, FinalWorkspaceSnapshot } from "./runner.ts" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" +import type { AgentTrace } from "./trace.ts" +import { codingBundle } from "./coding-suite.ts" + +// Deterministic clock: each pause advances time by 100 ms so the substrate's +// bounded windows expire without real waiting. +interface SyntheticOptions { + finalFiles: AgentFixtureWorkspaceFile[] + complete: boolean +} + +const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; traces: AgentTrace[] } => { + const traces: AgentTrace[] = [] + let listener: ((event: unknown) => void) | undefined + let fakeNow = 0 + const host: AgentHost = { + get exited() { + return true + }, + onExit: () => {}, + signal: () => {}, + } + const client: AgentClient = { + isReady: true, + onTaskEvent: (l) => { + listener = l + }, + onDisconnect: () => {}, + sendCommand: (command: TaskCommand) => { + if (command.commandName !== TaskCommandName.StartNewTask) return + const requestId = (command.data as { requestId: string }).requestId + const emit = (event: TaskEvent): void => listener?.(event) + emit({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1" }], + }) + if (options.complete) + emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "task-1", + { totalTokensIn: 1_000, totalTokensOut: 200, totalCost: 0.01, contextTokens: 5_000 }, + { apply_diff: { attempts: 1, failures: 0 } }, + { isSubtask: false }, + ], + }) + }, + disconnect: () => {}, + } + const runtime: AgentRuntime = { + launch: async () => host, + connect: () => client, + cleanup: async () => {}, + writeArtifact: async (fileName, value) => { + if (fileName === "trace.json") traces.push(value as AgentTrace) + }, + onInterrupt: () => () => {}, + now: () => fakeNow, + pause: async () => { + fakeNow += 100 + }, + readFinalWorkspace: async (): Promise => ({ + files: options.finalFiles, + truncated: false, + }), + } + return { runtime, traces } +} + +const finalFilesWith = (solution: AgentFixtureWorkspaceFile): AgentFixtureWorkspaceFile[] => { + const bundle = codingBundle("clamp-range") + return bundle.fixture.workspace.files.map((file) => (file.path === solution.path ? { ...solution } : { ...file })) +} + +describe("coding run orchestration", () => { + const scratch = async (): Promise => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-run-test-")) + process.once("exit", () => { + void fs.rm(dir, { recursive: true, force: true }) + }) + return dir + } + + it("grades a completed correct run as correct", async () => { + const bundle = codingBundle("clamp-range") + const { runtime, traces } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 0, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + assert.equal(record.outcome.outcome, "correct") + assert.equal(record.executionCode, "COMPLETED") + assert.equal(record.grading?.kind, "passed") + assert.equal(record.metrics.outcome.outcome, "correct") + // Usage lacks cache counts, so cost accounting stays incomplete. + assert.equal(record.metrics.cost.accountingComplete, false) + assert.equal(traces.length, 1) + }) + + it("grades a completed but wrong workspace as incorrect", async () => { + const bundle = codingBundle("clamp-range") + const incorrect = bundle.grading.incorrectSolution[0]! + const { runtime } = syntheticRuntime({ finalFiles: finalFilesWith(incorrect), complete: true }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 1, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // The agent completed; the hidden tests still fail. + assert.equal(record.outcome.outcome, "incorrect") + assert.equal(record.grading?.kind, "failed") + assert.ok((record.grading?.failedCount ?? 0) > 0) + }) + + it("reports a run-window expiry as timed out without grading", async () => { + const { runtime } = syntheticRuntime({ finalFiles: [], complete: false }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 2, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + assert.equal(record.outcome.outcome, "timed-out") + assert.equal(record.grading, undefined) + assert.equal(record.executionCode, "TASK_RUN_TIMEOUT") + }) + + it("captures the workspace only through cleanup and never leaks scratch paths", async () => { + const bundle = codingBundle("clamp-range") + const inner = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: false, + }) + const captured = withWorkspaceCapture(inner.runtime) + assert.equal(captured.getSnapshot(), undefined) + await captured.runtime.cleanup() + const snapshot = captured.getSnapshot() + assert.ok(snapshot) + assert.equal(snapshot.truncated, false) + assert.ok(snapshot.files.length > 0) + }) +}) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts new file mode 100644 index 0000000000..3d7599268e --- /dev/null +++ b/scripts/agent-evals/coding-run.ts @@ -0,0 +1,144 @@ +import { runAgentFixture, type AgentRuntime, type FinalWorkspaceSnapshot } from "./runner.ts" +import { getFixture } from "./registry.ts" +import { ensureCodingSuiteRegistered, codingBundle } from "./coding-suite.ts" +import { runGrader, type GraderVerdict, type GradingRuntime } from "./coding-grader.ts" +import { classifyRunOutcome, type ClassifiedOutcome } from "./coding-outcomes.ts" +import { buildRunMetrics, type CodingRunMetrics } from "./coding-metrics.ts" +import { CODING_MODEL_ID } from "./coding-fixtures.ts" +import { measured, type AgentTrace, type Measured, type MeasuredString } from "./trace.ts" + +export interface CodingRunRecord { + fixtureName: string + fixtureVersion: string + repIndex: number + outcome: ClassifiedOutcome + executionCode?: string + trace?: AgentTrace + grading?: { + kind: GraderVerdict["kind"] + isolation?: string + reason?: string + failedCount?: number + failures?: { name: string; file: string }[] + } + finalWorkspaceTruncated?: boolean + metrics: CodingRunMetrics +} + +// Serializes a verdict for records and reports. +export const serializeVerdict = (verdict: GraderVerdict): CodingRunRecord["grading"] => { + switch (verdict.kind) { + case "passed": + case "failed": + return { + kind: verdict.kind, + isolation: verdict.isolation, + failedCount: verdict.result.failedCount, + failures: verdict.kind === "failed" ? verdict.result.failures.slice(0, 10) : undefined, + } + case "grading-failure": + return { kind: verdict.kind, isolation: verdict.isolation, reason: verdict.reason } + } +} + +const modelIdentifierFromProjection = (projection: Record): MeasuredString => + measured( + typeof projection["openAiModelId"] === "string" ? (projection["openAiModelId"] as string) : undefined, + "modelIdMissingInProjection", + ) + +// Wraps a runtime so the final workspace is captured inside cleanup, before +// the launcher removes its scratch directory. The captured file data feeds +// only the fresh grading copy. +export function withWorkspaceCapture(inner: AgentRuntime): { + runtime: AgentRuntime + getSnapshot: () => FinalWorkspaceSnapshot | undefined +} { + let snapshot: FinalWorkspaceSnapshot | undefined + // Local binding preserves the narrowing for the optional identity reader. + const readIdentity = inner.readIdentity?.bind(inner) + const runtime: AgentRuntime = { + launch: (files) => inner.launch(files), + connect: () => inner.connect(), + cleanup: async () => { + try { + snapshot = await inner.readFinalWorkspace?.() + } catch { + snapshot = { files: [], truncated: true } + } + await inner.cleanup() + }, + writeArtifact: (fileName, value) => inner.writeArtifact(fileName, value), + onInterrupt: (listener) => inner.onInterrupt(listener), + now: () => inner.now(), + pause: (ms) => inner.pause(ms), + readIdentity: readIdentity ? () => readIdentity() : undefined, + readFinalWorkspace: () => + inner.readFinalWorkspace?.() ?? Promise.reject(new Error("finalWorkspaceUnsupported")), + } + return { runtime, getSnapshot: () => snapshot } +} + +export interface CodingRunInputs { + fixtureName: string + repIndex: number + credential: string + runtime: AgentRuntime + gradingRuntime: GradingRuntime + scratchDir: string +} + +// One fixture repetition: substrate run, final-workspace capture, hidden-test +// grading, outcome classification, and metrics. The grader receives no +// credential. The record embeds the metadata-only trace. +export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { + ensureCodingSuiteRegistered() + const bundle = codingBundle(inputs.fixtureName) + const { runtime, getSnapshot } = withWorkspaceCapture(inputs.runtime) + const result = await runAgentFixture(runtime, { + fixtureName: inputs.fixtureName, + credential: inputs.credential, + }) + const trace: AgentTrace = result.trace + const snapshot = getSnapshot() + const finalWorkspaceAvailable = snapshot !== undefined + let grading: GraderVerdict | undefined + const code = result.execution.code + if ((code === "COMPLETED" || code === "STOPPED") && snapshot) { + grading = await runGrader({ + finalFiles: snapshot.files, + hiddenTests: bundle.grading.hiddenTests, + resources: bundle.grading.resources, + runtime: inputs.gradingRuntime, + scratchDir: inputs.scratchDir, + }) + } + const outcome = classifyRunOutcome({ + hasTrace: true, + executionCode: code, + finalWorkspaceAvailable, + grading, + }) + const metrics = buildRunMetrics({ + fixtureName: inputs.fixtureName, + repIndex: inputs.repIndex, + trace, + grading, + outcome, + modelIdentifier: modelIdentifierFromProjection(bundle.fixture.settingsProjection), + }) + return { + fixtureName: inputs.fixtureName, + fixtureVersion: bundle.fixture.identity.version, + repIndex: inputs.repIndex, + outcome, + executionCode: code, + trace, + grading: grading ? serializeVerdict(grading) : undefined, + finalWorkspaceTruncated: snapshot?.truncated, + metrics, + } +} + +// The suite model identifier, for pinned inputs. +export const CODING_RUN_MODEL_ID = CODING_MODEL_ID diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts new file mode 100644 index 0000000000..c4086e8252 --- /dev/null +++ b/scripts/agent-evals/coding-side.mts @@ -0,0 +1,135 @@ +// One comparison side. Runs every coding fixture for N repetitions on this +// commit through the substrate, grades each final workspace, and writes one +// side-results file with the pinned inputs. Trusted repo code only: the +// provider credential stays with this process and the extension host; the +// grader never receives it. +import fs from "node:fs/promises" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { createHostRuntime } from "./launcher.ts" +import { createContainerGradingRuntime } from "./container-runtime.ts" +import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" +import { runCodingFixtureOnce, type CodingRunRecord } from "./coding-run.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from "./coding-suite.ts" +import { GRADER_VERSION } from "./coding-grader.ts" +import { digestValue } from "./coding-digest.ts" +import type { SidePins } from "./coding-pins.ts" +import type { SideResultFile } from "./coding-compare.ts" +import { classifyRunOutcome } from "./coding-outcomes.ts" +import { buildRunMetrics } from "./coding-metrics.ts" + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +const readArgs = (): { reps: number; out: string; credentialEnv: string; allowUnisolatedGrading: boolean } => { + const args = process.argv.slice(2) + const read = (name: string): string | undefined => { + const index = args.indexOf(`--${name}`) + return index >= 0 ? args[index + 1] : undefined + } + const out = read("out") + if (!out) throw new Error("Set --out to a scratch directory") + return { + reps: Number(read("reps") ?? "3"), + out: path.resolve(out), + credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + } +} + +const readVscodeTarget = async (): Promise => { + try { + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies?: Record + } + return pkg.devDependencies?.["@types/vscode"] ?? "unavailable" + } catch { + return "unavailable" + } +} + +const readCommit = async (): Promise => { + const { execFile } = await import("node:child_process") + const { promisify } = await import("node:util") + try { + const { stdout } = await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) + return stdout.trim() + } catch { + return "commit-unavailable" + } +} + +const { reps, out, credentialEnv, allowUnisolatedGrading } = readArgs() +if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") +await fs.mkdir(out, { recursive: true }) +ensureCodingSuiteRegistered() + +const credential = process.env[credentialEnv] ?? "" +const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() + +const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} +const pins: SidePins = { + side: "baseline", + commit: await readCommit(), + suiteId: CODING_SUITE_ID, + suiteDigest: codingSuiteDigest(), + settingsDigest: digestValue(projection), + modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", + graderVersion: GRADER_VERSION, + runnerEnvironment: { + os: process.platform, + nodeVersion: process.version, + vscodeTarget: await readVscodeTarget(), + }, +} + +const runs: CodingRunRecord[] = [] +for (const bundle of CODING_FIXTURE_BUNDLES) { + const fixtureName = bundle.fixture.identity.name + for (let repIndex = 0; repIndex < reps; repIndex++) { + const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) + await fs.mkdir(artifactsDir, { recursive: true }) + try { + const record = await runCodingFixtureOnce({ + fixtureName, + repIndex, + credential, + runtime: createHostRuntime({ root, artifactsDir }), + gradingRuntime, + scratchDir: artifactsDir, + }) + runs.push(record) + } catch { + // The substrate never throws for task failures. This guards wiring + // errors only, and records them as infrastructure failures. + const outcome = classifyRunOutcome({ hasTrace: false, finalWorkspaceAvailable: false }) + runs.push({ + fixtureName, + fixtureVersion: bundle.fixture.identity.version, + repIndex, + outcome, + metrics: buildRunMetrics({ + fixtureName, + repIndex, + outcome, + modelIdentifier: { available: false, reason: "runFailed" }, + }), + }) + } + } +} + +// Side identity must reflect the actual side label in a comparison context. +const sideLabel = (process.env["CODING_EVALS_SIDE"] as SidePins["side"] | undefined) ?? "baseline" +const result: SideResultFile = { pins: { ...pins, side: sideLabel }, runs } + +// Keep the trace of each run with its artifacts for inspection. +for (const record of runs) { + if (!record.trace) continue + const artifactDir = path.join(out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) + await fs.writeFile(path.join(artifactDir, "run-record.json"), JSON.stringify(record, null, 2)) +} + +await fs.writeFile(path.join(out, "side-results.json"), JSON.stringify(result, null, 2)) +const correct = runs.filter((record) => record.outcome.outcome === "correct").length +console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-stats.test.ts b/scripts/agent-evals/coding-stats.test.ts new file mode 100644 index 0000000000..c7e86dcd47 --- /dev/null +++ b/scripts/agent-evals/coding-stats.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + DEFAULT_REPETITIONS, + STATISTICAL_POLICY, + pairedNumericDiffs, + pairedOutcomeDiffs, + successRate, + wilsonInterval, +} from "./coding-stats.ts" + +describe("statistics", () => { + it("defaults to three repetitions per fixture per commit", () => { + assert.equal(DEFAULT_REPETITIONS, 3) + assert.equal(STATISTICAL_POLICY.repetitionsDefault, 3) + }) + + it("computes the Wilson interval without claiming significance", () => { + const perfect = wilsonInterval(3, 3) + assert.ok(perfect.lower > 0.29, `lower bound too low: ${perfect.lower}`) + assert.ok(perfect.upper <= 1) + const none = wilsonInterval(0, 3) + assert.equal(none.lower, 0) + assert.ok(none.upper < 0.71) + assert.deepEqual(wilsonInterval(1, 0), { lower: 0, upper: 0 }) + assert.match(STATISTICAL_POLICY.significanceClaim, /cannot show small improvements/) + assert.equal(STATISTICAL_POLICY.method.includes("wilson"), true) + }) + + it("summarizes success rates over outcomes", () => { + const rate = successRate(["correct", "correct", "incorrect"] as const) + assert.equal(rate.successes, 2) + assert.equal(rate.total, 3) + assert.ok(Math.abs(rate.rate - 2 / 3) < 1e-9) + assert.ok(rate.interval !== undefined) + assert.equal(successRate([]).interval, undefined) + }) + + it("pairs outcome differences by repetition index", () => { + const diff = pairedOutcomeDiffs( + ["correct", "incorrect", "correct", "correct"], + ["incorrect", "correct", "correct", "grading-failure"], + ) + assert.equal(diff.pairs, 4) + assert.equal(diff.improved, 1) + assert.equal(diff.regressed, 2) + assert.equal(diff.unchanged, 1) + }) + + it("pairs numeric deltas and counts missing pairs without zero filling", () => { + const diff = pairedNumericDiffs([1, 2, undefined, 4], [3, undefined, 5, 8]) + assert.equal(diff.pairs.length, 2) + assert.equal(diff.missingPairs, 2) + assert.deepEqual( + diff.pairs.map((pair) => pair.delta).sort((a, b) => a - b), + [2, 4], + ) + assert.ok(Math.abs((diff.meanDelta ?? 0) - 3) < 1e-9) + const empty = pairedNumericDiffs([], []) + assert.equal(empty.meanDelta, undefined) + }) +}) diff --git a/scripts/agent-evals/coding-stats.ts b/scripts/agent-evals/coding-stats.ts new file mode 100644 index 0000000000..5ac1befefd --- /dev/null +++ b/scripts/agent-evals/coding-stats.ts @@ -0,0 +1,118 @@ +import type { CodingRunOutcome } from "./coding-outcomes.ts" + +// Uncertainty constant for the Wilson score interval at the 95% level. The +// interval describes uncertainty. It is not a significance test. +export const UNCERTAINTY_Z = 1.96 + +export const DEFAULT_REPETITIONS = 3 + +export const STATISTICAL_POLICY = { + method: "success-rate-with-wilson-95-percent-interval-and-paired-deltas", + significanceClaim: "none. Three repetitions can show large regressions. They cannot show small improvements.", + repetitionsDefault: DEFAULT_REPETITIONS, +} as const + +export interface SuccessRate { + successes: number + total: number + rate: number + interval: { lower: number; upper: number } | undefined +} + +// Wilson score interval. n = 0 yields no interval. +export function wilsonInterval( + successes: number, + n: number, + z: number = UNCERTAINTY_Z, +): { lower: number; upper: number } { + if (n <= 0) return { lower: 0, upper: 0 } + const p = successes / n + const z2 = z * z + const denominator = 1 + z2 / n + const center = (p + z2 / (2 * n)) / denominator + const spread = (z / denominator) * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n)) + return { lower: Math.max(0, center - spread), upper: Math.min(1, center + spread) } +} + +export function successRate(outcomes: readonly CodingRunOutcome[]): SuccessRate { + const total = outcomes.length + const successes = outcomes.filter((outcome) => outcome === "correct").length + return { + successes, + total, + rate: total === 0 ? 0 : successes / total, + interval: total === 0 ? undefined : wilsonInterval(successes, total), + } +} + +export interface PairedOutcomeDiff { + pairs: number + improved: number + regressed: number + unchanged: number + definition: string +} + +// Paired by repetition index. Improvement means the candidate run is correct +// where the baseline run is not. Regression is the reverse. No significance +// claim follows from these counts. +export function pairedOutcomeDiffs( + baseline: readonly CodingRunOutcome[], + candidate: readonly CodingRunOutcome[], +): PairedOutcomeDiff { + const pairs = Math.min(baseline.length, candidate.length) + let improved = 0 + let regressed = 0 + let unchanged = 0 + for (let index = 0; index < pairs; index++) { + const baseCorrect = baseline[index] === "correct" + const candCorrect = candidate[index] === "correct" + if (baseCorrect === candCorrect) unchanged++ + else if (candCorrect) improved++ + else regressed++ + } + return { + pairs, + improved, + regressed, + unchanged, + definition: "paired by repetition index; correct versus not correct", + } +} + +export interface PairedNumericDiff { + pairs: { baseline: number; candidate: number; delta: number }[] + missingPairs: number + meanDelta?: number + minDelta?: number + maxDelta?: number +} + +// Paired numeric deltas. A pair needs both sides reported; pairs with a +// missing value on either side are counted as missing, never as zero. +export function pairedNumericDiffs( + baseline: readonly (number | undefined)[], + candidate: readonly (number | undefined)[], +): PairedNumericDiff { + const pairs: { baseline: number; candidate: number; delta: number }[] = [] + let missingPairs = 0 + const count = Math.min(baseline.length, candidate.length) + for (let index = 0; index < count; index++) { + const base = baseline[index] + const cand = candidate[index] + if (base === undefined || cand === undefined) { + missingPairs++ + continue + } + pairs.push({ baseline: base, candidate: cand, delta: cand - base }) + } + if (pairs.length === 0) return { pairs, missingPairs } + const deltas = pairs.map((pair) => pair.delta) + return { + pairs, + missingPairs, + meanDelta: deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length, + minDelta: Math.min(...deltas), + maxDelta: Math.max(...deltas), + } +} diff --git a/scripts/agent-evals/coding-suite.ts b/scripts/agent-evals/coding-suite.ts new file mode 100644 index 0000000000..ca7b10c452 --- /dev/null +++ b/scripts/agent-evals/coding-suite.ts @@ -0,0 +1,53 @@ +import { registerFixture, getFixture, listFixtures } from "./registry.ts" +import { validateFixture } from "./contract.ts" +import { digestValue } from "./coding-digest.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { GRADER_VERSION } from "./coding-grader.ts" + +// Frozen suite identity. The digest pins the exact fixture content, hidden +// tests, reference solutions, and grader version. Any edit changes the digest +// and marks cross-commit comparisons incompatible. +export const CODING_SUITE_ID = "coding-basic-v1" + +let registered = false + +export function ensureCodingSuiteRegistered(): void { + if (registered) return + registered = true + for (const bundle of CODING_FIXTURE_BUNDLES) { + validateFixture(bundle.fixture) + registerFixture(bundle.fixture) + } +} + +export function codingBundle(fixtureName: string) { + const found = CODING_FIXTURE_BUNDLES.find((candidate) => candidate.fixture.identity.name === fixtureName) + if (!found) throw new Error(`unknown coding fixture: ${fixtureName}`) + return found +} + +// Suite digest. Canonical JSON over the frozen suite content and the grader +// identity. Computed from the in-repo data only, so the same digest holds on +// every machine. +export function codingSuiteDigest(): string { + return digestValue({ + kind: "agent-evals-coding-suite-v1", + suiteId: CODING_SUITE_ID, + graderVersion: GRADER_VERSION, + fixtures: CODING_FIXTURE_BUNDLES.map((bundle) => ({ + identity: bundle.fixture.identity, + prompt: bundle.fixture.prompt, + workspace: bundle.fixture.workspace, + toolPolicy: bundle.fixture.toolPolicy, + limits: bundle.fixture.limits, + settingsProjection: bundle.fixture.settingsProjection, + hiddenTests: bundle.grading.hiddenTests, + correctSolution: bundle.grading.correctSolution, + incorrectSolution: bundle.grading.incorrectSolution, + resources: bundle.grading.resources, + })), + }) +} + +// Re-exports for side runners and checks. +export { listFixtures } diff --git a/scripts/agent-evals/coding-workspace.test.ts b/scripts/agent-evals/coding-workspace.test.ts new file mode 100644 index 0000000000..aa3228b255 --- /dev/null +++ b/scripts/agent-evals/coding-workspace.test.ts @@ -0,0 +1,102 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import type { AgentFixture } from "./contract.ts" +import { buildGradingCopy, buildFreshWorkspace, initialStateDigest } from "./coding-workspace.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" + +const tmpdir = async (): Promise => fs.mkdtemp(path.join(os.tmpdir(), "coding-workspace-test-")) + +const anyBundle = (): (typeof CODING_FIXTURE_BUNDLES)[number] => CODING_FIXTURE_BUNDLES[0]! + +describe("initial state digest", () => { + it("is stable for the same fixture content", () => { + const fixture: AgentFixture = anyBundle().fixture + assert.equal(initialStateDigest(fixture), initialStateDigest(fixture)) + }) + + it("is independent of file declaration order", async () => { + const fixture: AgentFixture = anyBundle().fixture + const reordered: AgentFixture = { + ...fixture, + workspace: { files: [...fixture.workspace.files].reverse() }, + } + assert.equal(initialStateDigest(fixture), initialStateDigest(reordered)) + }) + + it("changes when any workspace file changes", () => { + const fixture: AgentFixture = anyBundle().fixture + const edited: AgentFixture = { + ...fixture, + workspace: { + files: fixture.workspace.files.map((file, index) => + index === 0 ? { ...file, content: `${file.content}\n` } : file, + ), + }, + } + assert.notEqual(initialStateDigest(fixture), initialStateDigest(edited)) + }) +}) + +describe("fresh workspace builder", () => { + it("writes the fixture files into a new directory", async () => { + const bundle = anyBundle() + const target = path.join(await tmpdir(), "workspace") + const result = await buildFreshWorkspace(bundle.fixture, target) + assert.equal(result.filesWritten, bundle.fixture.workspace.files.length) + for (const file of bundle.fixture.workspace.files) { + const written = await fs.readFile(path.join(target, file.path), "utf8") + assert.equal(written, file.content) + } + }) + + it("refuses to build into an existing directory", async () => { + const bundle = anyBundle() + const target = await tmpdir() + await assert.rejects(() => buildFreshWorkspace(bundle.fixture, target)) + }) + + it("keeps two builds independent", async () => { + const bundle = anyBundle() + const parent = await tmpdir() + const first = path.join(parent, "first") + const second = path.join(parent, "second") + await buildFreshWorkspace(bundle.fixture, first) + await buildFreshWorkspace(bundle.fixture, second) + // Mutating one build must not affect the other. + await fs.writeFile(path.join(first, bundle.fixture.workspace.files[0]!.path), "mutated") + const secondContent = await fs.readFile(path.join(second, bundle.fixture.workspace.files[0]!.path), "utf8") + assert.equal(secondContent, bundle.fixture.workspace.files[0]!.content) + }) +}) + +describe("grading copy", () => { + it("adds hidden tests only to the copy", async () => { + const bundle = anyBundle() + const hidden = bundle.grading.hiddenTests[0]! + const target = path.join(await tmpdir(), "grading") + await buildGradingCopy(bundle.fixture.workspace.files, bundle.grading.hiddenTests, target) + await fs.access(path.join(target, hidden.path)) + // The run workspace itself must never contain the hidden tests. + assert.equal( + bundle.fixture.workspace.files.some((file) => file.path === hidden.path), + false, + ) + }) + + it("lets hidden tests shadow an agent file with the same path", async () => { + const bundle = anyBundle() + const hidden = bundle.grading.hiddenTests[0]! + const target = path.join(await tmpdir(), "grading-shadow") + const tampered = [...bundle.fixture.workspace.files, { path: hidden.path, content: "// tampered" }] + await buildGradingCopy(tampered, bundle.grading.hiddenTests, target) + assert.equal(await fs.readFile(path.join(target, hidden.path), "utf8"), hidden.content) + }) + + it("rejects escaping paths", async () => { + const target = path.join(await tmpdir(), "grading-escape") + await assert.rejects(() => buildGradingCopy([{ path: "../escape.txt", content: "x" }], [], target)) + }) +}) diff --git a/scripts/agent-evals/coding-workspace.ts b/scripts/agent-evals/coding-workspace.ts new file mode 100644 index 0000000000..70182a2417 --- /dev/null +++ b/scripts/agent-evals/coding-workspace.ts @@ -0,0 +1,49 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { validateWorkspaceFilePath, type AgentFixture, type AgentFixtureWorkspaceFile } from "./contract.ts" +import { digestValue } from "./coding-digest.ts" + +// Immutable initial-state digest. The digest pins the exact fixture workspace +// content, so every run of the same fixture version rebuilds byte-identical +// initial state and any fixture edit changes the pinned suite digest. +export function initialStateDigest(fixture: AgentFixture): string { + const sorted = [...fixture.workspace.files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + return digestValue({ kind: "agent-evals-initial-workspace-v1", files: sorted }) +} + +// Builds the fixture workspace in a fresh directory. The directory must not +// exist yet, so no run can inherit files from an earlier run. +export async function buildFreshWorkspace(fixture: AgentFixture, targetDir: string): Promise<{ filesWritten: number }> { + await fs.mkdir(targetDir, { recursive: false }) + let filesWritten = 0 + for (const file of fixture.workspace.files) { + await writeContainedFile(targetDir, file) + filesWritten++ + } + return { filesWritten } +} + +// Writes one file inside root. The containment check mirrors the launcher's +// write-time check so grader-side copies cannot be tricked into escaping. +export async function writeContainedFile(root: string, file: AgentFixtureWorkspaceFile): Promise { + validateWorkspaceFilePath(file.path) + const rootReal = await fs.realpath(root) + const target = path.resolve(rootReal, file.path) + if (target !== rootReal && !target.startsWith(rootReal + path.sep)) + throw new Error("workspace file path escapes the workspace") + await fs.mkdir(path.dirname(target), { recursive: true }) + await fs.writeFile(target, file.content) + const written = await fs.realpath(target) + if (!written.startsWith(rootReal + path.sep)) throw new Error("workspace file escapes the workspace") +} + +// Builds a fresh grading copy of a final workspace. Hidden acceptance tests +// are added only to the copy, never to the workspace the agent used. +export async function buildGradingCopy( + finalFiles: readonly AgentFixtureWorkspaceFile[], + hiddenTests: readonly AgentFixtureWorkspaceFile[], + targetDir: string, +): Promise { + await fs.mkdir(targetDir, { recursive: false }) + for (const file of [...finalFiles, ...hiddenTests]) await writeContainedFile(targetDir, file) +} diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts new file mode 100644 index 0000000000..afc27c96fa --- /dev/null +++ b/scripts/agent-evals/container-runtime.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { buildContainerArgs, createContainerGradingRuntime, GRADER_IMAGE } from "./container-runtime.ts" +import type { GraderJob } from "./coding-grader.ts" + +const makeJob = (): GraderJob => ({ + gradingDir: "/tmp/evals/grading-copy", + driverPath: "/repo/scripts/agent-evals/grader-driver.mjs", + resources: { timeoutMs: 30_000, memoryMb: 512, cpus: 1, pidsLimit: 128, image: GRADER_IMAGE }, +}) + +describe("container grading runtime", () => { + it("builds a locked-down, network-less invocation", () => { + const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out") + const flagValue = (flag: string): string | undefined => { + const index = args.indexOf(flag) + return index === -1 ? undefined : args[index + 1] + } + assert.equal(flagValue("--network"), "none") + assert.equal(flagValue("--memory"), "512m") + assert.equal(flagValue("--memory-swap"), "512m") + assert.equal(flagValue("--cpus"), "1") + assert.equal(flagValue("--pids-limit"), "128") + assert.equal(flagValue("--cap-drop"), "ALL") + assert.equal(flagValue("--security-opt"), "no-new-privileges") + assert.ok(args.includes("--read-only")) + const tmpfs = flagValue("--tmpfs") + assert.match(tmpfs ?? "", /^\/tmp:rw/) + assert.match(tmpfs ?? "", /noexec/) + // The grading copy and driver mount read-only; only the result + // directory is writable. + assert.equal(flagValue("-v"), "/tmp/evals/grading-copy:/work:ro") + assert.ok(args.some((arg) => arg === "/tmp/evals/grading-out:/out:rw")) + assert.ok(args.some((arg) => arg.endsWith(":/driver/grader-driver.mjs:ro"))) + // The pinned image and driver command close the argv. + const imageIndex = args.indexOf(GRADER_IMAGE) + assert.ok(imageIndex > 0) + assert.deepEqual(args.slice(imageIndex), [ + GRADER_IMAGE, + "node", + "/driver/grader-driver.mjs", + "/out/result.json", + "test", + ]) + // No environment passes through, so no credential can leak in. + assert.equal(args.includes("-e"), false) + assert.equal(args.includes("--env"), false) + }) + + it("reports a missing container runtime as grading failure", async () => { + const runtime = createContainerGradingRuntime({ detectExecutable: async () => undefined }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "container", reason: "noContainerRuntime" }) + }) + + it("uses the detected executable and records container isolation", async () => { + // Detection seam returns undefined here; the executable contract is + // covered by the argv test above. This test pins the isolation label. + const runtime = createContainerGradingRuntime() + assert.equal(runtime.isolation, "container") + }) +}) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts new file mode 100644 index 0000000000..1d5a263326 --- /dev/null +++ b/scripts/agent-evals/container-runtime.ts @@ -0,0 +1,141 @@ +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import path from "node:path" +import type { GraderJob, GraderVerdict, GradingRuntime, DriverResult } from "./coding-grader.ts" + +// Pinned grader image. The image tag is part of the grader identity and is +// recorded in every pinned comparison through the grader version. +export const GRADER_IMAGE = "node:22-bookworm" + +export type ContainerExecutable = "docker" | "podman" + +export interface ContainerRuntimeOptions { + executable?: ContainerExecutable + // Test seam: detects the container binary instead of searching PATH. + detectExecutable?: () => Promise +} + +const defaultDetect = async (): Promise => { + for (const candidate of ["docker", "podman"] as const) { + const usable = await probeExecutable(candidate) + if (usable) return candidate + } + return undefined +} + +const probeExecutable = (candidate: ContainerExecutable): Promise => + new Promise((resolve) => { + try { + const child = spawn(candidate, ["--version"], { stdio: "ignore" }) + const timer = setTimeout(() => { + child.kill("SIGKILL") + resolve(false) + }, 5_000) + child.once("error", () => { + clearTimeout(timer) + resolve(false) + }) + child.once("exit", (code) => { + clearTimeout(timer) + resolve(code === 0) + }) + } catch { + resolve(false) + } + }) + +// Builds one `run` invocation. The container has no network, no Linux +// capabilities, no privilege escalation, a read-only root filesystem, fixed +// CPU, memory, and process limits, and a size-capped scratch tmpfs. The +// grading copy mounts read-only, so agent-written code cannot change the +// hidden tests at grading time. The driver mounts read-only from repo code. +// No environment variable passes through, so no credential can leak in. +export function buildContainerArgs(job: GraderJob, outDir: string): string[] { + return [ + "run", + "--rm", + "--network", + "none", + "--cpus", + String(job.resources.cpus), + "--memory", + `${job.resources.memoryMb}m`, + "--memory-swap", + `${job.resources.memoryMb}m`, + "--pids-limit", + String(job.resources.pidsLimit), + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=67108864", + "-v", + `${job.gradingDir}:/work:ro`, + "-v", + `${outDir}:/out:rw`, + "-v", + `${job.driverPath}:/driver/grader-driver.mjs:ro`, + "-w", + "/work", + job.resources.image, + "node", + "/driver/grader-driver.mjs", + "/out/result.json", + "test", + ] +} + +// Default grading runtime. Runs the driver in a disposable container. The +// host enforces the wall-clock limit and kills the container on expiry. +export function createContainerGradingRuntime(options: ContainerRuntimeOptions = {}): GradingRuntime { + const detect = options.detectExecutable ?? defaultDetect + return { + isolation: "container", + async run(job: GraderJob): Promise { + const executable = await detect() + if (!executable) return { kind: "grading-failure", isolation: "container", reason: "noContainerRuntime" } + const outDir = path.join(path.dirname(job.gradingDir), "grading-out") + await fs.mkdir(outDir, { recursive: true }) + try { + const args = buildContainerArgs(job, outDir) + const exit = await new Promise((resolve) => { + const child = spawn(executable, args, { + stdio: "ignore", + timeout: job.resources.timeoutMs, + killSignal: "SIGKILL", + }) + child.once("error", () => resolve("error")) + child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) + }) + if (exit === "timeout") + return { kind: "grading-failure", isolation: "container", reason: "graderTimeout" } + if (exit === "error" || exit !== 0) + return { kind: "grading-failure", isolation: "container", reason: "graderCrashed" } + const result = await readDriverResult(path.join(outDir, "result.json")) + if (!result) return { kind: "grading-failure", isolation: "container", reason: "graderResultMissing" } + return verdictFromDriver(result, "container") + } catch { + return { kind: "grading-failure", isolation: "container", reason: "graderSetupFailed" } + } finally { + await fs.rm(outDir, { recursive: true, force: true }) + } + }, + } +} + +export async function readDriverResult(outPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(outPath, "utf8")) as DriverResult + return parsed.kind === "completed" || parsed.kind === "driver-error" ? parsed : undefined + } catch { + return undefined + } +} + +export function verdictFromDriver(result: DriverResult, isolation: GraderVerdict["isolation"]): GraderVerdict { + if (result.kind !== "completed") return { kind: "grading-failure", isolation, reason: "driverError" } + if (result.noTestsDiscovered) return { kind: "grading-failure", isolation, reason: "noTestsDiscovered" } + return result.passed ? { kind: "passed", isolation, result } : { kind: "failed", isolation, result } +} diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs new file mode 100644 index 0000000000..203894323b --- /dev/null +++ b/scripts/agent-evals/grader-driver.mjs @@ -0,0 +1,139 @@ +// Acceptance-test driver. Runs the workspace test suite with the Node test +// runner and writes one result JSON file. Test failures are data, not exit +// codes: the driver exits 0 after writing any completed result. A nonzero exit +// or a missing result file means the grader itself failed, not the solution. +// The driver receives no credential and needs no network. +import { spawn } from "node:child_process" +import fs from "node:fs" +import path from "node:path" + +const outPath = process.argv[2] +if (!outPath) { + process.stderr.write("usage: grader-driver.mjs [test-dir]\n") + process.exit(2) +} +const testDir = process.argv[3] ?? "test" + +const writeResult = (result) => { + fs.mkdirSync(path.dirname(outPath), { recursive: true }) + fs.writeFileSync(outPath, JSON.stringify(result)) +} + +const TEST_EXTENSIONS = new Set([".js", ".mjs", ".cjs"]) + +const discoverTestFiles = (dir) => { + const files = [] + const visit = (current) => { + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const full = path.join(current, entry.name) + if (entry.isDirectory()) visit(full) + else if (entry.isFile() && TEST_EXTENSIONS.has(path.extname(entry.name)) && entry.name.includes(".test.")) + files.push(full) + } + } + visit(dir) + return files.sort() +} + +const readCounts = (tap) => { + const read = (name) => { + const match = tap.match(new RegExp(`^# ${name} (\\d+)\\s*$`, "m")) + return match ? Number(match[1]) : undefined + } + return { passed: read("pass"), failed: read("fail"), cancelled: read("cancelled") } +} + +const readFailureNames = (tap) => { + const matches = [...tap.matchAll(/^(\s*)not ok \d+ - (.+)$/gm)].map((match) => ({ + indented: match[1].length > 0, + name: match[2], + })) + // Prefer indented entries: they name the failing tests inside the files. + // Top-level entries name the files and only appear when the whole file + // failed. + const specific = matches.filter((match) => match.indented) + const chosen = specific.length > 0 ? specific : matches + return chosen.slice(0, 50).map((match) => match.name) +} + +// One child per test file. Failure names stay attributable to their file, +// which is how the grader separates hidden-test failures from visible-test +// failures. +const runOneFile = (file) => + new Promise((resolve) => { + const child = spawn(process.execPath, ["--test", "--test-reporter", "tap", file], { + cwd: process.cwd(), + stdio: ["ignore", "pipe", "pipe"], + }) + let tap = "" + child.stdout.on("data", (chunk) => { + tap += String(chunk) + }) + child.stderr.on("data", (chunk) => { + tap += String(chunk) + }) + child.once("error", () => resolve({ ok: false, tap: "" })) + child.once("close", () => resolve({ ok: true, tap })) + }) + +try { + const files = fs.existsSync(testDir) ? discoverTestFiles(testDir) : [] + if (files.length === 0) { + writeResult({ + kind: "completed", + passed: false, + noTestsDiscovered: true, + testFiles: [], + passedCount: 0, + failedCount: 0, + failures: [], + durationMs: 0, + }) + process.exit(0) + } + const started = Date.now() + let passedCount = 0 + let failedCount = 0 + let runnerBroke = false + const failures = [] + for (const file of files) { + const outcome = await runOneFile(file) + if (!outcome.ok) { + runnerBroke = true + break + } + const counts = readCounts(outcome.tap) + if (counts.passed === undefined || counts.failed === undefined) { + // A file without a TAP summary means the runner never reported. + runnerBroke = true + break + } + passedCount += counts.passed + failedCount += counts.failed + if (counts.failed > 0 && failures.length < 50) + for (const name of readFailureNames(outcome.tap)) failures.push({ name, file }) + } + if (runnerBroke) { + writeResult({ kind: "driver-error", message: "testRunnerUnavailable" }) + process.exit(0) + } + writeResult({ + kind: "completed", + passed: failedCount === 0 && passedCount > 0, + noTestsDiscovered: false, + testFiles: files, + passedCount, + failedCount, + failures, + durationMs: Date.now() - started, + }) + process.exit(0) +} catch (error) { + // The driver itself crashed. Record it as a grader failure, never as a + // solution failure. + writeResult({ + kind: "driver-error", + message: error instanceof Error ? error.name : "driver-error", + }) + process.exit(0) +} diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index bfcd92ed0e..5164fc240e 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -7,7 +7,8 @@ import { downloadAndUnzipVSCode } from "@vscode/test-electron" import { IpcClient } from "@roo-code/ipc" import { IpcMessageType } from "@roo-code/types" import { buildChildEnvironment, buildLaunchArgs } from "./isolation.ts" -import type { AgentClient, AgentHost, AgentRuntime } from "./runner.ts" +import type { AgentClient, AgentHost, AgentRuntime, FinalWorkspaceSnapshot } from "./runner.ts" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" import { identityUnavailable, measured, type MeasuredString, type TraceRunIdentity } from "./trace.ts" const execFileAsync = promisify(execFile) @@ -51,6 +52,51 @@ const readJsonField = async ( } } +const FINAL_WORKSPACE_MAX_FILES = 4_096 +const FINAL_WORKSPACE_MAX_FILE_BYTES = 1_048_576 +const FINAL_WORKSPACE_MAX_TOTAL_BYTES = 8_388_608 +const FINAL_WORKSPACE_SKIPPED_DIRS = new Set(["node_modules", ".git"]) + +// Bounded final-workspace capture. The walker skips dependency and VCS +// directories, caps file count and size, and reports truncation instead of +// reading unbounded agent-written trees. Paths stay relative and forward +// slashed, matching the fixture workspace file shape. +async function readWorkspaceFiles(workspaceRoot: string): Promise { + const files: AgentFixtureWorkspaceFile[] = [] + let truncated = false + let totalBytes = 0 + const visit = async (dir: string, prefix: string): Promise => { + const entries = await fs.readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + if (files.length >= FINAL_WORKSPACE_MAX_FILES || totalBytes > FINAL_WORKSPACE_MAX_TOTAL_BYTES) { + truncated = true + return + } + const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (FINAL_WORKSPACE_SKIPPED_DIRS.has(entry.name)) continue + await visit(full, relativePath) + continue + } + if (!entry.isFile()) continue + const stat = await fs.stat(full) + if ( + stat.size > FINAL_WORKSPACE_MAX_FILE_BYTES || + totalBytes + stat.size > FINAL_WORKSPACE_MAX_TOTAL_BYTES + ) { + truncated = true + continue + } + const content = await fs.readFile(full, "utf8") + totalBytes += stat.size + files.push({ path: relativePath, content }) + } + } + await visit(workspaceRoot, "") + return { files, truncated } +} + // The only place where VS Code is downloaded, spawned, isolated and torn // down. Both the smoke driver and future adapters get the same launcher, so // process and credential isolation cannot drift between entry points. @@ -196,5 +242,9 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { arch: fallback.arch, } }, + async readFinalWorkspace(): Promise { + if (!temp) throw new Error("workspace not created") + return readWorkspaceFiles(path.join(temp, "workspace")) + }, } } diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index c671dabc83..d233766dc6 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -37,6 +37,11 @@ export interface AgentClient { // The adapter owns processes, sockets, files and credentials. The controller // only sees these operations, so tests can drive it with synthetic IPC and a // deterministic clock. +export interface FinalWorkspaceSnapshot { + files: readonly AgentFixtureWorkspaceFile[] + truncated: boolean +} + export interface AgentRuntime { launch(workspaceFiles: readonly AgentFixtureWorkspaceFile[]): Promise connect(): AgentClient @@ -46,6 +51,10 @@ export interface AgentRuntime { now(): number pause(ms: number): Promise readIdentity?(): Promise + // Reads the final workspace as bounded file data. Graders consume it from a + // fresh copy; the launcher's scratch paths never leak. Capture must happen + // before cleanup removes the directory. + readFinalWorkspace?(): Promise } export interface AgentRunObservers { From f49cdb2b8458ae0763d4bc9bc9ad0c621da4583c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 23 Oct 2026 03:55:33 +0000 Subject: [PATCH 02/22] fix(evals): secure coding comparisons and correct grading and cost accounting --- scripts/agent-evals/README.md | 31 +- scripts/agent-evals/coding-compare.mts | 118 ++++++-- scripts/agent-evals/coding-compare.ts | 22 ++ scripts/agent-evals/coding-fixtures.test.ts | 18 +- scripts/agent-evals/coding-grader.test.ts | 34 +++ scripts/agent-evals/coding-grader.ts | 35 ++- scripts/agent-evals/coding-metrics.test.ts | 58 ++++ scripts/agent-evals/coding-metrics.ts | 47 +-- scripts/agent-evals/coding-pins.test.ts | 6 +- scripts/agent-evals/coding-pins.ts | 13 +- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/coding-report.test.ts | 41 ++- scripts/agent-evals/coding-run.test.ts | 111 ++++++- scripts/agent-evals/coding-run.ts | 40 ++- scripts/agent-evals/coding-side.mts | 97 ++++-- scripts/agent-evals/coding-trust.test.ts | 98 ++++++ scripts/agent-evals/coding-trust.ts | 100 +++++++ scripts/agent-evals/container-runtime.test.ts | 260 +++++++++++++++- scripts/agent-evals/container-runtime.ts | 279 ++++++++++++++++-- scripts/agent-evals/fs-atomic.test.ts | 52 ++++ scripts/agent-evals/fs-atomic.ts | 26 ++ scripts/agent-evals/grader-driver.mjs | 8 +- scripts/agent-evals/launcher.ts | 8 +- scripts/agent-evals/runner.ts | 30 +- 24 files changed, 1383 insertions(+), 151 deletions(-) create mode 100644 scripts/agent-evals/coding-trust.test.ts create mode 100644 scripts/agent-evals/coding-trust.ts create mode 100644 scripts/agent-evals/fs-atomic.test.ts create mode 100644 scripts/agent-evals/fs-atomic.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 48db29e872..16734fee5b 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -119,25 +119,42 @@ release process: a maintainer runs it locally or on demand. | `coding-stats.ts` | Three repetitions, success rates with Wilson intervals, paired differences. | | `coding-report.ts` | The self-contained JSON report and the Markdown renderer. | | `coding-compare.ts` | The comparison orchestration and the side result shape. | -| `coding-side.mts` | One comparison side: all fixtures times N repetitions on this commit, graded, with pinned inputs. | +| `coding-side.mts` | One comparison side: all fixtures times N repetitions on the evaluated root, graded, with pinned inputs. | | `coding-compare.mts` | Two-commit comparison driver. Writes `comparison-report.json` and `comparison-report.md`. | +| `coding-trust.ts` | The comparison trust policy: approved full commit SHAs, resolution, the approval gate, and the side environment allowlist. | +| `fs-atomic.ts` | Atomic file publication through same-directory temp files and renames. | ### Grading - The grader is model-free and credential-free. It judges the final workspace state only. - The final workspace is captured as bounded file data and rebuilt as a fresh copy. Hidden acceptance tests enter only that copy, so the agent can never read or change them, and one run can never inherit another run's files. -- The default runtime executes the driver in a disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. -- The grader has its own wall-clock limit. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. +- The default runtime executes the driver in a uniquely named disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. +- The grader has its own wall-clock limit. On expiry the runtime removes the container by name and verifies the removal before it deletes any output. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. The image pull happens before the timed run and never consumes grading time. The runtime resolves the immutable grader image digest from the local engine and the side records it in the comparison pins; it never fabricates one. +- The driver's result file is untrusted data: it is written inside the container where agent-written test code executes. The host reads it with a size bound and validates shape, bounds, and internal consistency. An invalid or inconsistent result is a `grading failure`, never a verdict. - Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. - `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. ### Comparisons -A paired comparison pins both commit SHAs, the suite digest, the settings digest, the model identifier, the grader version, and the runner environment on both sides. Any difference in a pinned value marks the pair incompatible. The report records the values and flags the pair instead of dropping it. +A paired comparison pins both commit SHAs, the suite digest, the settings digest, the model identifier, the grader version, the resolved grader image digest, and the runner environment on both sides. Any difference in a pinned value marks the pair incompatible. The report records the values and flags the pair instead of dropping it. The default is three repetitions per fixture per commit. Reports show success rates with Wilson 95% intervals and paired differences. They make no significance claim: three repetitions can show large regressions, not small improvements. -Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct or a correct run lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. +Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct, when a correct run lacks complete cost data, or when any attempt, including a failed one, lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. + +### Trust policy for comparisons + +A comparison runs the extension code of two commits with the provider credential. A commit runs only when its full 40-hex SHA is listed in an explicit trust policy file. Ref names are resolved first and checked against the policy before any worktree is created or any process starts. A policy entry is a full SHA; short SHAs and ref names are rejected. + +```json +{ "version": 1, "approvedCommits": [""] } +``` + +The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). + +### Output publication contract + +Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. `side-results.json` appears only after every repetition of that side finished, so its existence means a complete side. A missing `side-results.json` means an interrupted side; consumers must treat it as a failed side and never pair partial output. The comparison reports are written atomically and only after both sides published complete results; a failed comparison leaves no report files. ### Commands @@ -150,8 +167,8 @@ pnpm agent-evals:check-types # environment variable and the container runtime for grading. pnpm coding-evals:side --reps 3 --out /tmp/side-out -# Full paired comparison of two commits. -pnpm coding-evals:compare --baseline --candidate --reps 3 --out /tmp/compare-out +# Full paired comparison of two approved commits. +pnpm coding-evals:compare --baseline --candidate --trust-file trust-policy.json --reps 3 --out /tmp/compare-out ``` ### Follow-up milestones diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index 771c5c48b6..89bd85fa64 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -1,20 +1,35 @@ -// Paired comparison driver. Checks out both commits into temporary worktrees, -// runs one coding-evals side per commit with identical repetition counts, -// then writes the JSON and Markdown comparison reports. Both commits must -// contain this framework. The driver is trusted repo code and only runs -// trusted repo commits with the credential. +// Paired comparison driver. Verifies both commits against an explicit trust +// policy, checks out each approved commit into a temporary worktree, prepares +// pinned dependencies and builds that exact commit, runs one trusted coding +// side per commit against its worktree with identical repetition counts, then +// writes the JSON and Markdown comparison reports. +// +// Trust model. The side script, fixtures, grader, and report builder all run +// from this trusted checkout. Only the extension host loads the evaluated +// worktree. The credential never leaves the trusted processes except through +// the one named provider variable. A commit runs only when its full SHA is +// listed in the operator's trust policy; refs are resolved and checked +// against the policy before any worktree or process exists. import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" -import { sideResultFileName, toSideResult, type SideResultFile } from "./coding-compare.ts" +import { sideResultFileName, toSideResult, validateSideResultFile, type SideResultFile } from "./coding-compare.ts" import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" +import { assertCommitsApproved, buildSideEnvironment, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" +import { writeJsonAtomic, writeFileAtomic } from "./fs-atomic.ts" const execFileAsync = promisify(execFile) const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const INSTALL_TIMEOUT_MS = 10 * 60_000 +const BUILD_TIMEOUT_MS = 30 * 60_000 +const WORKTREE_TIMEOUT_MS = 120_000 +const SIDE_KILL_GRACE_MS = 30_000 +const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 + const readArgs = (): { baseline: string candidate: string @@ -22,6 +37,8 @@ const readArgs = (): { out: string credentialEnv: string allowUnisolatedGrading: boolean + trustFile: string + sideTimeoutMs: number } => { const args = process.argv.slice(2) const read = (name: string): string | undefined => { @@ -31,10 +48,16 @@ const readArgs = (): { const baseline = read("baseline") const candidate = read("candidate") const out = read("out") + const trustFile = read("trust-file") if (!baseline || !candidate) throw new Error("Set --baseline and --candidate to commit SHAs") if (!out) throw new Error("Set --out to a scratch directory") + if (!trustFile) throw new Error("Set --trust-file to a policy file listing the approved full commit SHAs") const reps = Number(read("reps") ?? "3") if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") + const sideTimeoutMinutes = Number(read("side-timeout-minutes") ?? String(DEFAULT_SIDE_TIMEOUT_MINUTES)) + if (!Number.isInteger(sideTimeoutMinutes) || sideTimeoutMinutes < 1) { + throw new Error("--side-timeout-minutes must be a positive integer") + } return { baseline, candidate, @@ -42,9 +65,26 @@ const readArgs = (): { out: path.resolve(out), credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + trustFile: path.resolve(trustFile), + sideTimeoutMs: sideTimeoutMinutes * 60_000, } } +const runBounded = async (command: string, args: readonly string[], cwd: string, timeoutMs: number): Promise => { + await execFileAsync(command, [...args], { cwd, timeout: timeoutMs, killSignal: "SIGKILL" }) +} + +// Prepares the worktree of one evaluated commit. Dependencies install from +// the worktree's own frozen lockfile, and that exact commit's extension and +// webview build from its own sources. No artifact crosses sides. +const prepareWorktree = async (worktree: string): Promise => { + await runBounded("pnpm", ["install", "--frozen-lockfile"], worktree, INSTALL_TIMEOUT_MS) + await runBounded("pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) +} + +// Runs one trusted side against the evaluated worktree. The side script comes +// from this checkout, receives only the allowlisted environment plus the +// named credential variable, and must finish inside its own wall clock. const runSide = async ( side: "baseline" | "candidate", commit: string, @@ -52,48 +92,90 @@ const runSide = async ( scratchDir: string, ): Promise => { const worktree = path.join(scratchDir, side) - await execFileAsync("git", ["worktree", "add", "--detach", worktree, commit], { cwd: root, timeout: 120_000 }) + await runBounded("git", ["worktree", "add", "--detach", worktree, commit], root, WORKTREE_TIMEOUT_MS) + await prepareWorktree(worktree) const sideOut = path.join(scratchDir, `${side}-out`) await fs.mkdir(sideOut, { recursive: true }) - const exit = await new Promise((resolve, reject) => { + const exit = await new Promise((resolve) => { const child = spawn( process.execPath, [ "--import", "tsx", - "scripts/agent-evals/coding-side.mts", + path.join(root, "scripts/agent-evals/coding-side.mts"), "--reps", String(inputs.reps), "--out", sideOut, "--credential-env", inputs.credentialEnv, + "--evaluated-root", + worktree, ...(inputs.allowUnisolatedGrading ? ["--allow-unisolated-grading"] : []), ], { - cwd: worktree, - env: { ...process.env, CODING_EVALS_SIDE: side }, + cwd: root, + env: buildSideEnvironment(process.env, inputs.credentialEnv, side), stdio: "inherit", + detached: true, }, ) - child.once("error", reject) - child.once("exit", (code) => resolve(code ?? 1)) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGTERM") + } catch { + child.kill("SIGTERM") + } + setTimeout(() => { + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, SIDE_KILL_GRACE_MS).unref() + }, inputs.sideTimeoutMs) + child.once("error", () => { + clearTimeout(timer) + resolve(1) + }) + child.once("exit", (code) => { + clearTimeout(timer) + resolve(timedOut ? 1 : (code ?? 1)) + }) }) if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) - return JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) as SideResultFile + const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) + // The side output must describe exactly the approved commit, or the pair + // is invalid. + return validateSideResultFile(parsed, commit) } const inputs = readArgs() +if (process.env[inputs.credentialEnv] === undefined || process.env[inputs.credentialEnv]?.length === 0) { + throw new Error(`provider credential environment variable ${inputs.credentialEnv} is unset or empty`) +} +const policy = await loadTrustPolicy(inputs.trustFile) +// Trust gate. Both commits resolve to full SHAs and must be approved before +// any worktree is created or any code runs. +const baselineCommit = await resolveCommit(inputs.baseline, root) +const candidateCommit = await resolveCommit(inputs.candidate, root) +assertCommitsApproved(policy, [baselineCommit, candidateCommit]) + const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-compare-")) await fs.mkdir(inputs.out, { recursive: true }) try { const report = buildComparisonReport({ - baseline: toSideResult(await runSide("baseline", inputs.baseline, inputs, scratchDir)), - candidate: toSideResult(await runSide("candidate", inputs.candidate, inputs, scratchDir)), + baseline: toSideResult(await runSide("baseline", baselineCommit, inputs, scratchDir)), + candidate: toSideResult(await runSide("candidate", candidateCommit, inputs, scratchDir)), repetitions: inputs.reps, }) - await fs.writeFile(path.join(inputs.out, "comparison-report.json"), JSON.stringify(report, null, 2)) - await fs.writeFile(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) + // The reports are the comparison's publication point. They appear + // atomically and only after both sides published complete results; any + // failure above leaves no report files behind. + await writeJsonAtomic(path.join(inputs.out, "comparison-report.json"), report) + await writeFileAtomic(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) console.log( report.compatibility.compatible ? `comparison complete: compatible pair, report in ${inputs.out}` diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 56357f419d..0a8b9cb04e 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -52,6 +52,28 @@ export interface SideResultFile { runs: CodingRunRecord[] } +// Validates one side result file before the report builder consumes it. The +// file is data, so its shape is checked, and it must record exactly the +// approved commit of its side. Anything else aborts the comparison instead of +// producing a report from partial output. +export function validateSideResultFile(parsed: unknown, expectedCommit: string): SideResultFile { + if (typeof parsed !== "object" || parsed === null) throw new Error("side result is not a JSON object") + const record = parsed as Record + const pins = record["pins"] + if (typeof pins !== "object" || pins === null) throw new Error("side result has no pins") + const pinsRecord = pins as Record + for (const field of ["side", "commit", "suiteId", "suiteDigest", "graderVersion"] as const) { + if (typeof pinsRecord[field] !== "string" || (pinsRecord[field] as string).length === 0) { + throw new Error(`side result pins lack ${field}`) + } + } + if (pinsRecord["commit"] !== expectedCommit) { + throw new Error(`side result records commit ${String(pinsRecord["commit"])} instead of ${expectedCommit}`) + } + if (!Array.isArray(record["runs"])) throw new Error("side result has no runs array") + return parsed as SideResultFile +} + // Reduces a side result file to the shape the report builder consumes. export const toSideResult = (file: SideResultFile): SideResult => ({ pins: file.pins, diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index d91d63b474..48aa0f7222 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -69,12 +69,28 @@ describe("coding fixture suite", () => { } }) + it("keeps the settings projection equal to the built configuration", () => { + // The suite and settings digests hash the projection, so the built + // configuration must never drift from it. The credential is removed + // and the serialized extra body is parsed before the comparison. + for (const bundle of CODING_FIXTURE_BUNDLES) { + const built: Record = { + ...(bundle.fixture.buildConfiguration(CREDENTIAL) as Record), + } + assert.equal(typeof built["openAiApiKey"], "string") + delete built["openAiApiKey"] + assert.equal(typeof built["openAiExtraBody"], "string") + built["openAiExtraBody"] = JSON.parse(built["openAiExtraBody"] as string) + assert.deepEqual(built, bundle.fixture.settingsProjection) + } + }) + it("produces a stable suite digest that tracks the grader version", () => { assert.equal(codingSuiteDigest(), codingSuiteDigest()) assert.equal(codingBundle("clamp-range").fixture.identity.name, "clamp-range") assert.throws(() => codingBundle("unknown-fixture")) assert.ok(codingSuiteDigest().length === 64) assert.equal(CODING_SUITE_ID, "coding-basic-v1") - assert.equal(GRADER_VERSION, "coding-grader-v1") + assert.equal(GRADER_VERSION, "coding-grader-v2") }) }) diff --git a/scripts/agent-evals/coding-grader.test.ts b/scripts/agent-evals/coding-grader.test.ts index dd60644366..ec2e946c13 100644 --- a/scripts/agent-evals/coding-grader.test.ts +++ b/scripts/agent-evals/coding-grader.test.ts @@ -28,6 +28,7 @@ const driverCompleted = (overrides: Partial = {}): DriverComple testFiles: [], passedCount: 3, failedCount: 0, + cancelledCount: 0, failures: [], durationMs: 1, ...overrides, @@ -125,6 +126,39 @@ describe("runGrader", () => { await fs.rm(scratch, { recursive: true, force: true }) } }) + + it("separates a grader runtime crash from a grading copy failure", async () => { + const scratch = await tmpdir() + try { + const runtime: GradingRuntime = { + isolation: "container", + run: async () => { + throw new Error("runtime exploded") + }, + } + const verdict = await runGrader({ + finalFiles: [{ path: "marker.txt", content: "x" }], + hiddenTests: [], + resources: { timeoutMs: 1_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + runtime, + scratchDir: scratch, + }) + // A runtime exception must not be reported as a copy failure, and + // a failed cleanup must not replace the verdict. + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderRuntimeFailed", + }) + const gradingCopyGone = await fs + .access(path.join(scratch, "grading-copy")) + .then(() => false) + .catch(() => true) + assert.equal(gradingCopyGone, true) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) }) describe("local subprocess runtime failure paths", () => { diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts index 3950bc9168..0f9800ee32 100644 --- a/scripts/agent-evals/coding-grader.ts +++ b/scripts/agent-evals/coding-grader.ts @@ -4,11 +4,12 @@ import fs from "node:fs/promises" import path from "node:path" import type { AgentFixtureWorkspaceFile } from "./contract.ts" import { buildGradingCopy } from "./coding-workspace.ts" -import { readDriverResult, verdictFromDriver } from "./container-runtime.ts" +import { loadDriverResult, verdictFromDriver } from "./container-runtime.ts" +import type { Measured } from "./trace.ts" // The grader version is a pinned comparison input. Any change to the driver, // the image, or the verdict rules must bump it. -export const GRADER_VERSION = "coding-grader-v1" +export const GRADER_VERSION = "coding-grader-v2" export interface GraderResources { timeoutMs: number @@ -25,6 +26,9 @@ export interface DriverCompleted { testFiles: string[] passedCount: number failedCount: number + // Tests the runner cancelled. A cancelled test never ran, so a passing + // verdict requires zero cancellations. + cancelledCount: number failures: { name: string; file: string }[] durationMs: number } @@ -53,6 +57,9 @@ export type GraderVerdict = export interface GradingRuntime { readonly isolation: GradingIsolation run(job: GraderJob): Promise + // Resolves the immutable grader image identity for the comparison pins. + // Optional: runtimes without an image report unavailability. + imageDigest?(): Promise> } export const driverFilePath = (): string => fileURLToPath(new URL("./grader-driver.mjs", import.meta.url)) @@ -71,18 +78,26 @@ export async function runGrader(inputs: { scratchDir: string }): Promise { const gradingDir = path.join(inputs.scratchDir, "grading-copy") + // Cleanup is best effort. A failed removal must never replace the verdict. + const removeCopy = (): Promise => fs.rm(gradingDir, { recursive: true, force: true }).catch(() => {}) + let verdict: GraderVerdict try { await buildGradingCopy(inputs.finalFiles, inputs.hiddenTests, gradingDir) - return await inputs.runtime.run({ + } catch { + await removeCopy() + return { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "gradingCopyFailed" } + } + try { + verdict = await inputs.runtime.run({ gradingDir, driverPath: driverFilePath(), resources: inputs.resources, }) } catch { - return { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "gradingCopyFailed" } - } finally { - await fs.rm(gradingDir, { recursive: true, force: true }) + verdict = { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "graderRuntimeFailed" } } + await removeCopy() + return verdict } const INPROCESS_ENV_ALLOWLIST = ["PATH", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"] as const @@ -120,10 +135,12 @@ export function createLocalSubprocessGradingRuntime(nodeExecutable = process.exe return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderTimeout" } if (exit === "error" || exit !== 0) return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderCrashed" } - const result = await readDriverResult(resultPath) - if (!result) + const loaded = await loadDriverResult(resultPath) + if (loaded === "missing") return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderResultMissing" } - return verdictFromDriver(result, "local-subprocess") + if (loaded === "invalid") + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderResultInvalid" } + return verdictFromDriver(loaded, "local-subprocess") } finally { await fs.rm(outDir, { recursive: true, force: true }) } diff --git a/scripts/agent-evals/coding-metrics.test.ts b/scripts/agent-evals/coding-metrics.test.ts index 9b20ff52dd..f28d56ce07 100644 --- a/scripts/agent-evals/coding-metrics.test.ts +++ b/scripts/agent-evals/coding-metrics.test.ts @@ -168,6 +168,46 @@ describe("run metrics", () => { assert.equal(metrics.cost.accountingComplete, false) }) + it("marks accounting incomplete when trace completeness is partial even with a configured price table", () => { + // Usage is fully reported and the table prices the model, but some + // requests lack usage, so the trace accounting is partial. The cost + // estimate exists; the accounting is still incomplete. + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + requestCount: 2, + requestsMissingUsage: 1, + usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage }, + completeness: { status: "partial", reasons: ["usageMissingForSomeRequests"] }, + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, true) + assert.equal(metrics.accounting.status, "partial") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("reports a missing model identifier without an empty-string fallback", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage } }), + }), + outcome: correctOutcome, + modelIdentifier: { available: false, reason: "modelIdMissingInProjection" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "modelIdentifierMissing") + assert.equal(metrics.cost.accountingComplete, false) + }) + it("summarizes tool calls and failures from the tool snapshot", () => { const metrics = buildRunMetrics({ fixtureName: "clamp-range", @@ -302,4 +342,22 @@ describe("cost per successful task", () => { assert.equal(result.excludedRuns, 1) assert.equal(result.policy, "excludeIncompleteFromAggregates-v1") }) + + it("is unavailable when a failed attempt lacks complete cost accounting", () => { + // One success, one known-cost failure, one unknown-cost failure. The + // unknown attempt cost makes the total spend per success unknowable. + const result = costPerSuccessfulTask([ + metricsRun({ repIndex: 0, cost: completeCost(0.5) }), + metricsRun({ + repIndex: 1, + outcome: { outcome: "incorrect", reason: "hiddenTestsFailed" }, + cost: completeCost(0.3), + }), + metricsRun({ repIndex: 2, outcome: { outcome: "incorrect", reason: "hiddenTestsFailed" } }), + ]) + assert.equal(result.available, false) + assert.equal(result.reason, "failedAttemptCostIncomplete") + assert.equal(result.correctRuns, 1) + assert.equal(result.excludedRuns, 1) + }) }) diff --git a/scripts/agent-evals/coding-metrics.ts b/scripts/agent-evals/coding-metrics.ts index 3b8898075e..574cbbb8eb 100644 --- a/scripts/agent-evals/coding-metrics.ts +++ b/scripts/agent-evals/coding-metrics.ts @@ -77,16 +77,21 @@ export function buildRunMetrics(input: { const trace = input.trace const accounting = trace?.accounting const last = accounting?.usage.lastSnapshot - const estimated: EstimatedCost = last - ? estimateRunCost( - input.modelIdentifier.available ? input.modelIdentifier.value : "", - { available: true, value: last.tokensIn }, - { available: true, value: last.tokensOut }, - last.cacheReads, - last.cacheWrites, - input.priceTable, - ) - : { available: false, reason: last ? "modelIdentifierMissing" : "noUsageEventsObserved" } + // Usage completeness stays separate from price availability. The reasons + // name the first missing input: no usage snapshot, then no model + // identifier, then the price table decision. + const estimated: EstimatedCost = !last + ? { available: false, reason: "noUsageEventsObserved" } + : !input.modelIdentifier.available + ? { available: false, reason: "modelIdentifierMissing" } + : estimateRunCost( + input.modelIdentifier.value, + { available: true, value: last.tokensIn }, + { available: true, value: last.tokensOut }, + last.cacheReads, + last.cacheWrites, + input.priceTable, + ) const snapshot = accounting?.toolSnapshot.snapshot const byTool: Record = {} let totalAttempts = 0 @@ -101,7 +106,10 @@ export function buildRunMetrics(input: { providerReported: last ? { available: true, value: last.cost } : { available: false, reason: "noUsageEventsObserved" }, - accountingComplete: estimated.available, + // Complete requires both: the trace accounted for every request, and + // the price table could price the usage. Partial trace accounting with + // a configured price stays incomplete. + accountingComplete: estimated.available && accounting?.completeness.status === "complete", } return { fixtureName: input.fixtureName, @@ -167,10 +175,11 @@ export interface CostPerSuccessfulTask { usd?: number } -// Total estimated cost of all attempts with complete cost accounting, divided -// by the number of correct runs. Runs with incomplete cost accounting are -// flagged and excluded from aggregates. If no run is correct, or if any -// correct run lacks complete cost accounting, the value is unavailable. +// Total estimated cost of all attempts, divided by the number of correct +// runs. The value is unavailable when any attempt lacks complete cost +// accounting, including failed attempts: an unknown attempt cost makes the +// total spend per success unknowable, not merely smaller. Usage completeness +// and price availability both feed accountingComplete. export function costPerSuccessfulTask(runs: readonly CodingRunMetrics[]): CostPerSuccessfulTask { const included = runs.filter((run) => run.cost.accountingComplete) const excluded = runs.length - included.length @@ -182,10 +191,10 @@ export function costPerSuccessfulTask(runs: readonly CodingRunMetrics[]): CostPe correctRuns, } if (correctRuns === 0) return { ...base, available: false, reason: "noCorrectRuns" } - const correctIncomplete = runs.filter( - (run) => run.outcome.outcome === "correct" && !run.cost.accountingComplete, - ).length - if (correctIncomplete > 0) return { ...base, available: false, reason: "correctRunCostIncomplete" } + const correctIncomplete = runs.some((run) => run.outcome.outcome === "correct" && !run.cost.accountingComplete) + if (correctIncomplete) return { ...base, available: false, reason: "correctRunCostIncomplete" } + const failedIncomplete = runs.some((run) => run.outcome.outcome !== "correct" && !run.cost.accountingComplete) + if (failedIncomplete) return { ...base, available: false, reason: "failedAttemptCostIncomplete" } const total = included.reduce((sum, run) => { if (run.cost.estimated.available) return sum + run.cost.estimated.usd return sum diff --git a/scripts/agent-evals/coding-pins.test.ts b/scripts/agent-evals/coding-pins.test.ts index 10a8bd2179..88e9ed6ddf 100644 --- a/scripts/agent-evals/coding-pins.test.ts +++ b/scripts/agent-evals/coding-pins.test.ts @@ -9,7 +9,8 @@ const pins = (side: "baseline" | "candidate", overrides: Partial = {}) suiteDigest: "d1".repeat(32), settingsDigest: "d2".repeat(32), modelIdentifier: "qwen3.8-27b", - graderVersion: "coding-grader-v1", + graderVersion: "coding-grader-v2", + graderImageDigest: `node:22-bookworm@sha256:${"e".repeat(64)}`, runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, ...overrides, }) @@ -27,7 +28,8 @@ describe("pinned comparison inputs", () => { { suiteDigest: "zz".repeat(32) }, { settingsDigest: "zz".repeat(32) }, { modelIdentifier: "other-model" }, - { graderVersion: "coding-grader-v2" }, + { graderVersion: "coding-grader-v9" }, + { graderImageDigest: "unavailable" }, { runnerEnvironment: { os: "darwin", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" } }, { runnerEnvironment: { os: "linux", nodeVersion: "v24.0.0", vscodeTarget: "1.104.0" } }, { runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.100.0" } }, diff --git a/scripts/agent-evals/coding-pins.ts b/scripts/agent-evals/coding-pins.ts index 3433d11bfd..7449e0dff7 100644 --- a/scripts/agent-evals/coding-pins.ts +++ b/scripts/agent-evals/coding-pins.ts @@ -6,6 +6,10 @@ export interface SidePins { settingsDigest: string modelIdentifier: string graderVersion: string + // Immutable grader image identity resolved from the local engine, for + // example a repo digest. The value "unavailable" records that no identity + // was resolvable; it is never a fabricated digest. + graderImageDigest: string runnerEnvironment: { os: string nodeVersion: string @@ -21,7 +25,14 @@ export interface PinComparison { // Pinned comparison inputs. The two commit SHAs differ by design and are // recorded, never compared. Every other pinned value must match on both // sides; any difference marks the pair incompatible. -const COMPARED_FIELDS = ["suiteId", "suiteDigest", "settingsDigest", "modelIdentifier", "graderVersion"] as const +const COMPARED_FIELDS = [ + "suiteId", + "suiteDigest", + "settingsDigest", + "modelIdentifier", + "graderVersion", + "graderImageDigest", +] as const export function comparePins(baseline: SidePins, candidate: SidePins): PinComparison { const differences: string[] = [] diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index a4034affa7..0c89c339ac 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -66,7 +66,7 @@ describe("coding graders against reference solutions", () => { } it("pins one grader version across the suite", () => { - assert.equal(GRADER_VERSION, "coding-grader-v1") + assert.equal(GRADER_VERSION, "coding-grader-v2") assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) }) }) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index e92aa27aa7..8cf8667a2c 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { buildComparisonReport, renderMarkdownReport, type SideResult } from "./coding-report.ts" -import { runComparison, type SideExecutor } from "./coding-compare.ts" +import { runComparison, validateSideResultFile, type SideExecutor } from "./coding-compare.ts" import type { CodingRunMetrics } from "./coding-metrics.ts" import type { SidePins } from "./coding-pins.ts" @@ -12,7 +12,8 @@ const pins = (side: "baseline" | "candidate", commit: string, overrides: Partial suiteDigest: "d1".repeat(32), settingsDigest: "d2".repeat(32), modelIdentifier: "qwen3.8-27b", - graderVersion: "coding-grader-v1", + graderVersion: "coding-grader-v2", + graderImageDigest: `node:22-bookworm@sha256:${"e".repeat(64)}`, runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, ...overrides, }) @@ -96,7 +97,7 @@ describe("comparison report", () => { assert.deepEqual(report.compatibility.differences, ["modelIdentifier"]) }) - it("keeps grading failures out of the incorrect count and excludes incomplete cost runs", () => { + it("keeps grading failures out of the incorrect count and is unavailable with unknown attempt costs", () => { const baseline = sideResult("baseline", "a".repeat(40), [ metrics("clamp-range", 0, "correct", 0.2), metrics("clamp-range", 1, "grading-failure", undefined), @@ -106,8 +107,10 @@ describe("comparison report", () => { const summary = report.costSummary.baseline assert.equal(summary.correctRuns, 1) assert.equal(summary.excludedRuns, 2) - assert.equal(summary.available, true) - assert.equal(summary.usd, 0.2) + // The failed attempts have unknown costs, so the total spend per + // success is unknowable instead of merely smaller. + assert.equal(summary.available, false) + assert.equal(summary.reason, "failedAttemptCostIncomplete") assert.equal(report.partialDataPolicy, "excludeIncompleteFromAggregates-v1") }) @@ -123,7 +126,16 @@ describe("comparison report", () => { assert.match(markdown, /cannot show small improvements/) assert.match(markdown, /## Results per fixture/) assert.match(markdown, /## Cost per successful task/) - assert.match(markdown, /unavailable|over 1 correct run/) + // Each deterministic cost line is asserted on its own. An alternation + // would pass when only one line matched. + assert.ok( + markdown.includes("- Baseline: $0.2000 over 1 correct run(s), 0 excluded"), + `baseline cost line missing:\n${markdown}`, + ) + assert.ok( + markdown.includes("- Candidate: unavailable (noCorrectRuns; 0 excluded)"), + `candidate cost line missing:\n${markdown}`, + ) assert.match(markdown, /## Per-run appendix/) assert.match(markdown, /baseline \| clamp-range \| 0 \| correct /) }) @@ -179,3 +191,20 @@ describe("comparison orchestration", () => { assert.equal(report.repetitions, 3) }) }) + +describe("side result validation", () => { + it("accepts a result that records exactly the approved commit", () => { + const file = { pins: pins("baseline", "a".repeat(40)), runs: [] } + const validated = validateSideResultFile(JSON.parse(JSON.stringify(file)), "a".repeat(40)) + assert.equal(validated.pins.side, "baseline") + }) + + it("rejects a result that records a different commit, or has no shape", () => { + assert.throws(() => + validateSideResultFile({ pins: pins("baseline", "b".repeat(40)), runs: [] }, "a".repeat(40)), + ) + assert.throws(() => validateSideResultFile({ runs: [] }, "a".repeat(40))) + assert.throws(() => validateSideResultFile(null, "a".repeat(40))) + assert.throws(() => validateSideResultFile({ pins: pins("baseline", "a".repeat(40)) }, "a".repeat(40))) + }) +}) diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts index dce16722b6..d26c627323 100644 --- a/scripts/agent-evals/coding-run.test.ts +++ b/scripts/agent-evals/coding-run.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { describe, it } from "node:test" +import { after, describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" import { runCodingFixtureOnce, withWorkspaceCapture } from "./coding-run.ts" import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" @@ -16,6 +16,11 @@ import { codingBundle } from "./coding-suite.ts" interface SyntheticOptions { finalFiles: AgentFixtureWorkspaceFile[] complete: boolean + // Emits the model's final non-partial completion_result message instead + // of the accepted TaskCompleted event. + completeByMessageOnly?: boolean + // Rejects the final-workspace read, as a broken capture does. + failWorkspaceRead?: boolean } const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; traces: AgentTrace[] } => { @@ -43,6 +48,28 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t eventName: RooCodeEventName.TaskStartResponse, payload: [{ requestId, success: true, taskId: "task-1" }], }) + if (options.completeByMessageOnly) { + // The model reports completion, but the accepted TaskCompleted + // event never fires. This is the headless completion path. + // The synthetic message satisfies the Message payload shape; + // the cast names the variant the schema accepts. + emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "task-1", + action: "created", + message: { + type: "say", + say: "completion_result", + ts: 1, + text: "Task finished.", + }, + }, + ], + } as unknown as TaskEvent) + return + } if (options.complete) emit({ eventName: RooCodeEventName.TaskCompleted, @@ -68,10 +95,10 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t pause: async () => { fakeNow += 100 }, - readFinalWorkspace: async (): Promise => ({ - files: options.finalFiles, - truncated: false, - }), + readFinalWorkspace: async (): Promise => { + if (options.failWorkspaceRead) throw new Error("unreadable workspace") + return { files: options.finalFiles, truncated: false } + }, } return { runtime, traces } } @@ -82,11 +109,15 @@ const finalFilesWith = (solution: AgentFixtureWorkspaceFile): AgentFixtureWorksp } describe("coding run orchestration", () => { + // Every scratch directory is removed in the after hook, which completes + // before the test process exits. No async work rides on an exit handler. + const scratchDirs: string[] = [] + after(async () => { + for (const dir of scratchDirs) await fs.rm(dir, { recursive: true, force: true }) + }) const scratch = async (): Promise => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-run-test-")) - process.once("exit", () => { - void fs.rm(dir, { recursive: true, force: true }) - }) + scratchDirs.push(dir) return dir } @@ -160,4 +191,68 @@ describe("coding run orchestration", () => { assert.equal(snapshot.truncated, false) assert.ok(snapshot.files.length > 0) }) + + it("stops on the final completion message without an accepted TaskCompleted and grades independently", async () => { + const bundle = codingBundle("clamp-range") + const { runtime } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: false, + completeByMessageOnly: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 3, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // The run stopped on the headless policy instead of waiting for the + // task-run window, never claimed an accepted completion, and the + // hidden tests still decide the outcome. + assert.equal(record.executionCode, "STOPPED") + assert.equal(record.outcome.outcome, "correct") + assert.equal(record.grading?.kind, "passed") + assert.equal(record.trace?.execution.stopReason, "adapterStopAfterCompletionMessage") + assert.equal(record.trace?.execution.completionMessageSeen, true) + }) + + it("reports a rejected final-workspace read as finalWorkspaceUnavailable without grading", async () => { + const { runtime } = syntheticRuntime({ finalFiles: [], complete: true, failWorkspaceRead: true }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 4, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // An infrastructure fault must never become an incorrect verdict from + // an empty workspace. + assert.equal(record.outcome.outcome, "infrastructure-failure") + assert.equal(record.outcome.reason, "finalWorkspaceUnavailable") + assert.equal(record.grading, undefined) + }) + + it("never rereads the workspace once a snapshot is captured", async () => { + const bundle = codingBundle("clamp-range") + const inner = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: false, + }) + let reads = 0 + const original = inner.runtime.readFinalWorkspace!.bind(inner.runtime) + inner.runtime.readFinalWorkspace = async () => { + reads++ + // A second cleanup finds the directory already removed. + return reads === 1 ? await original() : Promise.reject(new Error("removed")) + } + const captured = withWorkspaceCapture(inner.runtime) + await captured.runtime.cleanup() + const first = captured.getSnapshot() + assert.ok(first && first.files.length > 0) + // Repeated cleanup must not overwrite the captured snapshot. + await captured.runtime.cleanup() + assert.equal(captured.getSnapshot(), first) + }) }) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts index 3d7599268e..b09cc0e973 100644 --- a/scripts/agent-evals/coding-run.ts +++ b/scripts/agent-evals/coding-run.ts @@ -1,4 +1,10 @@ -import { runAgentFixture, type AgentRuntime, type FinalWorkspaceSnapshot } from "./runner.ts" +import { + runAgentFixture, + isFinalCompletionMessage, + type AgentRuntime, + type AgentRunObservers, + type FinalWorkspaceSnapshot, +} from "./runner.ts" import { getFixture } from "./registry.ts" import { ensureCodingSuiteRegistered, codingBundle } from "./coding-suite.ts" import { runGrader, type GraderVerdict, type GradingRuntime } from "./coding-grader.ts" @@ -47,9 +53,28 @@ const modelIdentifierFromProjection = (projection: Record): Mea "modelIdMissingInProjection", ) +// Headless stop policy. A headless extension host may never fire the accepted +// TaskCompleted event, so the run must not wait for the whole task-run window. +// The policy stops once the model's final non-partial completion_result +// message arrives for the accepted task. The substrate records the stop as +// adapterStopAfterCompletionMessage; it never claims an accepted completion. +// Grading below stays independent of the completion event. +export function stopOnFinalCompletionMessage(): AgentRunObservers { + let sawFinalCompletionMessage = false + return { + onAcceptedEvent: (event) => { + if (isFinalCompletionMessage(event)) sawFinalCompletionMessage = true + }, + shouldStop: () => sawFinalCompletionMessage, + } +} + // Wraps a runtime so the final workspace is captured inside cleanup, before // the launcher removes its scratch directory. The captured file data feeds -// only the fresh grading copy. +// only the fresh grading copy. A rejected read leaves the snapshot undefined, +// so a broken capture becomes an infrastructure failure, never an empty +// workspace scored against the agent. A captured snapshot is never reread, so +// repeated cleanup calls cannot overwrite it. export function withWorkspaceCapture(inner: AgentRuntime): { runtime: AgentRuntime getSnapshot: () => FinalWorkspaceSnapshot | undefined @@ -61,10 +86,12 @@ export function withWorkspaceCapture(inner: AgentRuntime): { launch: (files) => inner.launch(files), connect: () => inner.connect(), cleanup: async () => { - try { - snapshot = await inner.readFinalWorkspace?.() - } catch { - snapshot = { files: [], truncated: true } + if (snapshot === undefined) { + try { + snapshot = await inner.readFinalWorkspace?.() + } catch { + snapshot = undefined + } } await inner.cleanup() }, @@ -98,6 +125,7 @@ export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { +const readArgs = (): { + reps: number + out: string + credentialEnv: string + allowUnisolatedGrading: boolean + evaluatedRoot: string +} => { const args = process.argv.slice(2) const read = (name: string): string | undefined => { const index = args.indexOf(`--${name}`) @@ -34,10 +42,25 @@ const readArgs = (): { reps: number; out: string; credentialEnv: string; allowUn out: path.resolve(out), credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + // Repository root of the extension under test. The trusted harness + // code drives the run; only the extension host loads this tree. + evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), } } -const readVscodeTarget = async (): Promise => { +// Side identity. The value must be one of the comparison labels; the default +// applies only when the variable is unset. Validation happens before any run +// spends provider budget or writes results. +const readSideLabel = (): SidePins["side"] => { + const raw = process.env["CODING_EVALS_SIDE"] + if (raw === undefined) return "baseline" + if (raw !== "baseline" && raw !== "candidate") { + throw new Error(`CODING_EVALS_SIDE must be "baseline" or "candidate", got "${raw}"`) + } + return raw +} + +const readVscodeTarget = async (root: string): Promise => { try { const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { devDependencies?: Record @@ -48,7 +71,7 @@ const readVscodeTarget = async (): Promise => { } } -const readCommit = async (): Promise => { +const readCommit = async (root: string): Promise => { const { execFile } = await import("node:child_process") const { promisify } = await import("node:util") try { @@ -59,29 +82,20 @@ const readCommit = async (): Promise => { } } -const { reps, out, credentialEnv, allowUnisolatedGrading } = readArgs() +const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") -await fs.mkdir(out, { recursive: true }) +const sideLabel = readSideLabel() +// Fail before any directory, run, or result file exists. An empty credential +// would otherwise produce a plausible file of infrastructure failures. +const credential = process.env[credentialEnv] +if (credential === undefined || credential.length === 0) { + throw new Error(`provider credential environment variable ${credentialEnv} is unset or empty`) +} ensureCodingSuiteRegistered() -const credential = process.env[credentialEnv] ?? "" const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} -const pins: SidePins = { - side: "baseline", - commit: await readCommit(), - suiteId: CODING_SUITE_ID, - suiteDigest: codingSuiteDigest(), - settingsDigest: digestValue(projection), - modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", - graderVersion: GRADER_VERSION, - runnerEnvironment: { - os: process.platform, - nodeVersion: process.version, - vscodeTarget: await readVscodeTarget(), - }, -} const runs: CodingRunRecord[] = [] for (const bundle of CODING_FIXTURE_BUNDLES) { @@ -94,7 +108,11 @@ for (const bundle of CODING_FIXTURE_BUNDLES) { fixtureName, repIndex, credential, - runtime: createHostRuntime({ root, artifactsDir }), + runtime: createHostRuntime({ + root: evaluatedRoot, + artifactsDir, + vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), + }), gradingRuntime, scratchDir: artifactsDir, }) @@ -119,17 +137,38 @@ for (const bundle of CODING_FIXTURE_BUNDLES) { } } -// Side identity must reflect the actual side label in a comparison context. -const sideLabel = (process.env["CODING_EVALS_SIDE"] as SidePins["side"] | undefined) ?? "baseline" -const result: SideResultFile = { pins: { ...pins, side: sideLabel }, runs } +// Pins are recorded after the runs so they describe what actually executed: +// the evaluated commit and, when the container runtime resolved one, the +// immutable grader image identity. +const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined +const pins: SidePins = { + side: sideLabel, + commit: await readCommit(evaluatedRoot), + suiteId: CODING_SUITE_ID, + suiteDigest: codingSuiteDigest(), + settingsDigest: digestValue(projection), + modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", + graderVersion: GRADER_VERSION, + graderImageDigest: digest?.available === true ? digest.value : "unavailable", + runnerEnvironment: { + os: process.platform, + nodeVersion: process.version, + vscodeTarget: await readVscodeTarget(evaluatedRoot), + }, +} +const result: SideResultFile = { pins, runs } -// Keep the trace of each run with its artifacts for inspection. +// Per-run artifacts are published atomically as each record exists. for (const record of runs) { if (!record.trace) continue const artifactDir = path.join(out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) - await fs.writeFile(path.join(artifactDir, "run-record.json"), JSON.stringify(record, null, 2)) + await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) } -await fs.writeFile(path.join(out, "side-results.json"), JSON.stringify(result, null, 2)) +// Publication point. side-results.json appears atomically only after every +// repetition finished, so a reader that sees the file sees a complete side. +// A missing file means an interrupted side; consumers must treat it as a +// failed side and never pair partial output. +await writeJsonAtomic(path.join(out, "side-results.json"), result) const correct = runs.filter((record) => record.outcome.outcome === "correct").length console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts new file mode 100644 index 0000000000..db7a96b5ae --- /dev/null +++ b/scripts/agent-evals/coding-trust.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { + assertCommitsApproved, + buildSideEnvironment, + loadTrustPolicy, + UntrustedCommitError, + type TrustPolicy, +} from "./coding-trust.ts" + +const sha = (c: string): string => c.repeat(40) + +const writePolicy = async (dir: string, content: unknown): Promise => { + const file = path.join(dir, "trust-policy.json") + await fs.writeFile(file, JSON.stringify(content)) + return file +} + +describe("trust policy", () => { + it("loads a policy of full commit SHAs", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const file = await writePolicy(dir, { version: 1, approvedCommits: [sha("a"), sha("B")] }) + const policy = await loadTrustPolicy(file) + assert.deepEqual(policy, { version: 1, approvedCommits: [sha("a"), sha("b")] }) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects missing files, wrong versions, empty lists, and non-SHA entries", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + await assert.rejects(loadTrustPolicy(path.join(dir, "absent.json")), /missing or unreadable/) + const bad: unknown[] = [ + { version: 2, approvedCommits: [sha("a")] }, + { version: 1, approvedCommits: [] }, + { version: 1, approvedCommits: ["main"] }, + { version: 1, approvedCommits: [sha("a").slice(0, 8)] }, + { version: 1, approvedCommits: [`${sha("a")}..${sha("b")}`] }, + { version: 1, approvedCommits: ["not-a-sha"] }, + ] + for (const content of bad) { + const file = await writePolicy(dir, content) + await assert.rejects(loadTrustPolicy(file)) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("approves listed commits and rejects unapproved ones", () => { + const policy: TrustPolicy = { version: 1, approvedCommits: [sha("a")] } + assertCommitsApproved(policy, [sha("a")]) + // The gate is case insensitive on the approved side. + assertCommitsApproved(policy, [sha("A").toUpperCase()]) + assert.throws(() => assertCommitsApproved(policy, [sha("b"), sha("c")]), UntrustedCommitError) + try { + assertCommitsApproved(policy, [sha("b")]) + } catch (error) { + assert.ok(error instanceof UntrustedCommitError) + assert.match(error.message, /not approved by the trust policy/) + } + }) +}) + +describe("side environment allowlist", () => { + it("passes only the allowlisted variables, the credential, and the side label", () => { + const child = buildSideEnvironment( + { + PATH: "/usr/bin", + HOME: "/home/dev", + SOLHEIM_API_KEY: "sk-test", + DISPLAY: ":0", + AWS_SECRET_ACCESS_KEY: "leak-me", + OTHER_SECRET: "leak-me-too", + }, + "SOLHEIM_API_KEY", + "candidate", + ) + assert.deepEqual(child, { + PATH: "/usr/bin", + HOME: "/home/dev", + DISPLAY: ":0", + SOLHEIM_API_KEY: "sk-test", + CODING_EVALS_SIDE: "candidate", + }) + }) + + it("never invents a missing credential variable", () => { + const child = buildSideEnvironment({ PATH: "/usr/bin" }, "SOLHEIM_API_KEY", "baseline") + assert.equal(child["SOLHEIM_API_KEY"], undefined) + assert.equal(child["CODING_EVALS_SIDE"], "baseline") + }) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts new file mode 100644 index 0000000000..d1280943a3 --- /dev/null +++ b/scripts/agent-evals/coding-trust.ts @@ -0,0 +1,100 @@ +import fs from "node:fs/promises" + +// Trust policy for paired comparisons. A comparison executes the extension +// code of two commits with the provider credential. Nothing about a ref name +// proves trust: a policy that resolved arbitrary refs and called them trusted +// would approve anything. The operator therefore lists approved full commit +// SHAs in an explicit policy file, and the driver rejects any other resolved +// commit before it creates a worktree or spawns a process. +export interface TrustPolicy { + version: 1 + approvedCommits: string[] +} + +// Git prints lowercase SHAs, but the policy accepts either case and the +// loader normalizes to lowercase. +const SHA_PATTERN = /^[0-9a-fA-F]{40}$/ + +export class UntrustedCommitError extends Error { + constructor(readonly commits: string[]) { + super(`commits not approved by the trust policy: ${commits.join(", ")}`) + this.name = "UntrustedCommitError" + } +} + +// Loads and validates one trust policy file. Every approved commit must be a +// full, lowercase-or-uppercase 40-hex SHA; short SHAs, ref names, and ranges +// are rejected. +export async function loadTrustPolicy(filePath: string): Promise { + let parsed: unknown + try { + parsed = JSON.parse(await fs.readFile(filePath, "utf8")) + } catch { + throw new Error(`trust policy file is missing or unreadable: ${filePath}`) + } + if (typeof parsed !== "object" || parsed === null) throw new Error("trust policy must be a JSON object") + const record = parsed as Record + if (record["version"] !== 1) throw new Error("trust policy version must be 1") + const commits = record["approvedCommits"] + if (!Array.isArray(commits) || commits.length === 0) { + throw new Error("trust policy must list at least one approved commit SHA") + } + const invalid = commits.filter((entry) => typeof entry !== "string" || !SHA_PATTERN.test(entry)) + if (invalid.length > 0) throw new Error("trust policy accepts only full 40-hex commit SHAs") + return { version: 1, approvedCommits: (commits as string[]).map((sha) => sha.toLowerCase()) } +} + +// Resolves one ref or SHA to a full commit SHA through the local repository. +export const resolveCommit = async (ref: string, root: string): Promise => { + const { execFile } = await import("node:child_process") + const { promisify } = await import("node:util") + const { stdout } = await promisify(execFile)("git", ["rev-parse", `${ref}^{commit}`], { + cwd: root, + timeout: 30_000, + }) + const sha = stdout.trim().toLowerCase() + if (!SHA_PATTERN.test(sha)) throw new Error(`cannot resolve a full commit SHA from ${ref}`) + return sha +} + +// Gate before any execution. Each resolved commit must appear in the policy. +export function assertCommitsApproved(policy: TrustPolicy, commits: readonly string[]): void { + const unapproved = commits.filter((sha) => !policy.approvedCommits.includes(sha.toLowerCase())) + if (unapproved.length > 0) throw new UntrustedCommitError(unapproved) +} + +// Environment allowlist for the side process. The comparison driver may hold +// unrelated secrets in its own environment; the side gets only what it needs +// to spawn git, node, and the extension host, plus the one named credential +// variable. +const SIDE_ENV_ALLOWLIST = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "TMPDIR", + "TEMP", + "TMP", + "DISPLAY", + "XAUTHORITY", + "SYSTEMROOT", + "WINDIR", + "COMSPEC", + "PATHEXT", +] as const + +export function buildSideEnvironment( + env: NodeJS.ProcessEnv, + credentialEnv: string, + side: "baseline" | "candidate", +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of SIDE_ENV_ALLOWLIST) { + const value = env[name] + if (value !== undefined) child[name] = value + } + const credential = env[credentialEnv] + if (credential !== undefined) child[credentialEnv] = credential + child["CODING_EVALS_SIDE"] = side + return child +} diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index afc27c96fa..931b3dadda 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -1,6 +1,15 @@ import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" import { describe, it } from "node:test" -import { buildContainerArgs, createContainerGradingRuntime, GRADER_IMAGE } from "./container-runtime.ts" +import { + buildContainerArgs, + createContainerGradingRuntime, + GRADER_IMAGE, + parseDriverResult, + type ContainerCli, +} from "./container-runtime.ts" import type { GraderJob } from "./coding-grader.ts" const makeJob = (): GraderJob => ({ @@ -9,6 +18,58 @@ const makeJob = (): GraderJob => ({ resources: { timeoutMs: 30_000, memoryMb: 512, cpus: 1, pidsLimit: 128, image: GRADER_IMAGE }, }) +const driverCompletedJson = (): string => + JSON.stringify({ + kind: "completed", + passed: true, + noTestsDiscovered: false, + testFiles: ["test/solution.test.js"], + passedCount: 3, + failedCount: 0, + cancelledCount: 0, + failures: [], + durationMs: 12, + }) + +// Locates the host outDir from the run argv, the way the real driver finds +// its result path through the /out mount. +const outDirFromArgs = (args: readonly string[]): string => { + const mount = args.find((arg) => arg.endsWith(":/out:rw")) + if (!mount) throw new Error("no /out mount in run argv") + return mount.slice(0, -":/out:rw".length) +} + +const recordingCli = ( + overrides: Partial = {}, +): ContainerCli & { calls: string[]; runArgs: string[][] } => { + const calls: string[] = [] + const runArgs: string[][] = [] + return { + calls, + runArgs, + pull: async (image) => { + calls.push(`pull ${image}`) + return "ok" + }, + run: async (args, timeoutMs) => { + runArgs.push([...args]) + calls.push(`run ${timeoutMs}`) + await fs.mkdir(outDirFromArgs(args), { recursive: true }) + await fs.writeFile(path.join(outDirFromArgs(args), "result.json"), driverCompletedJson()) + return 0 + }, + remove: async (name) => { + calls.push(`remove ${name}`) + return "removed" + }, + imageDigest: async () => { + calls.push("inspect") + return "node:22-bookworm@sha256:" + "a".repeat(64) + }, + ...overrides, + } +} + describe("container grading runtime", () => { it("builds a locked-down, network-less invocation", () => { const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out") @@ -47,6 +108,14 @@ describe("container grading runtime", () => { assert.equal(args.includes("--env"), false) }) + it("names the container when one is provided", () => { + const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out", "agent-evals-grader-1") + const nameIndex = args.indexOf("--name") + assert.ok(nameIndex > 0) + assert.equal(args[nameIndex + 1], "agent-evals-grader-1") + assert.equal(args[0], "run") + }) + it("reports a missing container runtime as grading failure", async () => { const runtime = createContainerGradingRuntime({ detectExecutable: async () => undefined }) const verdict = await runtime.run(makeJob()) @@ -59,4 +128,193 @@ describe("container grading runtime", () => { const runtime = createContainerGradingRuntime() assert.equal(runtime.isolation, "container") }) + + it("pulls the image outside the grading timeout and runs with a unique name", async () => { + const cli = recordingCli() + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.equal(verdict.kind, "passed") + // The pull happens first, the image identity is resolved from the + // local engine, and the timed run comes after both. + assert.equal(cli.calls[0], `pull ${GRADER_IMAGE}`) + assert.equal(cli.calls.indexOf("run 30000"), 2) + assert.equal(cli.calls[1], "inspect") + // The run argv names a unique container. + const nameIndex = cli.runArgs[0]!.indexOf("--name") + assert.ok(nameIndex > 0) + assert.match(cli.runArgs[0]![nameIndex + 1] ?? "", /^agent-evals-grader-\d+-\d+-[0-9a-f]{8}$/) + // The immutable image identity is resolvable for the pins. + const digest = await runtime.imageDigest!() + assert.equal(digest.available, true) + assert.match(digest.available ? digest.value : "", /^node:22-bookworm@sha256:[0-9a-f]{64}$/) + }) + + it("removes the container by name on timeout and verifies the removal before deleting output", async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-runtime-test-")) + try { + const cli = recordingCli({ + run: async () => "timeout", + remove: async (name) => { + cli.calls.push(`remove ${name}`) + // The verified removal runs before the output directory + // disappears. + await fs.access(path.join(scratch, "grading-out")).then( + () => {}, + () => { + throw new Error("output removed before container removal") + }, + ) + return "removed" + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run({ ...makeJob(), gradingDir: path.join(scratch, "grading-copy") }) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "container", reason: "graderTimeout" }) + assert.equal(cli.calls.filter((call) => call.startsWith("remove ")).length, 1) + assert.match(cli.calls.find((call) => call.startsWith("remove ")) ?? "", /^remove agent-evals-grader-/) + // The output directory is gone afterwards. + await fs.access(path.join(scratch, "grading-out")).then( + () => { + throw new Error("output directory survived") + }, + () => {}, + ) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a failed timeout removal instead of a leaked container", async () => { + const cli = recordingCli({ + run: async () => "timeout", + remove: async () => "failed", + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderContainerRemovalFailed", + }) + }) + + it("verifies removal after a crashed run", async () => { + const cli = recordingCli({ + run: async () => 3, + remove: async (name) => { + cli.calls.push(`remove ${name}`) + return "absent" + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "container", reason: "graderCrashed" }) + assert.equal(cli.calls.filter((call) => call.startsWith("remove ")).length, 1) + }) + + it("reports a failed image pull before any container runs", async () => { + const cli = recordingCli({ pull: async () => "failed" }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderImagePullFailed", + }) + assert.deepEqual( + cli.calls.filter((call) => call.startsWith("run")), + [], + ) + }) + + it("reports an invalid driver result as graderResultInvalid", async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-runtime-test-")) + try { + const outDir = path.join(scratch, "grading-out") + const cli = recordingCli({ + run: async () => { + await fs.mkdir(outDir, { recursive: true }) + await fs.writeFile(path.join(outDir, "result.json"), '{"kind":"completed","passed":true,"lie":1}') + return 0 + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run({ ...makeJob(), gradingDir: path.join(scratch, "grading-copy") }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderResultInvalid", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) + +describe("untrusted driver result validation", () => { + const valid = JSON.parse(driverCompletedJson()) as Record + const build = (overrides: Partial>): string => JSON.stringify({ ...valid, ...overrides }) + + it("accepts a well-formed completed result", () => { + const parsed = parseDriverResult(driverCompletedJson()) + assert.ok(parsed && parsed.kind === "completed") + assert.equal(parsed.cancelledCount, 0) + }) + + it("accepts a driver-error result", () => { + const parsed = parseDriverResult(JSON.stringify({ kind: "driver-error", message: "testRunnerUnavailable" })) + assert.ok(parsed && parsed.kind === "driver-error") + }) + + it("rejects malformed JSON and wrong shapes", () => { + assert.equal(parseDriverResult("{not json"), undefined) + assert.equal(parseDriverResult("42"), undefined) + assert.equal(parseDriverResult("null"), undefined) + assert.equal(parseDriverResult('{"kind":"other"}'), undefined) + }) + + it("rejects non-boolean flags, bad counts, and bad durations", () => { + assert.equal(parseDriverResult(build({ passed: "yes" })), undefined) + assert.equal(parseDriverResult(build({ noTestsDiscovered: 0 })), undefined) + assert.equal(parseDriverResult(build({ passedCount: -1 })), undefined) + assert.equal(parseDriverResult(build({ failedCount: 1.5 })), undefined) + assert.equal(parseDriverResult(build({ cancelledCount: "0" })), undefined) + assert.equal(parseDriverResult(build({ durationMs: -1 })), undefined) + assert.equal(parseDriverResult(build({ durationMs: Number.POSITIVE_INFINITY })), undefined) + }) + + it("rejects pass/fail inconsistency and cancelled passes", () => { + assert.equal(parseDriverResult(build({ passed: true, failedCount: 2 })), undefined) + assert.equal(parseDriverResult(build({ passed: true, passedCount: 0 })), undefined) + assert.equal(parseDriverResult(build({ passed: true, cancelledCount: 1 })), undefined) + assert.equal(parseDriverResult(build({ passed: false })), undefined) + }) + + it("rejects bad failure entries and failure counts that exceed failedCount", () => { + assert.equal(parseDriverResult(build({ failures: [{ name: "x" }] })), undefined) + assert.equal(parseDriverResult(build({ failures: [{ name: 1, file: "f" }] })), undefined) + assert.equal( + parseDriverResult( + build({ + failedCount: 1, + failures: [ + { name: "a", file: "f" }, + { name: "b", file: "f" }, + ], + }), + ), + undefined, + ) + }) + + it("rejects bad test file lists", () => { + assert.equal(parseDriverResult(build({ testFiles: [] })), undefined) + assert.equal(parseDriverResult(build({ testFiles: [""] })), undefined) + assert.equal(parseDriverResult(build({ testFiles: [42] })), undefined) + assert.equal(parseDriverResult(build({ testFiles: Array.from({ length: 1_001 }, () => "t.js") })), undefined) + }) + + it("rejects oversized payloads", () => { + assert.equal(parseDriverResult(`{"kind":"completed","pad":"${"x".repeat(70_000)}"}`), undefined) + }) }) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 1d5a263326..98da8ea0c6 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -1,10 +1,13 @@ import { spawn } from "node:child_process" +import { randomUUID } from "node:crypto" import fs from "node:fs/promises" import path from "node:path" import type { GraderJob, GraderVerdict, GradingRuntime, DriverResult } from "./coding-grader.ts" +import type { Measured } from "./trace.ts" -// Pinned grader image. The image tag is part of the grader identity and is -// recorded in every pinned comparison through the grader version. +// Pinned grader image tag. The digest resolved from the local engine travels +// in the comparison pins, so the recorded identity is immutable even though +// the tag alone is not. export const GRADER_IMAGE = "node:22-bookworm" export type ContainerExecutable = "docker" | "podman" @@ -13,6 +16,23 @@ export interface ContainerRuntimeOptions { executable?: ContainerExecutable // Test seam: detects the container binary instead of searching PATH. detectExecutable?: () => Promise + // Test seam: replaces the container CLI process calls. + cli?: ContainerCli +} + +// One container CLI. The default implementation spawns docker or podman. Tests +// replace it, so timeout cleanup is testable without a real engine. +export interface ContainerCli { + // Pulls the grader image. Runs outside the grading timeout. + pull(image: string): Promise<"ok" | "failed"> + // Runs one container. Resolves "timeout" when the CLI hit the wall clock. + run(args: readonly string[], timeoutMs: number): Promise + // Removes the named container. "absent" means the container is already + // gone, which counts as verified removal. + remove(name: string): Promise<"removed" | "absent" | "failed"> + // Resolves the immutable local image identity: the repo digest when one + // exists, otherwise the image ID. Never a fabricated value. + imageDigest(image: string): Promise } const defaultDetect = async (): Promise => { @@ -44,15 +64,89 @@ const probeExecutable = (candidate: ContainerExecutable): Promise => } }) +const exec = ( + file: string, + args: readonly string[], + timeoutMs: number, +): Promise<{ code: number | "timeout" | "error"; stdout: string; stderr: string }> => + new Promise((resolve) => { + try { + const child = spawn(file, [...args], { + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutMs, + killSignal: "SIGKILL", + }) + let stdout = "" + let stderr = "" + child.stdout?.on("data", (chunk) => { + stdout += String(chunk) + }) + child.stderr?.on("data", (chunk) => { + stderr += String(chunk) + }) + child.once("error", () => resolve({ code: "error", stdout, stderr })) + child.once("close", (code, signal) => + resolve({ code: signal === "SIGKILL" ? "timeout" : (code ?? "error"), stdout, stderr }), + ) + } catch { + resolve({ code: "error", stdout: "", stderr: "" }) + } + }) + +const IMAGE_PULL_TIMEOUT_MS = 10 * 60_000 +const CONTAINER_REMOVE_TIMEOUT_MS = 30_000 +const IMAGE_INSPECT_TIMEOUT_MS = 30_000 + +const containerGone = (stderr: string): boolean => /no such container/i.test(stderr) + +export function createDefaultContainerCli(executable: ContainerExecutable): ContainerCli { + return { + async pull(image) { + const outcome = await exec(executable, ["pull", "-q", image], IMAGE_PULL_TIMEOUT_MS) + return outcome.code === 0 ? "ok" : "failed" + }, + run(args, timeoutMs) { + return exec(executable, args, timeoutMs).then((outcome) => outcome.code) + }, + async remove(name) { + const outcome = await exec(executable, ["rm", "--force", name], CONTAINER_REMOVE_TIMEOUT_MS) + if (outcome.code === 0) return "removed" + if (outcome.code !== "timeout" && containerGone(outcome.stderr)) return "absent" + return "failed" + }, + async imageDigest(image) { + const repoDigest = await exec( + executable, + ["image", "inspect", "--format", "{{index .RepoDigests 0}}", image], + IMAGE_INSPECT_TIMEOUT_MS, + ) + if (repoDigest.code === 0) { + const digest = repoDigest.stdout.trim() + if (digest.length > 0) return digest + } + // A locally built image has no repo digest. The engine image ID is + // still an immutable local identity. + const imageId = await exec( + executable, + ["image", "inspect", "--format", "{{.Id}}", image], + IMAGE_INSPECT_TIMEOUT_MS, + ) + return imageId.code === 0 ? imageId.stdout.trim() || undefined : undefined + }, + } +} + // Builds one `run` invocation. The container has no network, no Linux // capabilities, no privilege escalation, a read-only root filesystem, fixed // CPU, memory, and process limits, and a size-capped scratch tmpfs. The // grading copy mounts read-only, so agent-written code cannot change the // hidden tests at grading time. The driver mounts read-only from repo code. // No environment variable passes through, so no credential can leak in. -export function buildContainerArgs(job: GraderJob, outDir: string): string[] { +export function buildContainerArgs(job: GraderJob, outDir: string, containerName?: string): string[] { + const leading: string[] = ["run"] + if (containerName !== undefined) leading.push("--name", containerName) return [ - "run", + ...leading, "--rm", "--network", "none", @@ -87,51 +181,176 @@ export function buildContainerArgs(job: GraderJob, outDir: string): string[] { ] } +// Unique per grading run, so concurrent or repeated runs never collide and a +// leftover container is always removable by name. +const containerNameFor = (): string => `agent-evals-grader-${process.pid}-${Date.now()}-${randomUUID().slice(0, 8)}` + // Default grading runtime. Runs the driver in a disposable container. The -// host enforces the wall-clock limit and kills the container on expiry. +// host enforces the wall-clock limit; on expiry it removes the container by +// name and verifies the removal before any output is deleted. A `timeout` +// spawn option alone would kill only the CLI client and leave the container +// running. First-run image pulls happen before the timed run, so grading +// time is never consumed by pulling the image. export function createContainerGradingRuntime(options: ContainerRuntimeOptions = {}): GradingRuntime { - const detect = options.detectExecutable ?? defaultDetect + let cli: ContainerCli | undefined + const resolveCli = async (): Promise => { + if (options.cli) return options.cli + if (!cli) { + const detect = options.detectExecutable ?? defaultDetect + const executable = options.executable ?? (await detect()) + if (!executable) return undefined + cli = createDefaultContainerCli(executable) + } + return cli + } + let resolvedDigest: string | undefined return { isolation: "container", async run(job: GraderJob): Promise { - const executable = await detect() - if (!executable) return { kind: "grading-failure", isolation: "container", reason: "noContainerRuntime" } + const failure = (reason: string): GraderVerdict => ({ + kind: "grading-failure", + isolation: "container", + reason, + }) + const selected = await resolveCli() + if (!selected) return failure("noContainerRuntime") + // The pull stays outside the grading timeout on purpose. + if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") + resolvedDigest = (await selected.imageDigest(job.resources.image)) ?? resolvedDigest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) + // The container runs as root with all capabilities dropped, so it + // cannot rely on CAP_DAC_OVERRIDE. The mount target must grant the + // write to any uid. + await fs.chmod(outDir, 0o777) + const name = containerNameFor() + const args = buildContainerArgs(job, outDir, name) + let exit: number | "timeout" | "error" try { - const args = buildContainerArgs(job, outDir) - const exit = await new Promise((resolve) => { - const child = spawn(executable, args, { - stdio: "ignore", - timeout: job.resources.timeoutMs, - killSignal: "SIGKILL", - }) - child.once("error", () => resolve("error")) - child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) - }) - if (exit === "timeout") - return { kind: "grading-failure", isolation: "container", reason: "graderTimeout" } - if (exit === "error" || exit !== 0) - return { kind: "grading-failure", isolation: "container", reason: "graderCrashed" } - const result = await readDriverResult(path.join(outDir, "result.json")) - if (!result) return { kind: "grading-failure", isolation: "container", reason: "graderResultMissing" } - return verdictFromDriver(result, "container") + exit = await selected.run(args, job.resources.timeoutMs) } catch { - return { kind: "grading-failure", isolation: "container", reason: "graderSetupFailed" } + exit = "error" + } + if (exit === "timeout") { + // Verify the container is gone before the output mount is + // deleted. A failed removal leaves an unbounded process behind, + // so it must surface as a grading failure. + const removal = await selected.remove(name) + await fs.rm(outDir, { recursive: true, force: true }).catch(() => {}) + return removal === "failed" ? failure("graderContainerRemovalFailed") : failure("graderTimeout") + } + if (exit === "error" || exit !== 0) { + // A normally exited container removes itself through --rm; the + // explicit removal verifies it and cleans a killed client's + // leftover. + await selected.remove(name) + await fs.rm(outDir, { recursive: true, force: true }).catch(() => {}) + return failure("graderCrashed") + } + try { + const loaded = await loadDriverResult(path.join(outDir, "result.json")) + if (loaded === "missing") return failure("graderResultMissing") + if (loaded === "invalid") return failure("graderResultInvalid") + return verdictFromDriver(loaded, "container") } finally { - await fs.rm(outDir, { recursive: true, force: true }) + await fs.rm(outDir, { recursive: true, force: true }).catch(() => {}) } }, + async imageDigest(): Promise> { + if (resolvedDigest !== undefined) return { available: true, value: resolvedDigest } + const selected = await resolveCli() + if (!selected) return { available: false, reason: "noContainerRuntime" } + return { available: false, reason: "digestUnavailable" } + }, } } -export async function readDriverResult(outPath: string): Promise { +export const DRIVER_RESULT_MAX_BYTES = 65_536 +const MAX_TEST_FILES = 1_000 +const MAX_FAILURE_ENTRIES = 50 +const MAX_COUNT = 100_000 + +const isBoundedInt = (value: unknown, max: number): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= max + +const isBoundedString = (value: unknown, max: number): value is string => + typeof value === "string" && value.length >= 1 && value.length <= max + +// Validates one untrusted driver result. The result file is written inside +// the container where agent-written test code executes, so nothing about its +// content is trusted. Enforces shape, bounds, and internal consistency; +// anything else is rejected. +export function parseDriverResult(text: string): DriverResult | undefined { + if (text.length > DRIVER_RESULT_MAX_BYTES) return undefined + let parsed: unknown try { - const parsed = JSON.parse(await fs.readFile(outPath, "utf8")) as DriverResult - return parsed.kind === "completed" || parsed.kind === "driver-error" ? parsed : undefined + parsed = JSON.parse(text) } catch { return undefined } + if (typeof parsed !== "object" || parsed === null) return undefined + const record = parsed as Record + if (record["kind"] === "driver-error") { + if (!isBoundedString(record["message"], 512)) return undefined + return { kind: "driver-error", message: record["message"] } + } + if (record["kind"] !== "completed") return undefined + if (typeof record["passed"] !== "boolean") return undefined + if (typeof record["noTestsDiscovered"] !== "boolean") return undefined + const passedCount = record["passedCount"] + const failedCount = record["failedCount"] + const cancelledCount = record["cancelledCount"] + if (!isBoundedInt(passedCount, MAX_COUNT)) return undefined + if (!isBoundedInt(failedCount, MAX_COUNT)) return undefined + if (!isBoundedInt(cancelledCount, MAX_COUNT)) return undefined + const testFiles = record["testFiles"] + if (!Array.isArray(testFiles) || testFiles.length > MAX_TEST_FILES) return undefined + if (testFiles.some((entry) => !isBoundedString(entry, 4_096))) return undefined + if (!record["noTestsDiscovered"] && testFiles.length === 0) return undefined + const failures = record["failures"] + if (!Array.isArray(failures) || failures.length > MAX_FAILURE_ENTRIES) return undefined + for (const entry of failures) { + if (typeof entry !== "object" || entry === null) return undefined + const item = entry as Record + if (!isBoundedString(item["name"], 1_024)) return undefined + if (!isBoundedString(item["file"], 4_096)) return undefined + } + if (failures.length > failedCount) return undefined + const durationMs = record["durationMs"] + if (typeof durationMs !== "number" || !Number.isFinite(durationMs) || durationMs < 0) return undefined + // Consistency: a passing verdict requires zero failures, zero + // cancellations, and at least one passing test. + const expectedPassed = failedCount === 0 && cancelledCount === 0 && passedCount > 0 + if (record["passed"] !== expectedPassed) return undefined + return { + kind: "completed", + passed: record["passed"], + noTestsDiscovered: record["noTestsDiscovered"], + testFiles: testFiles as string[], + passedCount, + failedCount, + cancelledCount, + failures: failures as { name: string; file: string }[], + durationMs, + } +} + +export type LoadedDriverResult = DriverResult | "missing" | "invalid" + +// Reads one driver result file with a hard size bound. "missing" covers an +// absent or unreadable file; "invalid" covers any untrusted content that +// fails validation. +export async function loadDriverResult(outPath: string): Promise { + let text: string + try { + const stat = await fs.stat(outPath) + if (!stat.isFile() || stat.size > DRIVER_RESULT_MAX_BYTES) return "invalid" + text = await fs.readFile(outPath, "utf8") + } catch { + return "missing" + } + const parsed = parseDriverResult(text) + return parsed ?? "invalid" } export function verdictFromDriver(result: DriverResult, isolation: GraderVerdict["isolation"]): GraderVerdict { diff --git a/scripts/agent-evals/fs-atomic.test.ts b/scripts/agent-evals/fs-atomic.test.ts new file mode 100644 index 0000000000..c34d86f91b --- /dev/null +++ b/scripts/agent-evals/fs-atomic.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" + +describe("atomic file publication", () => { + it("publishes the content and leaves no temp files", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const target = path.join(dir, "report.json") + await writeFileAtomic(target, '{"a":1}') + assert.equal(await fs.readFile(target, "utf8"), '{"a":1}') + const entries = await fs.readdir(dir) + assert.deepEqual(entries, ["report.json"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("replaces an existing file in place", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const target = path.join(dir, "side-results.json") + await writeJsonAtomic(target, { version: 1 }) + await writeJsonAtomic(target, { version: 2 }) + assert.equal(await fs.readFile(target, "utf8"), JSON.stringify({ version: 2 }, null, 2)) + const entries = await fs.readdir(dir) + assert.deepEqual(entries, ["side-results.json"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("preserves the previous file and leaves no temp residue when a write fails", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const target = path.join(dir, "run-record.json") + await writeFileAtomic(target, "previous") + // The temp file is created before the rename; a rename into a + // missing directory fails and must clean its temp file while the + // original stays intact. + await assert.rejects(writeFileAtomic(path.join(dir, "missing", "run-record.json"), "next")) + assert.equal(await fs.readFile(target, "utf8"), "previous") + const entries = await fs.readdir(dir) + assert.deepEqual(entries, ["run-record.json"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/fs-atomic.ts b/scripts/agent-evals/fs-atomic.ts new file mode 100644 index 0000000000..7f627e1754 --- /dev/null +++ b/scripts/agent-evals/fs-atomic.ts @@ -0,0 +1,26 @@ +import { randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import path from "node:path" + +// Publishes one file atomically. The content lands in a temp file in the +// target's own directory and is renamed into place, so readers never observe +// a partial write and a failed write never destroys the previous file. The +// temp name is unique per process and call, so concurrent writers cannot +// collide. +export async function writeFileAtomic(filePath: string, data: string): Promise { + const directory = path.dirname(filePath) + const temp = path.join(directory, `.${path.basename(filePath)}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`) + try { + await fs.writeFile(temp, data) + await fs.rename(temp, filePath) + } catch (error) { + await fs.rm(temp, { force: true }).catch(() => {}) + throw error + } +} + +// Serializes one value to JSON and publishes it atomically. Serialization +// failures throw before any file is touched. +export async function writeJsonAtomic(filePath: string, value: unknown): Promise { + await writeFileAtomic(filePath, JSON.stringify(value, null, 2)) +} diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs index 203894323b..9242597534 100644 --- a/scripts/agent-evals/grader-driver.mjs +++ b/scripts/agent-evals/grader-driver.mjs @@ -86,6 +86,7 @@ try { testFiles: [], passedCount: 0, failedCount: 0, + cancelledCount: 0, failures: [], durationMs: 0, }) @@ -94,6 +95,9 @@ try { const started = Date.now() let passedCount = 0 let failedCount = 0 + // Cancelled tests never ran. A passing verdict requires zero of them, so a + // solution cannot pass while a hidden test stayed pending. + let cancelledCount = 0 let runnerBroke = false const failures = [] for (const file of files) { @@ -110,6 +114,7 @@ try { } passedCount += counts.passed failedCount += counts.failed + cancelledCount += counts.cancelled ?? 0 if (counts.failed > 0 && failures.length < 50) for (const name of readFailureNames(outcome.tap)) failures.push({ name, file }) } @@ -119,11 +124,12 @@ try { } writeResult({ kind: "completed", - passed: failedCount === 0 && passedCount > 0, + passed: failedCount === 0 && cancelledCount === 0 && passedCount > 0, noTestsDiscovered: false, testFiles: files, passedCount, failedCount, + cancelledCount, failures, durationMs: Date.now() - started, }) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 5164fc240e..66ef28b270 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -14,8 +14,14 @@ import { identityUnavailable, measured, type MeasuredString, type TraceRunIdenti const execFileAsync = promisify(execFile) export interface HostRuntimeOptions { + // Repository root of the evaluated extension. Reads of package metadata, + // commit identity, and the extension source all stay inside it. root: string artifactsDir: string + // Optional trusted cache for the downloaded VS Code build. Defaults to + // the evaluated root, so callers driving historical worktrees can keep + // one cache in the harness checkout. + vscodeCachePath?: string now?: () => number pause?: (ms: number) => Promise // Optional --extensionTestsPath module. The module's run() must never @@ -128,7 +134,7 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { } const executable = await downloadAndUnzipVSCode({ version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", - cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + cachePath: options.vscodeCachePath ?? path.join(root, "apps/vscode-e2e/.vscode-test"), }) const args = buildLaunchArgs( workspace, diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index d233766dc6..a9dafea6d3 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -89,6 +89,24 @@ const EVENT_BUFFER_LIMIT = 256 type BufferedEvent = { kind: "parsed"; event: TaskEvent } +// True when the event carries the model's final, non-partial completion_result +// say message for the accepted task. This is model output, not the +// accepted-completion event. The headless stop policy in coding-run.ts uses +// the same predicate as the trace's completionMessageSeen refinement. +export const isFinalCompletionMessage = (event: TaskEvent): boolean => { + if (event.eventName !== RooCodeEventName.Message) return false + const message = event.payload[0]?.message + if (typeof message !== "object" || message === null) return false + const record = message as { type?: unknown; say?: unknown; partial?: unknown; text?: unknown } + return ( + record.type === "say" && + record.say === "completion_result" && + record.partial !== true && + typeof record.text === "string" && + record.text.trim().length > 0 + ) +} + const traceSettings = (fixtureProjection: Record, credential: string): AgentTrace["settings"] => { try { const projectionJson = JSON.stringify(fixtureProjection) @@ -179,17 +197,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp return } accountant.observe(event) - if (event.eventName === RooCodeEventName.Message) { - const message = event.payload[0].message - if ( - message.type === "say" && - message.say === "completion_result" && - message.partial !== true && - typeof message.text === "string" && - message.text.trim().length > 0 - ) - completionMessageSeen = true - } + if (isFinalCompletionMessage(event)) completionMessageSeen = true if (event.eventName === RooCodeEventName.TaskCompleted) completedSeen = true options.observers?.onAcceptedEvent?.(event, acceptedTaskId) } From 95a525cf862d83852b06cd21370a2d639251f226 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 24 Oct 2026 01:29:01 +0000 Subject: [PATCH 03/22] fix(evals): enforce trust checks and reliable comparison cleanup and reports --- scripts/agent-evals/README.md | 22 +- scripts/agent-evals/coding-compare.mts | 109 +++++-- scripts/agent-evals/coding-compare.ts | 181 ++++++++++- scripts/agent-evals/coding-outcomes.test.ts | 18 ++ scripts/agent-evals/coding-outcomes.ts | 7 + scripts/agent-evals/coding-report.test.ts | 291 +++++++++++++++++- scripts/agent-evals/coding-run.test.ts | 116 ++++++- scripts/agent-evals/coding-run.ts | 6 +- scripts/agent-evals/coding-side.mts | 58 ++-- scripts/agent-evals/coding-side.test.ts | 178 +++++++++++ scripts/agent-evals/coding-trust.test.ts | 60 +++- scripts/agent-evals/coding-trust.ts | 50 ++- scripts/agent-evals/container-runtime.test.ts | 28 ++ scripts/agent-evals/container-runtime.ts | 15 +- scripts/agent-evals/fs-atomic.test.ts | 22 ++ scripts/agent-evals/runner.test.ts | 94 +++++- scripts/agent-evals/runner.ts | 8 +- .../agent-evals/side-process-guard.test.ts | 266 ++++++++++++++++ scripts/agent-evals/side-process-guard.ts | 70 +++++ scripts/solheim-smoke/run.test.ts | 6 +- 20 files changed, 1510 insertions(+), 95 deletions(-) create mode 100644 scripts/agent-evals/coding-side.test.ts create mode 100644 scripts/agent-evals/side-process-guard.test.ts create mode 100644 scripts/agent-evals/side-process-guard.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 16734fee5b..d24043fa7e 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -85,8 +85,10 @@ them: message still appears in `completionMessageSeen`. - `SESSION_LOST` covers disconnects and interrupts. A manual stop after completion output stays `SESSION_LOST`. -- The usage drain runs only after a `taskCompleted` event. It stays inside the - bounded drain window. +- The usage drain runs after a `taskCompleted` event and after an observer stop + that followed the final `completion_result` message. It stays inside the + bounded drain window. A plain stop, a run timeout, and a session loss get no + drain. ## Checks @@ -152,9 +154,17 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. + +Every driver child, preparation steps and the side alike, runs detached in its own process group and is tracked by one guard. A SIGINT or SIGTERM that reaches only the driver terminates the tracked group through a bounded SIGTERM-then-SIGKILL escalation, and the worktree cleanup waits for that escalation before it removes any tree. + ### Output publication contract -Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. `side-results.json` appears only after every repetition of that side finished, so its existence means a complete side. A missing `side-results.json` means an interrupted side; consumers must treat it as a failed side and never pair partial output. The comparison reports are written atomically and only after both sides published complete results; a failed comparison leaves no report files. +Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. + +A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and the whole `artifacts` tree, so stale results from an earlier rerun can never survive. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. + +The comparison reports are published as one generation. `comparison-report.json` and `comparison-report.md` carry the report's `generatedAt` stamp, and `comparison-report.complete` holds that stamp; it is written last. Consumers must require the marker and match its stamp against the report files. `readPublishedComparisonReport` enforces this: it returns the report only when the marker exists and its stamp matches both files. The marker is removed before publication starts, so a crash between the two file publications leaves a detectable incomplete generation instead of a mixed one. A thrown failure rolls both files back; a failed comparison leaves no report files. ### Commands @@ -163,9 +173,9 @@ Artifacts are published atomically: content lands in a same-directory temp file pnpm test:agent-evals:unit pnpm agent-evals:check-types -# One comparison side against the real provider. Requires the credential -# environment variable and the container runtime for grading. -pnpm coding-evals:side --reps 3 --out /tmp/side-out +# One comparison side against the real provider. Requires the trust policy, +# the credential environment variable, and the container runtime for grading. +pnpm coding-evals:side --reps 3 --out /tmp/side-out --trust-file trust-policy.json # Full paired comparison of two approved commits. pnpm coding-evals:compare --baseline --candidate --trust-file trust-policy.json --reps 3 --out /tmp/compare-out diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index 89bd85fa64..cf3863f041 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -9,17 +9,38 @@ // worktree. The credential never leaves the trusted processes except through // the one named provider variable. A commit runs only when its full SHA is // listed in the operator's trust policy; refs are resolved and checked -// against the policy before any worktree or process exists. +// against the policy before any worktree or process exists. The side process +// re-checks the root it was handed against the same policy. +// +// Process containment. Install, build, worktree, and side children receive +// only an allowlisted environment, so the driver's secrets never reach them. +// Each runs detached in its own process group under one guard, so SIGINT and +// SIGTERM aimed at the driver alone terminate the tracked group with a bounded +// SIGTERM-then-SIGKILL escalation before the scratch cleanup runs, and an +// interrupt leaves no orphan host behind. import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" -import { sideResultFileName, toSideResult, validateSideResultFile, type SideResultFile } from "./coding-compare.ts" +import { + publishComparisonReport, + runBoundedPreparation, + sideResultFileName, + toSideResult, + validateSideResultFile, + type SideResultFile, +} from "./coding-compare.ts" import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" -import { assertCommitsApproved, buildSideEnvironment, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" -import { writeJsonAtomic, writeFileAtomic } from "./fs-atomic.ts" +import { + assertCommitsApproved, + assertCredentialEnvAllowed, + buildSideEnvironment, + loadTrustPolicy, + resolveCommit, +} from "./coding-trust.ts" +import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" const execFileAsync = promisify(execFile) const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -30,6 +51,22 @@ const WORKTREE_TIMEOUT_MS = 120_000 const SIDE_KILL_GRACE_MS = 30_000 const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 +// Interrupt state. The handlers are registered before any child is spawned. +// A signal terminates the tracked child group at once; the main flow observes +// the child exit, the cleanup awaits the escalation before it removes the +// worktrees, and the driver exits with the conventional signal exit code. +const sideGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) +let interruptSignal: NodeJS.Signals | undefined +let interruptKill: Promise | undefined +const isInterrupted = (): boolean => interruptSignal !== undefined +const onInterruptSignal = (signal: NodeJS.Signals): void => { + if (interruptSignal !== undefined) return + interruptSignal = signal + interruptKill = sideGuard.interrupt() +} +process.on("SIGINT", () => onInterruptSignal("SIGINT")) +process.on("SIGTERM", () => onInterruptSignal("SIGTERM")) + const readArgs = (): { baseline: string candidate: string @@ -58,42 +95,59 @@ const readArgs = (): { if (!Number.isInteger(sideTimeoutMinutes) || sideTimeoutMinutes < 1) { throw new Error("--side-timeout-minutes must be a positive integer") } + const credentialEnv = read("credential-env") ?? "SOLHEIM_API_KEY" + // The variable name must be one the framework supports. An arbitrary name + // could forward an unrelated parent secret to the side. + assertCredentialEnvAllowed(credentialEnv) return { baseline, candidate, reps, out: path.resolve(out), - credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + credentialEnv, allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), trustFile: path.resolve(trustFile), sideTimeoutMs: sideTimeoutMinutes * 60_000, } } -const runBounded = async (command: string, args: readonly string[], cwd: string, timeoutMs: number): Promise => { - await execFileAsync(command, [...args], { cwd, timeout: timeoutMs, killSignal: "SIGKILL" }) -} - // Prepares the worktree of one evaluated commit. Dependencies install from // the worktree's own frozen lockfile, and that exact commit's extension and // webview build from its own sources. No artifact crosses sides. const prepareWorktree = async (worktree: string): Promise => { - await runBounded("pnpm", ["install", "--frozen-lockfile"], worktree, INSTALL_TIMEOUT_MS) - await runBounded("pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) + await runBoundedPreparation( + sideGuard, + isInterrupted, + "pnpm", + ["install", "--frozen-lockfile"], + worktree, + INSTALL_TIMEOUT_MS, + ) + await runBoundedPreparation(sideGuard, isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) } // Runs one trusted side against the evaluated worktree. The side script comes // from this checkout, receives only the allowlisted environment plus the -// named credential variable, and must finish inside its own wall clock. +// named credential variable, re-checks the worktree commit against the trust +// policy, and must finish inside its own wall clock. const runSide = async ( side: "baseline" | "candidate", commit: string, inputs: ReturnType, scratchDir: string, ): Promise => { + if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) const worktree = path.join(scratchDir, side) - await runBounded("git", ["worktree", "add", "--detach", worktree, commit], root, WORKTREE_TIMEOUT_MS) + await runBoundedPreparation( + sideGuard, + isInterrupted, + "git", + ["worktree", "add", "--detach", worktree, commit], + root, + WORKTREE_TIMEOUT_MS, + ) await prepareWorktree(worktree) + if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) const sideOut = path.join(scratchDir, `${side}-out`) await fs.mkdir(sideOut, { recursive: true }) const exit = await new Promise((resolve) => { @@ -107,6 +161,8 @@ const runSide = async ( String(inputs.reps), "--out", sideOut, + "--trust-file", + inputs.trustFile, "--credential-env", inputs.credentialEnv, "--evaluated-root", @@ -120,6 +176,7 @@ const runSide = async ( detached: true, }, ) + sideGuard.track(child.pid) let timedOut = false const timer = setTimeout(() => { timedOut = true @@ -138,18 +195,20 @@ const runSide = async ( }, inputs.sideTimeoutMs) child.once("error", () => { clearTimeout(timer) + sideGuard.track(undefined) resolve(1) }) child.once("exit", (code) => { clearTimeout(timer) + sideGuard.track(undefined) resolve(timedOut ? 1 : (code ?? 1)) }) }) if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) - const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) - // The side output must describe exactly the approved commit, or the pair - // is invalid. - return validateSideResultFile(parsed, commit) + const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) + // The side output must describe exactly this side and its approved commit, + // or the pair is invalid. + return validateSideResultFile(parsed, side, commit) } const inputs = readArgs() @@ -171,17 +230,19 @@ try { candidate: toSideResult(await runSide("candidate", candidateCommit, inputs, scratchDir)), repetitions: inputs.reps, }) - // The reports are the comparison's publication point. They appear - // atomically and only after both sides published complete results; any - // failure above leaves no report files behind. - await writeJsonAtomic(path.join(inputs.out, "comparison-report.json"), report) - await writeFileAtomic(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) + // The reports are the comparison's publication point. The pair is + // published as one generation with a completion marker; any failure above + // or during publication leaves no complete report generation behind. + await publishComparisonReport(inputs.out, report) console.log( report.compatibility.compatible ? `comparison complete: compatible pair, report in ${inputs.out}` : `comparison complete: INCOMPATIBLE pair (${report.compatibility.differences.join(", ")}), report in ${inputs.out}`, ) } finally { + // An interrupt escalation must finish before the worktrees go away, so + // no child is still writing while its tree is deleted. + await interruptKill await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "baseline")], { cwd: root, timeout: 30_000, @@ -192,3 +253,7 @@ try { }).catch(() => {}) await fs.rm(scratchDir, { recursive: true, force: true }).catch(() => {}) } +// An interrupt terminated the side group above, and the cleanup in the +// finally block has completed. Exit with the conventional signal code. +if (interruptSignal === "SIGINT") process.exit(130) +if (interruptSignal === "SIGTERM") process.exit(143) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 0a8b9cb04e..921b901283 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -1,7 +1,13 @@ +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import path from "node:path" import type { SidePins } from "./coding-pins.ts" import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" import { DEFAULT_REPETITIONS } from "./coding-stats.ts" import type { CodingRunRecord } from "./coding-run.ts" +import { buildWorktreeEnvironment } from "./coding-trust.ts" +import type { SideProcessGuard } from "./side-process-guard.ts" +import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" export { buildComparisonReport, renderMarkdownReport } export type { ComparisonReport, SideResult } @@ -45,6 +51,63 @@ export async function runComparison(request: ComparisonRequest): Promise boolean, + command: string, + args: readonly string[], + cwd: string, + timeoutMs: number, +): Promise => { + if (isInterrupted()) throw new Error(`interrupted before ${command} started`) + await new Promise((resolve, reject) => { + const child = spawn(command, [...args], { + cwd, + stdio: ["ignore", "ignore", "pipe"], + detached: true, + env: buildWorktreeEnvironment(process.env), + }) + guard.track(child.pid) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + if (stderr.length >= STDERR_LIMIT_BYTES) return + stderr += chunk.toString() + }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, timeoutMs) + child.once("error", (error) => { + clearTimeout(timer) + guard.track(undefined) + reject(error) + }) + child.once("exit", (code, signal) => { + clearTimeout(timer) + guard.track(undefined) + if (code === 0) { + resolve() + return + } + const reason = timedOut ? "its time bound expired" : code === null ? `signal ${signal}` : `code ${code}` + reject(new Error([`${command} failed: ${reason}`, stderr].filter(Boolean).join("\n"))) + }) + }) +} + // Serializes a side result for artifact storage. Full run records travel so // each side's traces and grading verdicts stay inspectable. export interface SideResultFile { @@ -54,23 +117,58 @@ export interface SideResultFile { // Validates one side result file before the report builder consumes it. The // file is data, so its shape is checked, and it must record exactly the -// approved commit of its side. Anything else aborts the comparison instead of -// producing a report from partial output. -export function validateSideResultFile(parsed: unknown, expectedCommit: string): SideResultFile { +// expected side label and the approved commit of that side. The pins must be +// complete, and every run must carry object metrics. Anything else aborts the +// comparison instead of producing a report from partial output. +const PIN_STRING_FIELDS = [ + "side", + "commit", + "suiteId", + "suiteDigest", + "settingsDigest", + "modelIdentifier", + "graderVersion", + "graderImageDigest", +] as const + +export function validateSideResultFile( + parsed: unknown, + expectedSide: SidePins["side"], + expectedCommit: string, +): SideResultFile { if (typeof parsed !== "object" || parsed === null) throw new Error("side result is not a JSON object") const record = parsed as Record const pins = record["pins"] if (typeof pins !== "object" || pins === null) throw new Error("side result has no pins") const pinsRecord = pins as Record - for (const field of ["side", "commit", "suiteId", "suiteDigest", "graderVersion"] as const) { + for (const field of PIN_STRING_FIELDS) { if (typeof pinsRecord[field] !== "string" || (pinsRecord[field] as string).length === 0) { throw new Error(`side result pins lack ${field}`) } } + if (pinsRecord["side"] !== expectedSide) { + throw new Error(`side result records side ${String(pinsRecord["side"])} instead of ${expectedSide}`) + } if (pinsRecord["commit"] !== expectedCommit) { throw new Error(`side result records commit ${String(pinsRecord["commit"])} instead of ${expectedCommit}`) } - if (!Array.isArray(record["runs"])) throw new Error("side result has no runs array") + const runnerEnvironment = pinsRecord["runnerEnvironment"] + if (typeof runnerEnvironment !== "object" || runnerEnvironment === null) { + throw new Error("side result pins lack runnerEnvironment") + } + for (const field of ["os", "nodeVersion", "vscodeTarget"] as const) { + const value = (runnerEnvironment as Record)[field] + if (typeof value !== "string" || value.length === 0) { + throw new Error(`side result runnerEnvironment lacks ${field}`) + } + } + const runs = record["runs"] + if (!Array.isArray(runs)) throw new Error("side result has no runs array") + for (const run of runs) { + if (typeof run !== "object" || run === null) throw new Error("side result run is not an object") + const metrics = (run as Record)["metrics"] + if (typeof metrics !== "object" || metrics === null) throw new Error("side result run lacks metrics") + } return parsed as SideResultFile } @@ -80,4 +178,75 @@ export const toSideResult = (file: SideResultFile): SideResult => ({ runs: file.runs.map((record) => record.metrics), }) -export const sideResultFileName = (side: "baseline" | "candidate"): string => `${side}-results.json` +// The one shared side result file name. The side writer and the comparison +// reader must agree on it. The name carries no side label, because each side +// writes into its own output directory. +export const sideResultFileName = (): string => "side-results.json" + +const COMPARISON_REPORT_JSON = "comparison-report.json" +const COMPARISON_REPORT_MD = "comparison-report.md" +const COMPARISON_REPORT_COMPLETE = "comparison-report.complete" + +// Publishes the JSON and Markdown report pair as one generation. The +// completion marker of any previous generation is removed first, so a crash +// between the two publications leaves a detectable incomplete generation +// instead of a mixed one. A thrown failure rolls both files back, so a failed +// comparison leaves no report files behind. Consumers must require the +// completion marker and match its generation against the report files. +export async function publishComparisonReport(outDir: string, report: ComparisonReport): Promise { + const generation = report.generatedAt + const jsonPath = path.join(outDir, COMPARISON_REPORT_JSON) + const markdownPath = path.join(outDir, COMPARISON_REPORT_MD) + const markerPath = path.join(outDir, COMPARISON_REPORT_COMPLETE) + await fs.rm(markerPath, { force: true }) + try { + await writeJsonAtomic(jsonPath, { ...report, generation }) + await writeFileAtomic(markdownPath, renderMarkdownReport(report)) + await writeFileAtomic(markerPath, `${generation}\n`) + } catch (error) { + await fs.rm(jsonPath, { force: true }).catch(() => {}) + await fs.rm(markdownPath, { force: true }).catch(() => {}) + await fs.rm(markerPath, { force: true }).catch(() => {}) + throw error + } +} + +// Reads the generation a complete publication carries. undefined means the +// publication is absent, partial, or crashed between its files. +export const publishedComparisonGeneration = async (outDir: string): Promise => { + try { + const generation = (await fs.readFile(path.join(outDir, COMPARISON_REPORT_COMPLETE), "utf8")).trim() + return generation.length > 0 ? generation : undefined + } catch { + return undefined + } +} + +// Reads one published comparison report. The completion marker is required +// and its generation stamp must match the JSON report and the Markdown +// render, so a partial or mixed generation is never consumed. undefined means +// no complete publication exists. +export async function readPublishedComparisonReport(outDir: string): Promise { + const generation = await publishedComparisonGeneration(outDir) + if (generation === undefined) return undefined + const parsed: unknown = JSON.parse(await fs.readFile(path.join(outDir, COMPARISON_REPORT_JSON), "utf8")) + if (typeof parsed !== "object" || parsed === null) throw new Error("comparison report is not a JSON object") + if ((parsed as Record)["generation"] !== generation) { + throw new Error("comparison report generation does not match the completion marker") + } + const markdown = await fs.readFile(path.join(outDir, COMPARISON_REPORT_MD), "utf8") + if (!markdown.includes(generation)) { + throw new Error("comparison report Markdown does not match the completion marker") + } + return parsed as ComparisonReport +} + +// Removes the previous side generation before a side executes. After this +// point a crash can no longer present stale results as fresh output, because +// consumers require side-results.json, and it exists only after the current +// invocation completes. A stale file that cannot be removed fails the side +// before any run spends budget. +export async function invalidateSideArtifacts(outDir: string): Promise { + await fs.rm(path.join(outDir, "artifacts"), { recursive: true, force: true }) + await fs.rm(path.join(outDir, sideResultFileName()), { force: true }) +} diff --git a/scripts/agent-evals/coding-outcomes.test.ts b/scripts/agent-evals/coding-outcomes.test.ts index 6b6dbf57fa..1a247ba859 100644 --- a/scripts/agent-evals/coding-outcomes.test.ts +++ b/scripts/agent-evals/coding-outcomes.test.ts @@ -57,4 +57,22 @@ describe("run outcome classification", () => { ) assert.equal(classifyRunOutcome({ ...base, grading: undefined }).outcome, "infrastructure-failure") }) + + it("reports a truncated final workspace as infrastructure failure without grading", () => { + // A truncated capture is incomplete. Even a passing verdict must not + // turn missing files into a correct answer. + assert.deepEqual(classifyRunOutcome({ ...base, finalWorkspaceTruncated: true, grading: { kind: "passed" } }), { + outcome: "infrastructure-failure", + reason: "finalWorkspaceTruncated", + }) + assert.deepEqual(classifyRunOutcome({ ...base, finalWorkspaceTruncated: true }), { + outcome: "infrastructure-failure", + reason: "finalWorkspaceTruncated", + }) + // A complete capture keeps grading. + assert.deepEqual(classifyRunOutcome({ ...base, finalWorkspaceTruncated: false, grading: { kind: "passed" } }), { + outcome: "correct", + reason: "hiddenTestsPassed", + }) + }) }) diff --git a/scripts/agent-evals/coding-outcomes.ts b/scripts/agent-evals/coding-outcomes.ts index 74738edc02..aca87ba7c4 100644 --- a/scripts/agent-evals/coding-outcomes.ts +++ b/scripts/agent-evals/coding-outcomes.ts @@ -7,6 +7,9 @@ export interface OutcomeInput { hasTrace: boolean executionCode?: AgentExecutionCode finalWorkspaceAvailable: boolean + // True when the captured workspace was incomplete. A truncated capture is + // a harness fault, never agent work, so it can never grade as incorrect. + finalWorkspaceTruncated?: boolean grading?: Pick & { reason?: string } } @@ -30,6 +33,8 @@ const INFRASTRUCTURE_CODES: readonly AgentExecutionCode[] = [ // - COMPLETED and STOPPED runs are graded. The hidden tests decide: // passed is correct, failed is incorrect, including when the agent // reported completion. +// - A truncated final workspace is infrastructure failure. The capture is +// incomplete, so grading it could turn missing files into a wrong answer. // - A grader that hung, crashed, or hit its limits is grading failure and is // never counted as incorrect. export function classifyRunOutcome(input: OutcomeInput): ClassifiedOutcome { @@ -42,6 +47,8 @@ export function classifyRunOutcome(input: OutcomeInput): ClassifiedOutcome { return { outcome: "infrastructure-failure", reason: `execution-${code ?? "unknown"}` } if (!input.finalWorkspaceAvailable) return { outcome: "infrastructure-failure", reason: "finalWorkspaceUnavailable" } + if (input.finalWorkspaceTruncated === true) + return { outcome: "infrastructure-failure", reason: "finalWorkspaceTruncated" } if (!input.grading) return { outcome: "infrastructure-failure", reason: "graderNotRun" } switch (input.grading.kind) { case "passed": diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 8cf8667a2c..07a7617a72 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -1,7 +1,19 @@ import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" import { describe, it } from "node:test" import { buildComparisonReport, renderMarkdownReport, type SideResult } from "./coding-report.ts" -import { runComparison, validateSideResultFile, type SideExecutor } from "./coding-compare.ts" +import { + invalidateSideArtifacts, + publishComparisonReport, + publishedComparisonGeneration, + readPublishedComparisonReport, + runComparison, + validateSideResultFile, + type SideExecutor, +} from "./coding-compare.ts" +import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" import type { CodingRunMetrics } from "./coding-metrics.ts" import type { SidePins } from "./coding-pins.ts" @@ -193,18 +205,277 @@ describe("comparison orchestration", () => { }) describe("side result validation", () => { - it("accepts a result that records exactly the approved commit", () => { - const file = { pins: pins("baseline", "a".repeat(40)), runs: [] } - const validated = validateSideResultFile(JSON.parse(JSON.stringify(file)), "a".repeat(40)) + const validatedFile = (side: "baseline" | "candidate", commit: string, overrides: Record = {}) => + JSON.parse(JSON.stringify({ pins: pins(side, commit), runs: [], ...overrides })) + + it("accepts a result that records exactly the expected side and commit", () => { + const validated = validateSideResultFile(validatedFile("baseline", "a".repeat(40)), "baseline", "a".repeat(40)) assert.equal(validated.pins.side, "baseline") + assert.equal(validated.pins.commit, "a".repeat(40)) + }) + + it("rejects a result that records the wrong side or a different commit", () => { + assert.throws( + () => validateSideResultFile(validatedFile("candidate", "a".repeat(40)), "baseline", "a".repeat(40)), + /records side candidate instead of baseline/, + ) + assert.throws( + () => validateSideResultFile(validatedFile("baseline", "b".repeat(40)), "baseline", "a".repeat(40)), + /records commit .* instead of/, + ) + }) + + it("rejects incomplete pins with the intended error", () => { + for (const field of [ + "side", + "commit", + "suiteId", + "suiteDigest", + "settingsDigest", + "modelIdentifier", + "graderVersion", + "graderImageDigest", + ]) { + const file = validatedFile("baseline", "a".repeat(40)) + delete (file.pins as Record)[field] + assert.throws( + () => validateSideResultFile(file, "baseline", "a".repeat(40)), + new RegExp(`pins lack ${field}`), + ) + } + assert.throws( + () => + validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { pins: { side: "baseline" } }), + "baseline", + "a".repeat(40), + ), + /pins lack commit/, + ) + }) + + it("rejects incomplete runnerEnvironment pins", () => { + const file = validatedFile("baseline", "a".repeat(40)) + delete (file.pins as Record)["runnerEnvironment"] + assert.throws(() => validateSideResultFile(file, "baseline", "a".repeat(40)), /pins lack runnerEnvironment/) + const partial = validatedFile("baseline", "a".repeat(40)) + ;(partial.pins as Record)["runnerEnvironment"] = { os: "linux", nodeVersion: "v22" } + assert.throws( + () => validateSideResultFile(partial, "baseline", "a".repeat(40)), + /runnerEnvironment lacks vscodeTarget/, + ) + }) + + it("rejects runs without object metrics", () => { + assert.throws( + () => + validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { runs: [{ fixtureName: "clamp-range", repIndex: 0 }] }), + "baseline", + "a".repeat(40), + ), + /run lacks metrics/, + ) + assert.throws( + () => + validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { runs: ["nope"] }), + "baseline", + "a".repeat(40), + ), + /run is not an object/, + ) }) - it("rejects a result that records a different commit, or has no shape", () => { - assert.throws(() => - validateSideResultFile({ pins: pins("baseline", "b".repeat(40)), runs: [] }, "a".repeat(40)), + it("rejects missing shape with the intended error", () => { + assert.throws(() => validateSideResultFile({ runs: [] }, "baseline", "a".repeat(40)), /has no pins/) + assert.throws(() => validateSideResultFile(null, "baseline", "a".repeat(40)), /not a JSON object/) + assert.throws( + () => validateSideResultFile({ pins: pins("baseline", "a".repeat(40)) }, "baseline", "a".repeat(40)), + /has no runs array/, ) - assert.throws(() => validateSideResultFile({ runs: [] }, "a".repeat(40))) - assert.throws(() => validateSideResultFile(null, "a".repeat(40))) - assert.throws(() => validateSideResultFile({ pins: pins("baseline", "a".repeat(40)) }, "a".repeat(40))) + }) +}) + +describe("comparison report publication", () => { + const report = buildComparisonReport({ + baseline: sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.1)]), + candidate: sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]), + }) + + it("publishes the pair with a matching completion marker", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + await publishComparisonReport(dir, report) + const generation = await publishedComparisonGeneration(dir) + assert.equal(generation, report.generatedAt) + const json = JSON.parse(await fs.readFile(path.join(dir, "comparison-report.json"), "utf8")) as { + generation?: string + } + assert.equal(json.generation, report.generatedAt) + const markdown = await fs.readFile(path.join(dir, "comparison-report.md"), "utf8") + assert.ok(markdown.includes(report.generatedAt)) + // The publication leaves no temp residue. + assert.deepEqual((await fs.readdir(dir)).sort(), [ + "comparison-report.complete", + "comparison-report.json", + "comparison-report.md", + ]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("replaces a prior report set and updates the marker to the new generation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + // An existing prior report set. + await writeFileAtomic(path.join(dir, "comparison-report.json"), "old") + await writeFileAtomic(path.join(dir, "comparison-report.md"), "old md") + await writeFileAtomic(path.join(dir, "comparison-report.complete"), "2020-01-01T00:00:00.000Z\n") + assert.equal(await publishedComparisonGeneration(dir), "2020-01-01T00:00:00.000Z") + await publishComparisonReport(dir, report) + assert.equal(await publishedComparisonGeneration(dir), report.generatedAt) + const markdown = await fs.readFile(path.join(dir, "comparison-report.md"), "utf8") + assert.ok(markdown.includes(report.generatedAt)) + assert.ok(!markdown.includes("old md")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rolls the pair back when the Markdown publication fails", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + // The Markdown target is a non-empty directory, so its atomic write + // fails after the JSON publication. + const markdownPath = path.join(dir, "comparison-report.md") + await fs.mkdir(markdownPath) + await fs.writeFile(path.join(markdownPath, "occupied"), "x") + await assert.rejects(publishComparisonReport(dir, report)) + // The JSON and marker are rolled back. No mixed generation and no + // temp residue remains. + assert.equal(await publishedComparisonGeneration(dir), undefined) + assert.equal( + await fs.access(path.join(dir, "comparison-report.json")).then( + () => true, + () => false, + ), + false, + ) + assert.deepEqual(await fs.readdir(dir), ["comparison-report.md"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("treats a crash between publications as an incomplete generation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + // Both files exist, but the process died before the marker. + await writeFileAtomic(path.join(dir, "comparison-report.json"), "partial") + await writeFileAtomic(path.join(dir, "comparison-report.md"), "partial md") + assert.equal(await publishedComparisonGeneration(dir), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) + +describe("comparison report consumption", () => { + const report = buildComparisonReport({ + baseline: sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.1)]), + candidate: sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]), + }) + + it("reads back the report only when the marker matches both files", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + const read = await readPublishedComparisonReport(dir) + assert.ok(read !== undefined) + assert.equal(read.generatedAt, report.generatedAt) + assert.equal(read.compatibility.compatible, report.compatibility.compatible) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("reports no publication when the marker is absent", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + // Both report files exist, but the process died before the marker. + await writeFileAtomic(path.join(dir, "comparison-report.json"), "{}") + await writeFileAtomic(path.join(dir, "comparison-report.md"), "partial md") + assert.equal(await readPublishedComparisonReport(dir), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("refuses a report whose generation differs from the marker", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + await writeFileAtomic( + path.join(dir, "comparison-report.json"), + JSON.stringify({ ...report, generation: "2020-01-01T00:00:00.000Z" }), + ) + await assert.rejects(readPublishedComparisonReport(dir), /does not match the completion marker/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("refuses a Markdown render that differs from the marker", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + await writeFileAtomic(path.join(dir, "comparison-report.md"), "stale render without the stamp") + await assert.rejects(readPublishedComparisonReport(dir), /does not match the completion marker/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("refuses a pair that lost its Markdown file", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + await fs.rm(path.join(dir, "comparison-report.md")) + await assert.rejects(readPublishedComparisonReport(dir)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) + +describe("side artifact invalidation", () => { + it("removes stale side results and run records before a rerun", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + const staleRecordDir = path.join(dir, "artifacts", "clamp-range-rep4") + await fs.mkdir(staleRecordDir, { recursive: true }) + await writeJsonAtomic(path.join(staleRecordDir, "run-record.json"), { stale: true }) + await invalidateSideArtifacts(dir) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("fails closed when the stale side result cannot be removed", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + // The stale result path is a non-empty directory, so the removal + // fails and the side must abort before any run executes. + const stalePath = path.join(dir, "side-results.json") + await fs.mkdir(stalePath) + await fs.writeFile(path.join(stalePath, "occupied"), "x") + await assert.rejects(invalidateSideArtifacts(dir)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } }) }) diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts index d26c627323..cbef1e6956 100644 --- a/scripts/agent-evals/coding-run.test.ts +++ b/scripts/agent-evals/coding-run.test.ts @@ -4,7 +4,7 @@ import os from "node:os" import path from "node:path" import { after, describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" -import { runCodingFixtureOnce, withWorkspaceCapture } from "./coding-run.ts" +import { runCodingFixtureOnce, stopOnFinalCompletionMessage, withWorkspaceCapture } from "./coding-run.ts" import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" import type { AgentClient, AgentHost, AgentRuntime, FinalWorkspaceSnapshot } from "./runner.ts" import type { AgentFixtureWorkspaceFile } from "./contract.ts" @@ -19,8 +19,14 @@ interface SyntheticOptions { // Emits the model's final non-partial completion_result message instead // of the accepted TaskCompleted event. completeByMessageOnly?: boolean + // Emits only a partial completion_result message. The headless stop + // policy must never treat it as final. + completeByPartialMessageOnly?: boolean // Rejects the final-workspace read, as a broken capture does. failWorkspaceRead?: boolean + // Reports the final-workspace capture as truncated, as a bounded walker + // over an oversized tree does. + truncateWorkspaceRead?: boolean } const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; traces: AgentTrace[] } => { @@ -48,11 +54,13 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t eventName: RooCodeEventName.TaskStartResponse, payload: [{ requestId, success: true, taskId: "task-1" }], }) - if (options.completeByMessageOnly) { + if (options.completeByMessageOnly || options.completeByPartialMessageOnly) { // The model reports completion, but the accepted TaskCompleted - // event never fires. This is the headless completion path. - // The synthetic message satisfies the Message payload shape; - // the cast names the variant the schema accepts. + // event never fires. This is the headless completion path. The + // partial variant streams the message with partial: true, which + // the stop policy must reject. The synthetic message satisfies + // the Message payload shape; the cast names the variant the + // schema accepts. emit({ eventName: RooCodeEventName.Message, payload: [ @@ -64,6 +72,7 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t say: "completion_result", ts: 1, text: "Task finished.", + ...(options.completeByPartialMessageOnly ? { partial: true } : {}), }, }, ], @@ -97,7 +106,7 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t }, readFinalWorkspace: async (): Promise => { if (options.failWorkspaceRead) throw new Error("unreadable workspace") - return { files: options.finalFiles, truncated: false } + return { files: options.finalFiles, truncated: options.truncateWorkspaceRead === true } }, } return { runtime, traces } @@ -255,4 +264,99 @@ describe("coding run orchestration", () => { await captured.runtime.cleanup() assert.equal(captured.getSnapshot(), first) }) + + const completionSayEvent = (message: Record): TaskEvent => + ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId: "task-1", action: "created", message: { ts: 1, ...message } }], + }) as unknown as TaskEvent + + it("stops on the final non-partial completion_result say message", () => { + const observer = stopOnFinalCompletionMessage() + observer.onAcceptedEvent!(completionSayEvent({ type: "say", say: "completion_result", text: "done" }), "task-1") + assert.equal(observer.shouldStop!(), true) + }) + + it("never stops on partial, blank, or wrong-say completion messages", () => { + const rejected = [ + { name: "partial", message: { type: "say", say: "completion_result", text: "working", partial: true } }, + { name: "blank", message: { type: "say", say: "completion_result", text: " \n\t " } }, + { name: "wrongSay", message: { type: "say", say: "text", text: "progress note" } }, + ] + for (const { name, message } of rejected) { + const observer = stopOnFinalCompletionMessage() + observer.onAcceptedEvent!(completionSayEvent(message), "task-1") + assert.equal(observer.shouldStop!(), false, `observer must not stop on a ${name} completion message`) + } + }) + + it( + "skips grading and reports infrastructure failure when the snapshot is truncated", + { timeout: 30_000 }, + async () => { + const bundle = codingBundle("clamp-range") + const { runtime } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: true, + truncateWorkspaceRead: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 5, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // A truncated capture is incomplete harness data. Grading it could + // score missing files as a wrong answer, so it never grades. + assert.equal(record.outcome.outcome, "infrastructure-failure") + assert.equal(record.outcome.reason, "finalWorkspaceTruncated") + assert.equal(record.grading, undefined) + assert.equal(record.finalWorkspaceTruncated, true) + }, + ) + + it("keeps grading a complete snapshot", { timeout: 30_000 }, async () => { + const bundle = codingBundle("clamp-range") + const { runtime } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: true, + truncateWorkspaceRead: false, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 6, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + assert.equal(record.outcome.outcome, "correct") + assert.equal(record.grading?.kind, "passed") + assert.equal(record.finalWorkspaceTruncated, false) + }) + + it("times out a run that only ever streams a partial completion message", { timeout: 30_000 }, async () => { + const { runtime } = syntheticRuntime({ + finalFiles: [], + complete: false, + completeByPartialMessageOnly: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 7, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // The partial message never satisfies the stop policy, so the run + // window expires instead of stopping early. A timeout is never graded. + assert.equal(record.executionCode, "TASK_RUN_TIMEOUT") + assert.equal(record.outcome.outcome, "timed-out") + assert.equal(record.grading, undefined) + assert.equal(record.trace?.execution.stopReason, "runTimeout") + assert.equal(record.trace?.execution.completionMessageSeen, false) + }) }) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts index b09cc0e973..07914237d2 100644 --- a/scripts/agent-evals/coding-run.ts +++ b/scripts/agent-evals/coding-run.ts @@ -132,7 +132,10 @@ export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { const args = process.argv.slice(2) const read = (name: string): string | undefined => { @@ -37,14 +41,21 @@ const readArgs = (): { } const out = read("out") if (!out) throw new Error("Set --out to a scratch directory") + const trustFile = read("trust-file") + if (!trustFile) throw new Error("Set --trust-file to the policy file listing the approved commit for this side") + const credentialEnv = read("credential-env") ?? "SOLHEIM_API_KEY" + // The variable name must be one the framework supports. An arbitrary name + // could forward an unrelated parent secret to the evaluated extension. + assertCredentialEnvAllowed(credentialEnv) return { reps: Number(read("reps") ?? "3"), out: path.resolve(out), - credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + credentialEnv, allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), // Repository root of the extension under test. The trusted harness // code drives the run; only the extension host loads this tree. evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), + trustFile: path.resolve(trustFile), } } @@ -71,20 +82,18 @@ const readVscodeTarget = async (root: string): Promise => { } } -const readCommit = async (root: string): Promise => { - const { execFile } = await import("node:child_process") - const { promisify } = await import("node:util") - try { - const { stdout } = await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) - return stdout.trim() - } catch { - return "commit-unavailable" - } -} - -const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot } = readArgs() +const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot, trustFile } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") const sideLabel = readSideLabel() +// Trust gate. The evaluated root is loaded by the extension host and receives +// the credential, so its resolved full commit must be approved before any +// directory is prepared, any host is created, or the credential is read. The +// comparison driver checks its resolved commits first; this second check binds +// the root that was actually passed in. The commit of an unapproved root is +// unreadable or foreign, and both fail closed here. +const policy = await loadTrustPolicy(trustFile) +const evaluatedCommit = await resolveCommit("HEAD", evaluatedRoot) +assertCommitsApproved(policy, [evaluatedCommit]) // Fail before any directory, run, or result file exists. An empty credential // would otherwise produce a plausible file of infrastructure failures. const credential = process.env[credentialEnv] @@ -95,6 +104,12 @@ ensureCodingSuiteRegistered() const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() +// Invalidate the previous generation before execution. After this point a +// crash can never leave stale side results or run records behind: consumers +// require side-results.json, and it exists only after this invocation +// completes. +await invalidateSideArtifacts(out) + const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} const runs: CodingRunRecord[] = [] @@ -138,12 +153,12 @@ for (const bundle of CODING_FIXTURE_BUNDLES) { } // Pins are recorded after the runs so they describe what actually executed: -// the evaluated commit and, when the container runtime resolved one, the -// immutable grader image identity. +// the approved evaluated commit and, when the container runtime resolved one, +// the immutable grader image identity. const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined const pins: SidePins = { side: sideLabel, - commit: await readCommit(evaluatedRoot), + commit: evaluatedCommit, suiteId: CODING_SUITE_ID, suiteDigest: codingSuiteDigest(), settingsDigest: digestValue(projection), @@ -165,10 +180,11 @@ for (const record of runs) { await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) } -// Publication point. side-results.json appears atomically only after every -// repetition finished, so a reader that sees the file sees a complete side. -// A missing file means an interrupted side; consumers must treat it as a -// failed side and never pair partial output. +// Publication point and completion marker. side-results.json appears +// atomically only after every repetition of this invocation finished, so a +// reader that sees the file sees a complete side of the current run. A +// missing file means an interrupted side; consumers must treat it as a failed +// side and never pair partial output. await writeJsonAtomic(path.join(out, "side-results.json"), result) const correct = runs.filter((record) => record.outcome.outcome === "correct").length console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts new file mode 100644 index 0000000000..fccf0c995a --- /dev/null +++ b/scripts/agent-evals/coding-side.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { describe, it } from "node:test" + +const execFileAsync = promisify(execFile) +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +interface RunOutcome { + code: number + stdout: string + stderr: string +} + +// Spawns the real side script with a minimal, credential-free environment. +// tsx resolves from the repository the same way the package scripts resolve +// it. The trust gate runs before any host creation, so these checks never +// download VS Code and never touch a provider. +const runSideScript = async (args: string[], env: NodeJS.ProcessEnv): Promise => { + try { + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", path.join(repoRoot, "scripts/agent-evals/coding-side.mts"), ...args], + { + cwd: repoRoot, + timeout: 120_000, + killSignal: "SIGKILL", + env, + }, + ) + return { code: 0, stdout, stderr: "" } + } catch (error) { + const failure = error as { code?: number | string; stdout?: string; stderr?: string; killed?: boolean } + if (failure.killed) throw new Error("side script hung and hit the test kill timer") + return { + code: typeof failure.code === "number" ? failure.code : 1, + stdout: failure.stdout ?? "", + stderr: failure.stderr ?? "", + } + } +} + +const minimalEnv = (extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv => ({ + PATH: process.env["PATH"] ?? "/usr/bin:/bin", + HOME: process.env["HOME"] ?? os.homedir(), + ...extra, +}) + +const writePolicy = async (dir: string, approvedCommits: string[]): Promise => { + const file = path.join(dir, "trust-policy.json") + await fs.writeFile(file, JSON.stringify({ version: 1, approvedCommits })) + return file +} + +// One empty fixture commit in a throwaway repository. It stands in for an +// evaluated root; no fixture code from it ever runs in these checks. +const makeFixtureCommit = async (dir: string): Promise<{ repo: string; commit: string }> => { + const repo = path.join(dir, "fixture-repo") + await fs.mkdir(repo) + await execFileAsync("git", ["init", "-q"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + // The operator's git config may force signed commits, and the + // throwaway fixture has no key or tty. + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "-q", + "-m", + "fixture", + ], + { cwd: repo }, + ) + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: repo }) + return { repo, commit: stdout.trim() } +} + +describe("coding side trust gate", () => { + it("rejects an unapproved evaluated root before any host launch", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + // The policy approves a different commit, so the fixture root is + // untrusted. + const trustFile = await writePolicy(dir, ["b".repeat(40)]) + const outcome = await runSideScript( + ["--reps", "1", "--out", path.join(dir, "out"), "--trust-file", trustFile, "--evaluated-root", repo], + minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /commits not approved by the trust policy/) + assert.ok(outcome.stderr.includes(commit.slice(0, 8))) + // The rejection happens before the run loop: nothing completed and + // no results file exists. + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects an unrelated secret variable before any process starts", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + // The root is approved; the credential variable name is not. + const trustFile = await writePolicy(dir, [commit]) + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, + "--credential-env", + "AWS_SECRET_ACCESS_KEY", + ], + minimalEnv({ AWS_SECRET_ACCESS_KEY: "leak-me" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /credential environment variable must be one of/) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it( + "accepts an approved root and stops at the credential check without launching a host", + { timeout: 180_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + const trustFile = await writePolicy(dir, [commit]) + // No credential variable is set. The gate must pass and the run + // must fail closed at the credential check, before any host or + // container work. + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, + ], + minimalEnv(), + ) + assert.notEqual(outcome.code, 0) + assert.match( + outcome.stderr, + /provider credential environment variable SOLHEIM_API_KEY is unset or empty/, + ) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) +}) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index db7a96b5ae..c30f55ab66 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -5,7 +5,9 @@ import path from "node:path" import { describe, it } from "node:test" import { assertCommitsApproved, + assertCredentialEnvAllowed, buildSideEnvironment, + buildWorktreeEnvironment, loadTrustPolicy, UntrustedCommitError, type TrustPolicy, @@ -58,12 +60,18 @@ describe("trust policy", () => { // The gate is case insensitive on the approved side. assertCommitsApproved(policy, [sha("A").toUpperCase()]) assert.throws(() => assertCommitsApproved(policy, [sha("b"), sha("c")]), UntrustedCommitError) - try { - assertCommitsApproved(policy, [sha("b")]) - } catch (error) { - assert.ok(error instanceof UntrustedCommitError) - assert.match(error.message, /not approved by the trust policy/) - } + // The single-commit rejection carries the exact rejected SHA. A silent + // pass here would approve an untrusted commit, so the assertion names + // the intended error instead of catching anything. + assert.throws( + () => assertCommitsApproved(policy, [sha("b")]), + (error: unknown) => { + assert.ok(error instanceof UntrustedCommitError) + assert.match(error.message, /not approved by the trust policy/) + assert.deepEqual(error.commits, [sha("b")]) + return true + }, + ) }) }) @@ -96,3 +104,43 @@ describe("side environment allowlist", () => { assert.equal(child["CODING_EVALS_SIDE"], "baseline") }) }) + +describe("credential environment allowlist", () => { + it("accepts the supported provider variable", () => { + assert.doesNotThrow(() => assertCredentialEnvAllowed("SOLHEIM_API_KEY")) + }) + + it("rejects unrelated secret variables before any process starts", () => { + for (const name of ["AWS_SECRET_ACCESS_KEY", "OPENAI_API_KEY", "GITHUB_TOKEN", "PATH"]) { + assert.throws(() => assertCredentialEnvAllowed(name), /credential environment variable must be one of/) + } + }) +}) + +describe("worktree process environment", () => { + it("passes the install and build allowlist and drops secrets", () => { + const child = buildWorktreeEnvironment({ + PATH: "/usr/bin", + HOME: "/home/dev", + PNPM_HOME: "/home/dev/.local/share/pnpm", + XDG_DATA_HOME: "/home/dev/.local/share", + npm_config_registry: "https://registry.example", + SOLHEIM_API_KEY: "sk-test", + AWS_SECRET_ACCESS_KEY: "leak-me", + OTHER_SECRET: "leak-me-too", + NODE_OPTIONS: "--require /tmp/never", + }) + assert.deepEqual(child, { + PATH: "/usr/bin", + HOME: "/home/dev", + PNPM_HOME: "/home/dev/.local/share/pnpm", + XDG_DATA_HOME: "/home/dev/.local/share", + npm_config_registry: "https://registry.example", + }) + }) + + it("never carries the provider credential into install or build", () => { + const child = buildWorktreeEnvironment({ PATH: "/usr/bin", SOLHEIM_API_KEY: "sk-test" }) + assert.equal(child["SOLHEIM_API_KEY"], undefined) + }) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index d1280943a3..6ff10a36e6 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -63,11 +63,23 @@ export function assertCommitsApproved(policy: TrustPolicy, commits: readonly str if (unapproved.length > 0) throw new UntrustedCommitError(unapproved) } -// Environment allowlist for the side process. The comparison driver may hold -// unrelated secrets in its own environment; the side gets only what it needs -// to spawn git, node, and the extension host, plus the one named credential -// variable. -const SIDE_ENV_ALLOWLIST = [ +// Provider credential variables this framework supports. The side script and +// the comparison driver accept only these names, so the tooling can never be +// pointed at an unrelated parent secret by naming an arbitrary variable. +export const CREDENTIAL_ENV_ALLOWLIST = ["SOLHEIM_API_KEY"] as const + +export function assertCredentialEnvAllowed(name: string): void { + if (!(CREDENTIAL_ENV_ALLOWLIST as readonly string[]).includes(name)) { + throw new Error(`credential environment variable must be one of: ${CREDENTIAL_ENV_ALLOWLIST.join(", ")}`) + } +} + +// Environment allowlist shared by every child this framework spawns. The +// comparison driver may hold unrelated secrets in its own environment; the +// children get only what they need to spawn git, node, and the extension +// host. Credentials never travel through this list; the side adds the one +// named credential variable separately. +const PROCESS_ENV_ALLOWLIST = [ "PATH", "HOME", "LANG", @@ -83,13 +95,15 @@ const SIDE_ENV_ALLOWLIST = [ "PATHEXT", ] as const +// Environment for the side process: the shared allowlist plus the one named +// credential variable and the side label. export function buildSideEnvironment( env: NodeJS.ProcessEnv, credentialEnv: string, side: "baseline" | "candidate", ): NodeJS.ProcessEnv { const child: NodeJS.ProcessEnv = {} - for (const name of SIDE_ENV_ALLOWLIST) { + for (const name of PROCESS_ENV_ALLOWLIST) { const value = env[name] if (value !== undefined) child[name] = value } @@ -98,3 +112,27 @@ export function buildSideEnvironment( child["CODING_EVALS_SIDE"] = side return child } + +// Environment for the worktree install and build processes. These children +// inherit neither the driver's secrets nor the provider credential. They get +// the shared process locations plus the explicitly approved pnpm variables a +// frozen-lockfile install and a build need to locate their store, cache, and +// registry. +const WORKTREE_ENV_ALLOWLIST = [ + ...PROCESS_ENV_ALLOWLIST, + "PNPM_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_STATE_HOME", + "npm_config_registry", +] as const + +export function buildWorktreeEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of WORKTREE_ENV_ALLOWLIST) { + const value = env[name] + if (value !== undefined) child[name] = value + } + return child +} diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 931b3dadda..891da30572 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -6,6 +6,8 @@ import { describe, it } from "node:test" import { buildContainerArgs, createContainerGradingRuntime, + EXEC_OUTPUT_MAX_BYTES, + exec, GRADER_IMAGE, parseDriverResult, type ContainerCli, @@ -318,3 +320,29 @@ describe("untrusted driver result validation", () => { assert.equal(parseDriverResult(`{"kind":"completed","pad":"${"x".repeat(70_000)}"}`), undefined) }) }) + +describe("bounded cli output capture", () => { + it("caps oversized stdout and stderr at the shared bound and keeps the head", { timeout: 30_000 }, async () => { + const outcome = await exec( + process.execPath, + ["-e", "process.stdout.write('a'.repeat(1000000)); process.stderr.write('b'.repeat(1000000))"], + 30_000, + ) + assert.equal(outcome.code, 0) + assert.equal(outcome.stdout.length, EXEC_OUTPUT_MAX_BYTES) + assert.equal(outcome.stderr.length, EXEC_OUTPUT_MAX_BYTES) + assert.ok(outcome.stdout.startsWith("a")) + assert.ok(outcome.stderr.startsWith("b")) + }) + + it("keeps short diagnostic output intact", { timeout: 30_000 }, async () => { + const outcome = await exec( + process.execPath, + ["-e", "process.stdout.write('sha256:digest'); process.stderr.write('no such container: gone')"], + 30_000, + ) + assert.equal(outcome.code, 0) + assert.equal(outcome.stdout, "sha256:digest") + assert.equal(outcome.stderr, "no such container: gone") + }) +}) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 98da8ea0c6..23f981e2ec 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -64,7 +64,16 @@ const probeExecutable = (candidate: ContainerExecutable): Promise => } }) -const exec = ( +// Shared bound for one CLI call's captured stdout and stderr. Digest, pull, +// and removal diagnostics are far smaller than this cap; the cap only bounds +// a pathological output stream, so captured output can never grow unbounded. +export const EXEC_OUTPUT_MAX_BYTES = 262_144 + +const appendBounded = (current: string, chunk: string): string => + current.length >= EXEC_OUTPUT_MAX_BYTES ? current : current + chunk.slice(0, EXEC_OUTPUT_MAX_BYTES - current.length) + +// Exported as a test seam. Production callers go through ContainerCli. +export const exec = ( file: string, args: readonly string[], timeoutMs: number, @@ -79,10 +88,10 @@ const exec = ( let stdout = "" let stderr = "" child.stdout?.on("data", (chunk) => { - stdout += String(chunk) + stdout = appendBounded(stdout, String(chunk)) }) child.stderr?.on("data", (chunk) => { - stderr += String(chunk) + stderr = appendBounded(stderr, String(chunk)) }) child.once("error", () => resolve({ code: "error", stdout, stderr })) child.once("close", (code, signal) => diff --git a/scripts/agent-evals/fs-atomic.test.ts b/scripts/agent-evals/fs-atomic.test.ts index c34d86f91b..c9e8415541 100644 --- a/scripts/agent-evals/fs-atomic.test.ts +++ b/scripts/agent-evals/fs-atomic.test.ts @@ -49,4 +49,26 @@ describe("atomic file publication", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it("preserves the previous file when the rename fails after a successful temp write", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const previous = path.join(dir, "report.json") + await writeFileAtomic(previous, "previous") + // The target path is a non-empty directory. The temp file is + // written successfully into the target's own directory, and only + // the rename fails. + const blocked = path.join(dir, "blocked-target") + await fs.mkdir(blocked) + await fs.writeFile(path.join(blocked, "occupied"), "x") + await assert.rejects(writeFileAtomic(blocked, "next")) + // The previous publication stays intact and no temp residue or + // partial content remains. + assert.equal(await fs.readFile(previous, "utf8"), "previous") + assert.deepEqual((await fs.readdir(dir)).sort(), ["blocked-target", "report.json"]) + assert.deepEqual(await fs.readdir(blocked), ["occupied"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) }) diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index 376d07b540..5d780473c5 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -846,9 +846,10 @@ describe("agent fixture controller", () => { }) assert.equal(trace.accounting.completeness.status, "complete") assert.deepEqual(trace.accounting.completeness.reasons, []) - // The drain is bound to the accepted-completion event. Without it, no - // drain window runs. - assert.equal(trace.execution.timings.usageDrainMs, undefined) + // The stop after the final completion message gets the same bounded + // usage drain a confirmed completion gets. The fixture drain window + // is 5_000 ms. + assert.equal(trace.execution.timings.usageDrainMs, 5_000) const serialized = JSON.stringify(trace) assert.ok(serialized.includes("pr1909-smoke-topology-synthetic")) assert.ok(!serialized.includes(CREDENTIAL)) @@ -1052,6 +1053,93 @@ describe("agent fixture controller", () => { assert.equal(result.teardown.status, "ok") }) + it("drains bounded usage after an observer stop on the final completion message", async () => { + registerTestFixture("controller-stop-drain", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 500, + teardownStepMs: 5_000, + }, + }) + let lateSent = false + const f = harness("no-events", { + onPause: (count) => { + if (count === 1) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "synthetic final answer", + }, + }, + ], + }) + // A usage snapshot lands inside the drain window that follows + // the stop. The bounded drain must capture it. + if (count === 2 && !lateSent) { + lateSent = true + f.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 40, totalTokensOut: 20, totalCost: 0, contextTokens: 30 }, + {}, + ], + }) + } + }, + }) + let completionOutput = false + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-stop-drain", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => { + if ( + event.eventName === RooCodeEventName.Message && + event.payload[0].message.say === "completion_result" + ) + completionOutput = true + }, + shouldStop: () => completionOutput, + }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStopAfterCompletionMessage") + const trace = writtenTrace(f) + // The stop path drains for the fixture's bounded window and captures + // the late usage snapshot. + assert.equal(trace.execution.timings.usageDrainMs, 500) + assert.equal(trace.accounting.usage.snapshotsObserved, 1) + assert.equal(trace.accounting.usage.lastSnapshot?.source, "taskTokenUsageUpdated") + assert.equal(trace.accounting.usage.lastSnapshot?.tokensIn, 40) + assert.equal(trace.accounting.completeness.status, "complete") + assert.deepEqual(trace.accounting.completeness.reasons, []) + }) + + it("skips the usage drain for a plain observer stop without the completion message", async () => { + registerTestFixture("controller-plain-stop") + const f = harness("success") + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-plain-stop", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.completionMessageSeen, false) + // Without the final completion message, no drain window runs. + assert.equal(writtenTrace(f).execution.timings.usageDrainMs, undefined) + }) + for (const hostExited of [false, true]) { it(`holds teardown only while the completed host is live (exited=${hostExited})`, async () => { const fixtureName = `controller-hold-${hostExited}` diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index a9dafea6d3..d5fa2447c2 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -306,7 +306,12 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp ) runSessionLost = sessionLost timings.taskRunMs = runtime.now() - runStart - if (completedSeen && !runSessionLost) { + const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true + // The model's final completion message already produced + // reportable usage, so a stop issued after it gets the + // same bounded drain a confirmed completion gets. A + // plain stop, a timeout, and a session loss get none. + if ((completedSeen || (stopped && completionMessageSeen)) && !runSessionLost) { // Fixed bound; ends early only on session loss. const drainStart = runtime.now() await until(() => false, fixture.limits.usageDrainMs) @@ -316,7 +321,6 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp // accounting must carry the caveat. lostDuringDrain = sessionLost } - const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true if (completedSeen) { // A confirmed accepted completion outranks a stop // request or a loss observed in the same window. diff --git a/scripts/agent-evals/side-process-guard.test.ts b/scripts/agent-evals/side-process-guard.test.ts new file mode 100644 index 0000000000..bd5b8106e7 --- /dev/null +++ b/scripts/agent-evals/side-process-guard.test.ts @@ -0,0 +1,266 @@ +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { runBoundedPreparation } from "./coding-compare.ts" +import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" + +// Credential-free process fixtures. Each child is a bare node process in its +// own detached group, the same topology the comparison driver creates. +const spawnDetachedChild = (script: string) => { + const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "ignore"], detached: true }) + const spawned = new Promise((resolve, reject) => { + child.once("spawn", resolve) + child.once("error", reject) + }) + const exited = new Promise((resolve) => child.once("exit", () => resolve())) + // Resolves when the child reports readiness, so a signal cannot land + // before the child finished its own setup. + const ready = new Promise((resolve, reject) => { + child.stdout?.once("data", () => resolve()) + child.once("error", reject) + child.once("exit", () => reject(new Error("child exited before reporting readiness"))) + }) + // Tests that never await readiness still keep the rejection handled. + ready.catch(() => {}) + return { child, spawned, ready, exited } +} + +describe("side process guard", () => { + it("resolves without work when nothing is tracked", async () => { + const guard = createSideProcessGuard(1_000) + assert.equal(await guard.interrupt(), "none") + // Repeated interrupts are no-ops, never a double kill. + assert.equal(await guard.interrupt(), "none") + }) + + it("terminates a detached child group on SIGTERM", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(10_000) + const { child, spawned, exited } = spawnDetachedChild("setInterval(() => {}, 1000000)") + try { + await spawned + guard.track(child.pid) + assert.equal(await guard.interrupt(), "terminated") + // The detached child must actually be gone, not orphaned. + await Promise.race([ + exited, + new Promise((_, reject) => setTimeout(() => reject(new Error("child survived the interrupt")), 20_000)), + ]) + } finally { + child.kill("SIGKILL") + } + }) + + it("escalates to SIGKILL when the child ignores SIGTERM", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(300) + const { child, ready, exited } = spawnDetachedChild( + "process.on('SIGTERM', () => {}); process.stdout.write('ready'); setInterval(() => {}, 1000000)", + ) + try { + // The signal must not land before the child installed its ignore + // handler. + await ready + guard.track(child.pid) + assert.equal(await guard.interrupt(), "killed") + await Promise.race([ + exited, + new Promise((_, reject) => setTimeout(() => reject(new Error("child survived SIGKILL")), 20_000)), + ]) + } finally { + child.kill("SIGKILL") + } + }) + + it("runs the escalation once across repeated calls", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(10_000) + const { child, spawned, exited } = spawnDetachedChild("setInterval(() => {}, 1000000)") + try { + await spawned + guard.track(child.pid) + const first = guard.interrupt() + // A second signal must not start a second kill sequence. + assert.equal(await guard.interrupt(), "none") + assert.equal(await first, "terminated") + await exited + } finally { + child.kill("SIGKILL") + } + }) + + it("keeps the escalation running when the tracked child exits mid-wait", { timeout: 30_000 }, async () => { + // The runner untracks the direct child as soon as it settles. The + // group id is snapshotted at interrupt time, so that untrack cannot + // cut the wait short while a group member still writes. + const guard = createSideProcessGuard(500) + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "guard-escalation-test-")) + const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") + const grandchildPidPath = path.join(dir, "grandchild-pid") + const grandchildReadyPath = path.join(dir, "grandchild-ready") + const grandchildScript = `const fs = require('node:fs'); process.on('SIGTERM', () => {}); fs.writeFileSync(${JSON.stringify(grandchildReadyPath)}, 'ready'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + const fixture = spawnDetachedChild( + `const fs = require('node:fs'); + const { spawn } = require('node:child_process'); + const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' }); + fs.writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid)); + process.stdout.write('ready'); + setTimeout(() => process.exit(0), 100);`, + ) + try { + // The signal must not land before the grandchild installed its + // ignore handler, so both fixture processes report readiness. + await Promise.all([fixture.ready, waitForFile(grandchildReadyPath)]) + const childPid = fixture.child.pid + assert.ok(childPid !== undefined) + guard.track(childPid) + const interrupt = guard.interrupt() + // The direct child settles and the runner untracks it while the + // grandchild, which ignores SIGTERM, still lives. + await fixture.exited + guard.track(undefined) + assert.equal(await interrupt, "killed") + // The SIGKILL escalation reached the surviving group member: its + // heartbeat went silent. + await assertHeartbeatStopped(grandchildHeartbeat) + } finally { + fixture.child.kill("SIGKILL") + const grandchildPid = await fs.readFile(grandchildPidPath, "utf8").then(Number, () => undefined) + if (grandchildPid !== undefined) { + try { + process.kill(grandchildPid, "SIGKILL") + } catch { + /* Already gone. */ + } + } + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +// Confirms the fixture writer stopped appending, so no child is still +// writing when the driver's cleanup would run. +const assertHeartbeatStopped = async (filePath: string): Promise => { + const sizeBefore = await fs.stat(filePath).then( + (entry) => entry.size, + () => -1, + ) + await delay(400) + const sizeAfter = await fs.stat(filePath).then( + (entry) => entry.size, + () => -1, + ) + assert.equal(sizeAfter, sizeBefore) +} + +// Polls until the file exists, so a signal cannot land before the fixture +// children finished their own setup. +const waitForFile = async (filePath: string): Promise => { + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + if ( + await fs.access(filePath).then( + () => true, + () => false, + ) + ) + return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`fixture file never appeared: ${filePath}`) +} + +describe("bounded preparation children", () => { + // Regression: a SIGTERM aimed at the driver process alone used to leave + // install and build children running, because only the coding side was + // tracked. The test replays the driver wiring credential-free: one guard, + // one handler that arms one escalation, and a cleanup that waits for it. + it("stops a tracked preparation group when only the parent gets SIGTERM", { timeout: 30_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "bounded-prep-test-")) + const guard = createSideProcessGuard(2_000) + let interruptSignal: NodeJS.Signals | undefined + let interruptKill: Promise | undefined + const onInterruptSignal = (signal: NodeJS.Signals): void => { + if (interruptSignal !== undefined) return + interruptSignal = signal + interruptKill = guard.interrupt() + } + process.on("SIGTERM", onInterruptSignal) + // The preparation child leads its own group and spawns a grandchild + // into it, the same topology a real pnpm install creates. Both keep + // writing a heartbeat file, so a surviving member cannot hide. + const readyPath = path.join(dir, "ready") + const childPidPath = path.join(dir, "child-pid") + const childHeartbeat = path.join(dir, "child-heartbeat") + const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") + const grandchildScript = `const fs = require('node:fs'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + const childScript = [ + "const fs = require('node:fs')", + "const { spawn } = require('node:child_process')", + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid))`, + `const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' })`, + `fs.writeFileSync(${JSON.stringify(readyPath)}, 'ready')`, + `setInterval(() => fs.appendFileSync(${JSON.stringify(childHeartbeat)}, 'x'), 50)`, + ].join("; ") + const bounded = runBoundedPreparation( + guard, + () => interruptSignal !== undefined, + process.execPath, + ["-e", childScript], + dir, + 60_000, + ) + try { + await waitForFile(readyPath) + // The signal targets this test process alone. The child group + // receives nothing from this kill call. + process.kill(process.pid, "SIGTERM") + await assert.rejects(bounded) + // The escalation finishes before any cleanup runs, and the whole + // group, grandchild included, is gone by then. + await interruptKill + assert.equal(interruptSignal, "SIGTERM") + // The whole group went silent before the cleanup ran, so nothing + // survives to write into the removed tree. + await assertHeartbeatStopped(childHeartbeat) + await assertHeartbeatStopped(grandchildHeartbeat) + await fs.rm(dir, { recursive: true, force: true }) + } finally { + process.removeListener("SIGTERM", onInterruptSignal) + const childPid = await fs.readFile(childPidPath, "utf8").then(Number, () => undefined) + if (childPid !== undefined) { + try { + process.kill(-childPid, "SIGKILL") + } catch { + /* The group is already gone. */ + } + } + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("refuses to start a preparation child after the interrupt", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(100) + // A signal that landed between children tracked nothing; the next + // spawn must refuse instead of escaping the stop. + assert.equal(await guard.interrupt(), "none") + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "bounded-prep-test-")) + try { + await assert.rejects( + runBoundedPreparation( + guard, + () => true, + process.execPath, + ["-e", "setInterval(() => {}, 1000000)"], + dir, + 1_000, + ), + /interrupted before/, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) diff --git a/scripts/agent-evals/side-process-guard.ts b/scripts/agent-evals/side-process-guard.ts new file mode 100644 index 0000000000..8745273b8e --- /dev/null +++ b/scripts/agent-evals/side-process-guard.ts @@ -0,0 +1,70 @@ +// Interrupt handling for the comparison driver's detached children. Each +// bounded child, the preparation steps and the coding side alike, runs in its +// own process group, so a terminal signal aimed at the driver alone never +// reaches it. When the driver receives SIGINT or SIGTERM, this guard +// terminates the tracked group: SIGTERM first, a bounded grace, then SIGKILL +// for anything that remained. Repeated calls are a no-op, so several signals +// can share one guard without double cleanup. +export type SideKillOutcome = "terminated" | "killed" | "none" + +export interface SideProcessGuard { + // Tracks the active detached child process id. undefined clears the + // tracking, for example after the child exited on its own. + track(pid: number | undefined): void + // Terminates the tracked process group. Resolves after the group is gone + // ("terminated"), after the SIGKILL escalation fired ("killed"), or at + // once when nothing is tracked or a kill already ran ("none"). + interrupt(): Promise +} + +const groupAlive = (pid: number): boolean => { + try { + process.kill(-pid, 0) + return true + } catch { + return false + } +} + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +export const createSideProcessGuard = (graceMs: number): SideProcessGuard => { + let pid: number | undefined + let interrupted = false + const signalGroup = (group: number, signal: NodeJS.Signals): void => { + try { + process.kill(-group, signal) + } catch { + // The group may already be gone; fall back to the direct child. + try { + process.kill(group, signal) + } catch { + /* The child already exited. */ + } + } + } + return { + track: (childPid) => { + pid = childPid + }, + interrupt: async (): Promise => { + // The group id is snapshotted, so an untrack that races the + // escalation, for example the tracked child's own exit handler, + // cannot cut the wait short while descendants still run. + const group = pid + if (interrupted || group === undefined) { + interrupted = true + return "none" + } + interrupted = true + signalGroup(group, "SIGTERM") + const deadline = Date.now() + graceMs + while (Date.now() < deadline) { + if (!groupAlive(group)) return "terminated" + await delay(25) + } + signalGroup(group, "SIGKILL") + return "killed" + }, + } +} diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts index d27a0bb229..99297a1179 100644 --- a/scripts/solheim-smoke/run.test.ts +++ b/scripts/solheim-smoke/run.test.ts @@ -167,7 +167,7 @@ describe("provider smoke orchestration", () => { assert.equal(smokeExitCode(verdict), 0) // The teardown hold runs once after the completion message; no other // pause fires in this scenario, so the duration is exactly the hold. - assert.equal(verdict.durationMs, SMOKE_TEARDOWN_HOLD_MS) + assert.equal(verdict.durationMs, 5_000 + SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(verdict.gates, { extensionActivated: true, ipcConnected: true, @@ -214,8 +214,8 @@ describe("provider smoke orchestration", () => { const f = fixture("success", { stuck: true }) const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "TEARDOWN_FAILED") - // The 3 second completion hold plus three 5 second termination windows. - assert.equal(verdict.durationMs, 15_000 + SMOKE_TEARDOWN_HOLD_MS) + // The bounded usage drain, completion hold, and three termination windows all run. + assert.equal(verdict.durationMs, 20_000 + SMOKE_TEARDOWN_HOLD_MS) assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) assert.equal(verdict.gates.childExited, false) assert.equal(f.cleaned, false) From bb1758c9f1f999a43d1676083d312730cdc25b43 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 24 Oct 2026 03:49:49 +0000 Subject: [PATCH 04/22] fix(evals): pass test fixture paths as arguments instead of embedding code --- .../agent-evals/side-process-guard.test.ts | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/scripts/agent-evals/side-process-guard.test.ts b/scripts/agent-evals/side-process-guard.test.ts index bd5b8106e7..adba5a2b64 100644 --- a/scripts/agent-evals/side-process-guard.test.ts +++ b/scripts/agent-evals/side-process-guard.test.ts @@ -9,8 +9,11 @@ import { createSideProcessGuard, type SideKillOutcome } from "./side-process-gua // Credential-free process fixtures. Each child is a bare node process in its // own detached group, the same topology the comparison driver creates. -const spawnDetachedChild = (script: string) => { - const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "ignore"], detached: true }) +const spawnDetachedChild = (script: string, args: readonly string[] = []) => { + const child = spawn(process.execPath, ["-e", script, ...args], { + stdio: ["ignore", "pipe", "ignore"], + detached: true, + }) const spawned = new Promise((resolve, reject) => { child.once("spawn", resolve) child.once("error", reject) @@ -98,14 +101,19 @@ describe("side process guard", () => { const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") const grandchildPidPath = path.join(dir, "grandchild-pid") const grandchildReadyPath = path.join(dir, "grandchild-ready") - const grandchildScript = `const fs = require('node:fs'); process.on('SIGTERM', () => {}); fs.writeFileSync(${JSON.stringify(grandchildReadyPath)}, 'ready'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + // Fixture sources stay static. Host paths and the grandchild source + // travel as argv, so no host value is embedded in generated code. + const grandchildScript = + "const fs = require('node:fs'); process.on('SIGTERM', () => {}); const [heartbeatPath, readyPath] = process.argv.slice(1); fs.writeFileSync(readyPath, 'ready'); setInterval(() => fs.appendFileSync(heartbeatPath, 'x'), 50)" const fixture = spawnDetachedChild( `const fs = require('node:fs'); const { spawn } = require('node:child_process'); - const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' }); - fs.writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid)); + const [grandchildPidPath, grandchildReadyPath, grandchildHeartbeatPath, grandchildScript] = process.argv.slice(1); + const grandchild = spawn(process.execPath, ['-e', grandchildScript, grandchildHeartbeatPath, grandchildReadyPath], { stdio: 'ignore' }); + fs.writeFileSync(grandchildPidPath, String(grandchild.pid)); process.stdout.write('ready'); setTimeout(() => process.exit(0), 100);`, + [grandchildPidPath, grandchildReadyPath, grandchildHeartbeat, grandchildScript], ) try { // The signal must not land before the grandchild installed its @@ -195,20 +203,27 @@ describe("bounded preparation children", () => { const childPidPath = path.join(dir, "child-pid") const childHeartbeat = path.join(dir, "child-heartbeat") const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") - const grandchildScript = `const fs = require('node:fs'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + // Fixture sources stay static. Host paths and the grandchild source + // travel as argv, so no host value is embedded in generated code. + const grandchildScript = + "const fs = require('node:fs'); const [heartbeatPath] = process.argv.slice(1); setInterval(() => fs.appendFileSync(heartbeatPath, 'x'), 50)" const childScript = [ "const fs = require('node:fs')", "const { spawn } = require('node:child_process')", - `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid))`, - `const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' })`, - `fs.writeFileSync(${JSON.stringify(readyPath)}, 'ready')`, - `setInterval(() => fs.appendFileSync(${JSON.stringify(childHeartbeat)}, 'x'), 50)`, + "const [childPidPath, readyPath, childHeartbeatPath, grandchildHeartbeatPath, grandchildScript] = process.argv.slice(1)", + "fs.writeFileSync(childPidPath, String(process.pid))", + "const grandchild = spawn(process.execPath, ['-e', grandchildScript, grandchildHeartbeatPath], { stdio: 'ignore' })", + "fs.writeFileSync(readyPath, 'ready')", + "setInterval(() => fs.appendFileSync(childHeartbeatPath, 'x'), 50)", ].join("; ") + // Security regression guard: generated fixture code must not embed + // any host path. Every host value travels as argv instead. + assert.ok(!childScript.includes(dir)) const bounded = runBoundedPreparation( guard, () => interruptSignal !== undefined, process.execPath, - ["-e", childScript], + ["-e", childScript, childPidPath, readyPath, childHeartbeat, grandchildHeartbeat, grandchildScript], dir, 60_000, ) From 12794d70fd41bf7108f907dc60d586b4ce0d21aa Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 24 Oct 2026 04:20:08 +0000 Subject: [PATCH 05/22] fix(evals): enforce isolated grading and safe interruption and result writes --- scripts/agent-evals/README.md | 4 + .../agent-evals/coding-compare-script.test.ts | 85 +++++++++ scripts/agent-evals/coding-compare.mts | 9 +- scripts/agent-evals/coding-fixtures.test.ts | 2 +- scripts/agent-evals/coding-grader.ts | 2 +- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/coding-report.test.ts | 11 ++ scripts/agent-evals/coding-side-flow.test.ts | 178 ++++++++++++++++++ scripts/agent-evals/coding-side-flow.ts | 81 ++++++++ scripts/agent-evals/coding-side.mts | 161 ++++++++-------- scripts/agent-evals/coding-side.test.ts | 31 +++ scripts/agent-evals/container-runtime.test.ts | 63 ++++++- scripts/agent-evals/container-runtime.ts | 14 +- scripts/agent-evals/grader-driver.mjs | 22 ++- scripts/agent-evals/grader-driver.test.ts | 80 ++++++++ .../agent-evals/side-process-guard.test.ts | 61 +++++- scripts/agent-evals/side-process-guard.ts | 42 ++++- 17 files changed, 747 insertions(+), 101 deletions(-) create mode 100644 scripts/agent-evals/coding-compare-script.test.ts create mode 100644 scripts/agent-evals/coding-side-flow.test.ts create mode 100644 scripts/agent-evals/coding-side-flow.ts create mode 100644 scripts/agent-evals/grader-driver.test.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index d24043fa7e..570c1744b2 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -135,6 +135,8 @@ release process: a maintainer runs it locally or on demand. - The driver's result file is untrusted data: it is written inside the container where agent-written test code executes. The host reads it with a size bound and validates shape, bounds, and internal consistency. An invalid or inconsistent result is a `grading failure`, never a verdict. - Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. - `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. +- The live `coding-evals:side` and `coding-evals:compare` entry points reject `--allow-unisolated-grading` instead of ignoring it. A recorded comparison always grades in the container runtime. +- The driver publishes its result file atomically: the content lands in a unique same-directory temp file and is renamed into place. A failed write or rename removes the temp file, and the host treats a missing or invalid result as a grading failure. ### Comparisons @@ -164,6 +166,8 @@ Artifacts are published atomically: content lands in a same-directory temp file A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and the whole `artifacts` tree, so stale results from an earlier rerun can never survive. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. +The side process arms one flag when it receives SIGINT or SIGTERM. The flag starts no further repetition and blocks the completion marker, so an interrupted side exits with the conventional signal code and publishes nothing. + The comparison reports are published as one generation. `comparison-report.json` and `comparison-report.md` carry the report's `generatedAt` stamp, and `comparison-report.complete` holds that stamp; it is written last. Consumers must require the marker and match its stamp against the report files. `readPublishedComparisonReport` enforces this: it returns the report only when the marker exists and its stamp matches both files. The marker is removed before publication starts, so a crash between the two file publications leaves a detectable incomplete generation instead of a mixed one. A thrown failure rolls both files back; a failed comparison leaves no report files. ### Commands diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts new file mode 100644 index 0000000000..05b68288d5 --- /dev/null +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { describe, it } from "node:test" + +const execFileAsync = promisify(execFile) +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +interface RunOutcome { + code: number + stdout: string + stderr: string +} + +// Spawns the real comparison driver with a minimal, credential-free +// environment. The argument gate runs before the trust policy loads, so the +// rejection happens before any worktree, child, or report exists. +const runCompareScript = async (args: string[], env: NodeJS.ProcessEnv): Promise => { + try { + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), ...args], + { + cwd: repoRoot, + timeout: 120_000, + killSignal: "SIGKILL", + env, + }, + ) + return { code: 0, stdout, stderr: "" } + } catch (error) { + const failure = error as { code?: number | string; stdout?: string; stderr?: string; killed?: boolean } + if (failure.killed) throw new Error("compare script hung and hit the test kill timer") + return { + code: typeof failure.code === "number" ? failure.code : 1, + stdout: failure.stdout ?? "", + stderr: failure.stderr ?? "", + } + } +} + +const minimalEnv = (): NodeJS.ProcessEnv => ({ + PATH: process.env["PATH"] ?? "/usr/bin:/bin", + HOME: process.env["HOME"] ?? os.homedir(), +}) + +describe("coding compare isolation gate", () => { + it("rejects --allow-unisolated-grading before any worktree or report exists", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-test-")) + try { + const trustFile = path.join(dir, "trust-policy.json") + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [] })) + const out = path.join(dir, "out") + const outcome = await runCompareScript( + [ + "--baseline", + "a".repeat(40), + "--candidate", + "b".repeat(40), + "--reps", + "1", + "--out", + out, + "--trust-file", + trustFile, + "--allow-unisolated-grading", + ], + minimalEnv(), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /--allow-unisolated-grading is not accepted here/) + // The rejection happens before any report generation exists, so + // the comparison can never publish host-graded artifacts. + for (const name of ["comparison-report.json", "comparison-report.md", "comparison-report.complete"]) { + await assert.rejects(fs.access(path.join(out, name))) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index cf3863f041..e201fbc431 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -73,11 +73,16 @@ const readArgs = (): { reps: number out: string credentialEnv: string - allowUnisolatedGrading: boolean trustFile: string sideTimeoutMs: number } => { const args = process.argv.slice(2) + // The local subprocess runtime has no container isolation: agent-written + // grading code would execute on the host filesystem. A comparison always + // grades in the container runtime, so the dangerous option is rejected + // loudly here and never forwarded to a side. + if (args.includes("--allow-unisolated-grading")) + throw new Error("--allow-unisolated-grading is not accepted here: a comparison grades in the container runtime") const read = (name: string): string | undefined => { const index = args.indexOf(`--${name}`) return index >= 0 ? args[index + 1] : undefined @@ -105,7 +110,6 @@ const readArgs = (): { reps, out: path.resolve(out), credentialEnv, - allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), trustFile: path.resolve(trustFile), sideTimeoutMs: sideTimeoutMinutes * 60_000, } @@ -167,7 +171,6 @@ const runSide = async ( inputs.credentialEnv, "--evaluated-root", worktree, - ...(inputs.allowUnisolatedGrading ? ["--allow-unisolated-grading"] : []), ], { cwd: root, diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index 48aa0f7222..66f2daafa3 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -91,6 +91,6 @@ describe("coding fixture suite", () => { assert.throws(() => codingBundle("unknown-fixture")) assert.ok(codingSuiteDigest().length === 64) assert.equal(CODING_SUITE_ID, "coding-basic-v1") - assert.equal(GRADER_VERSION, "coding-grader-v2") + assert.equal(GRADER_VERSION, "coding-grader-v3") }) }) diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts index 0f9800ee32..33039f4426 100644 --- a/scripts/agent-evals/coding-grader.ts +++ b/scripts/agent-evals/coding-grader.ts @@ -9,7 +9,7 @@ import type { Measured } from "./trace.ts" // The grader version is a pinned comparison input. Any change to the driver, // the image, or the verdict rules must bump it. -export const GRADER_VERSION = "coding-grader-v2" +export const GRADER_VERSION = "coding-grader-v3" export interface GraderResources { timeoutMs: number diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index 0c89c339ac..1593b4c245 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -66,7 +66,7 @@ describe("coding graders against reference solutions", () => { } it("pins one grader version across the suite", () => { - assert.equal(GRADER_VERSION, "coding-grader-v2") + assert.equal(GRADER_VERSION, "coding-grader-v3") assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) }) }) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 07a7617a72..7dee41eba0 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -214,6 +214,17 @@ describe("side result validation", () => { assert.equal(validated.pins.commit, "a".repeat(40)) }) + it("accepts a well-formed run with object metrics and keeps it", () => { + const run = { fixtureName: "clamp-range", repIndex: 0, metrics: metrics("clamp-range", 0, "correct", 0.1) } + const validated = validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { runs: [run] }), + "baseline", + "a".repeat(40), + ) + assert.equal(validated.runs.length, 1) + assert.deepEqual(validated.runs[0], run) + }) + it("rejects a result that records the wrong side or a different commit", () => { assert.throws( () => validateSideResultFile(validatedFile("candidate", "a".repeat(40)), "baseline", "a".repeat(40)), diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts new file mode 100644 index 0000000000..85a13c3f8f --- /dev/null +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { executeSideFlow, type SideRepetition } from "./coding-side-flow.ts" +import type { CodingRunRecord } from "./coding-run.ts" +import { buildRunMetrics } from "./coding-metrics.ts" +import type { AgentTrace } from "./trace.ts" +import type { SidePins } from "./coding-pins.ts" + +const plan = (count: number, fixtureName = "clamp-range"): SideRepetition[] => + Array.from({ length: count }, (_, repIndex) => ({ fixtureName, fixtureVersion: "1.0.0", repIndex })) + +const record = (repetition: SideRepetition, withTrace = false): CodingRunRecord => ({ + fixtureName: repetition.fixtureName, + fixtureVersion: repetition.fixtureVersion, + repIndex: repetition.repIndex, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + // The flow only checks trace presence before it writes the artifact, so + // an empty object stands in for the metadata-only trace here. + trace: withTrace ? ({} as AgentTrace) : undefined, + metrics: buildRunMetrics({ + fixtureName: repetition.fixtureName, + repIndex: repetition.repIndex, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + modelIdentifier: { available: true, value: "test-model" }, + }), +}) + +const pins = (): SidePins => ({ + side: "candidate", + commit: "a".repeat(40), + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "test-model", + graderVersion: "coding-grader-v2", + graderImageDigest: "unavailable", + runnerEnvironment: { os: "linux", nodeVersion: process.version, vscodeTarget: "test" }, +}) + +const flow = (inputs: { + repetitions: readonly SideRepetition[] + out: string + isInterrupted: () => boolean + runRepetition: (repetition: SideRepetition) => Promise +}) => + executeSideFlow({ + ...inputs, + buildResult: (runs) => ({ pins: pins(), runs }), + }) + +describe("side execution flow", () => { + it("runs every repetition and publishes the completion marker last", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const executed: string[] = [] + const repetitions = plan(2) + const outcome = await flow({ + repetitions, + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + executed.push(`${repetition.fixtureName}#${repetition.repIndex}`) + return record(repetition, true) + }, + }) + assert.equal(outcome.completed, true) + assert.deepEqual(executed, ["clamp-range#0", "clamp-range#1"]) + // The marker exists and records every run. + const marker = JSON.parse(await fs.readFile(path.join(dir, "side-results.json"), "utf8")) as { + runs: { repIndex: number }[] + } + assert.deepEqual( + marker.runs.map((run) => run.repIndex), + [0, 1], + ) + // Per-run artifacts exist for the records with a trace. + const artifact = JSON.parse( + await fs.readFile(path.join(dir, "artifacts", "clamp-range-rep0", "run-record.json"), "utf8"), + ) as { repIndex: number } + assert.equal(artifact.repIndex, 0) + // The atomic publication leaves no temp residue. + assert.deepEqual((await fs.readdir(path.join(dir, "artifacts", "clamp-range-rep0"))).sort(), [ + "run-record.json", + ]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("stops before the next repetition and publishes nothing when interrupted mid-flow", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const repetitions = plan(3) + let interrupted = false + const executed: number[] = [] + const outcome = await flow({ + repetitions, + out: dir, + isInterrupted: () => interrupted, + runRepetition: async (repetition) => { + executed.push(repetition.repIndex) + // The signal lands during the first repetition. + interrupted = true + return record(repetition, true) + }, + }) + assert.equal(outcome.completed, false) + // Exactly the in-flight repetition ran; no later repetition started. + assert.deepEqual(executed, [0]) + assert.equal(outcome.runs.length, 1) + // The interrupted side publishes neither artifacts nor the marker. + await assert.rejects(fs.access(path.join(dir, "side-results.json"))) + await assert.rejects(fs.access(path.join(dir, "artifacts"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("does not publish when the interrupt lands after the last repetition", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const repetitions = plan(2) + let completedRepetitions = 0 + const outcome = await flow({ + repetitions, + out: dir, + // The flag is read between the loop and the publication, so it + // goes live once every repetition finished. + isInterrupted: () => completedRepetitions === repetitions.length, + runRepetition: async (repetition) => { + completedRepetitions += 1 + return record(repetition, false) + }, + }) + assert.equal(outcome.completed, false) + assert.equal(outcome.runs.length, 2) + await assert.rejects(fs.access(path.join(dir, "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("keeps running after a wiring error and records it as an infrastructure failure", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const repetitions = plan(2) + const executed: number[] = [] + const outcome = await flow({ + repetitions, + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + executed.push(repetition.repIndex) + if (repetition.repIndex === 0) throw new Error("wiring broke") + return record(repetition, false) + }, + }) + assert.equal(outcome.completed, true) + assert.deepEqual(executed, [0, 1]) + const failed = outcome.runs[0] + assert.ok(failed) + assert.deepEqual(failed.outcome, { outcome: "infrastructure-failure", reason: "runTraceMissing" }) + assert.equal(failed.fixtureName, "clamp-range") + assert.equal(failed.repIndex, 0) + // A record without a trace writes no artifact; the marker still + // lists both runs. + const marker = JSON.parse(await fs.readFile(path.join(dir, "side-results.json"), "utf8")) as { + runs: unknown[] + } + assert.equal(marker.runs.length, 2) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts new file mode 100644 index 0000000000..cba0deaf5a --- /dev/null +++ b/scripts/agent-evals/coding-side-flow.ts @@ -0,0 +1,81 @@ +// The side execution loop: one repetition plan, interruption gates before +// each repetition and before publication, and the completion-marker +// publication. The live side wires its fixture runner into it; tests inject +// synthetic repetitions, so the gating and publication order is testable +// without a credential, a host, or a provider. +import fs from "node:fs/promises" +import path from "node:path" +import type { SideResultFile } from "./coding-compare.ts" +import type { CodingRunRecord } from "./coding-run.ts" +import { classifyRunOutcome } from "./coding-outcomes.ts" +import { buildRunMetrics } from "./coding-metrics.ts" +import { writeJsonAtomic } from "./fs-atomic.ts" + +export interface SideRepetition { + fixtureName: string + fixtureVersion: string + repIndex: number +} + +export interface SideFlowInputs { + // The full repetition plan of this invocation, in execution order. + repetitions: readonly SideRepetition[] + out: string + // True once a terminal signal arrived. The loop checks it before each + // repetition and again before publication. + isInterrupted: () => boolean + // Runs one repetition. The substrate never throws for task failures; a + // throw is a wiring error and becomes an infrastructure-failure record. + runRepetition: (repetition: SideRepetition) => Promise + // Builds the side result that publication writes as the completion + // marker. May be async, so the pins can describe what actually executed. + buildResult: (runs: CodingRunRecord[]) => SideResultFile | Promise +} + +export interface SideFlowOutcome { + runs: CodingRunRecord[] + // True when every repetition finished and the completion marker was + // published. False when an interrupt stopped the side; no marker and no + // artifacts exist in that case. + completed: boolean +} + +export async function executeSideFlow(inputs: SideFlowInputs): Promise { + const runs: CodingRunRecord[] = [] + for (const repetition of inputs.repetitions) { + // An interrupt stops the loop before the next repetition starts. The + // in-flight repetition is left to the substrate's own teardown. + if (inputs.isInterrupted()) return { runs, completed: false } + try { + runs.push(await inputs.runRepetition(repetition)) + } catch { + // The substrate never throws for task failures. This guards wiring + // errors only, and records them as infrastructure failures. + const outcome = classifyRunOutcome({ hasTrace: false, finalWorkspaceAvailable: false }) + runs.push({ + fixtureName: repetition.fixtureName, + fixtureVersion: repetition.fixtureVersion, + repIndex: repetition.repIndex, + outcome, + metrics: buildRunMetrics({ + fixtureName: repetition.fixtureName, + repIndex: repetition.repIndex, + outcome, + modelIdentifier: { available: false, reason: "runFailed" }, + }), + }) + } + } + // The completion marker must never exist for an interrupted side, so the + // publication stays gated even when the loop itself finished. + if (inputs.isInterrupted()) return { runs, completed: false } + // Per-run artifacts are published atomically as each record exists. + for (const record of runs) { + if (!record.trace) continue + const artifactDir = path.join(inputs.out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) + await fs.mkdir(artifactDir, { recursive: true }) + await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) + } + await writeJsonAtomic(path.join(inputs.out, "side-results.json"), await inputs.buildResult(runs)) + return { runs, completed: true } +} diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 8b6675d688..e07b52b113 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -10,19 +10,16 @@ import path from "node:path" import { fileURLToPath } from "node:url" import { createHostRuntime } from "./launcher.ts" import { createContainerGradingRuntime } from "./container-runtime.ts" -import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" -import { runCodingFixtureOnce, type CodingRunRecord } from "./coding-run.ts" +import { runCodingFixtureOnce } from "./coding-run.ts" import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from "./coding-suite.ts" import { GRADER_VERSION } from "./coding-grader.ts" import { digestValue } from "./coding-digest.ts" import type { SidePins } from "./coding-pins.ts" import { invalidateSideArtifacts } from "./coding-compare.ts" -import type { SideResultFile } from "./coding-compare.ts" import { assertCommitsApproved, assertCredentialEnvAllowed, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" -import { classifyRunOutcome } from "./coding-outcomes.ts" -import { buildRunMetrics } from "./coding-metrics.ts" -import { writeJsonAtomic } from "./fs-atomic.ts" +import { executeSideFlow, type SideRepetition } from "./coding-side-flow.ts" +import { createInterruptFlag } from "./side-process-guard.ts" const thisRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -30,11 +27,16 @@ const readArgs = (): { reps: number out: string credentialEnv: string - allowUnisolatedGrading: boolean evaluatedRoot: string trustFile: string } => { const args = process.argv.slice(2) + // The local subprocess runtime has no container isolation: agent-written + // grading code would execute on the host filesystem. A live side always + // grades in the container runtime, so the dangerous option is rejected + // loudly instead of ignored. + if (args.includes("--allow-unisolated-grading")) + throw new Error("--allow-unisolated-grading is not accepted here: a live side grades in the container runtime") const read = (name: string): string | undefined => { const index = args.indexOf(`--${name}`) return index >= 0 ? args[index + 1] : undefined @@ -51,7 +53,6 @@ const readArgs = (): { reps: Number(read("reps") ?? "3"), out: path.resolve(out), credentialEnv, - allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), // Repository root of the extension under test. The trusted harness // code drives the run; only the extension host loads this tree. evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), @@ -82,7 +83,7 @@ const readVscodeTarget = async (root: string): Promise => { } } -const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot, trustFile } = readArgs() +const { reps, out, credentialEnv, evaluatedRoot, trustFile } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") const sideLabel = readSideLabel() // Trust gate. The evaluated root is loaded by the extension host and receives @@ -102,7 +103,19 @@ if (credential === undefined || credential.length === 0) { } ensureCodingSuiteRegistered() -const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() +// Side-level interrupt. A SIGINT or SIGTERM that reaches this process arms +// one flag at most once. The flow checks it before each repetition and +// before publication, so an interrupted side starts no new repetition and +// never publishes the completion marker. The substrate registers its own +// host teardown listener per run; the two listeners stay independent and +// the flag never consumes their signals. +const interrupt = createInterruptFlag() +process.on("SIGINT", () => interrupt.signal("SIGINT")) +process.on("SIGTERM", () => interrupt.signal("SIGTERM")) + +// Live sides always grade in the container runtime. The local subprocess +// runtime stays available to unit tests only. +const gradingRuntime = createContainerGradingRuntime() // Invalidate the previous generation before execution. After this point a // crash can never leave stale side results or run records behind: consumers @@ -112,79 +125,69 @@ await invalidateSideArtifacts(out) const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} -const runs: CodingRunRecord[] = [] -for (const bundle of CODING_FIXTURE_BUNDLES) { - const fixtureName = bundle.fixture.identity.name - for (let repIndex = 0; repIndex < reps; repIndex++) { - const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) - await fs.mkdir(artifactsDir, { recursive: true }) - try { - const record = await runCodingFixtureOnce({ - fixtureName, - repIndex, - credential, - runtime: createHostRuntime({ - root: evaluatedRoot, - artifactsDir, - vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), - }), - gradingRuntime, - scratchDir: artifactsDir, - }) - runs.push(record) - } catch { - // The substrate never throws for task failures. This guards wiring - // errors only, and records them as infrastructure failures. - const outcome = classifyRunOutcome({ hasTrace: false, finalWorkspaceAvailable: false }) - runs.push({ - fixtureName, - fixtureVersion: bundle.fixture.identity.version, - repIndex, - outcome, - metrics: buildRunMetrics({ - fixtureName, - repIndex, - outcome, - modelIdentifier: { available: false, reason: "runFailed" }, - }), - }) - } +const repetitions: SideRepetition[] = CODING_FIXTURE_BUNDLES.flatMap((bundle) => + Array.from({ length: reps }, (_, repIndex) => ({ + fixtureName: bundle.fixture.identity.name, + fixtureVersion: bundle.fixture.identity.version, + repIndex, + })), +) + +// Pins are recorded when the side publishes, so they describe what actually +// executed: the approved evaluated commit and, when the container runtime +// resolved one, the immutable grader image identity. +const buildPins = async (): Promise => { + const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined + return { + side: sideLabel, + commit: evaluatedCommit, + suiteId: CODING_SUITE_ID, + suiteDigest: codingSuiteDigest(), + settingsDigest: digestValue(projection), + modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", + graderVersion: GRADER_VERSION, + graderImageDigest: digest?.available === true ? digest.value : "unavailable", + runnerEnvironment: { + os: process.platform, + nodeVersion: process.version, + vscodeTarget: await readVscodeTarget(evaluatedRoot), + }, } } -// Pins are recorded after the runs so they describe what actually executed: -// the approved evaluated commit and, when the container runtime resolved one, -// the immutable grader image identity. -const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined -const pins: SidePins = { - side: sideLabel, - commit: evaluatedCommit, - suiteId: CODING_SUITE_ID, - suiteDigest: codingSuiteDigest(), - settingsDigest: digestValue(projection), - modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", - graderVersion: GRADER_VERSION, - graderImageDigest: digest?.available === true ? digest.value : "unavailable", - runnerEnvironment: { - os: process.platform, - nodeVersion: process.version, - vscodeTarget: await readVscodeTarget(evaluatedRoot), +const { runs, completed } = await executeSideFlow({ + repetitions, + out, + isInterrupted: interrupt.isInterrupted, + runRepetition: async ({ fixtureName, repIndex }) => { + const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) + await fs.mkdir(artifactsDir, { recursive: true }) + return runCodingFixtureOnce({ + fixtureName, + repIndex, + credential, + runtime: createHostRuntime({ + root: evaluatedRoot, + artifactsDir, + vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), + }), + gradingRuntime, + scratchDir: artifactsDir, + }) }, -} -const result: SideResultFile = { pins, runs } + // Publication point and completion marker. side-results.json appears + // atomically only after every repetition of this invocation finished, so + // a reader that sees the file sees a complete side of the current run. A + // missing file means an interrupted side; consumers must treat it as a + // failed side and never pair partial output. + buildResult: async (runs) => ({ pins: await buildPins(), runs }), +}) -// Per-run artifacts are published atomically as each record exists. -for (const record of runs) { - if (!record.trace) continue - const artifactDir = path.join(out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) - await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) +if (!completed) { + // An interrupted side exits with the conventional signal code and without + // side-results.json, so consumers treat it as a failed side. + console.error(`coding-evals side interrupted by ${interrupt.name() ?? "signal"}; side-results.json not published`) + process.exit(interrupt.exitCode()) } - -// Publication point and completion marker. side-results.json appears -// atomically only after every repetition of this invocation finished, so a -// reader that sees the file sees a complete side of the current run. A -// missing file means an interrupted side; consumers must treat it as a failed -// side and never pair partial output. -await writeJsonAtomic(path.join(out, "side-results.json"), result) const correct = runs.filter((record) => record.outcome.outcome === "correct").length -console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) +console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${codingSuiteDigest().slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts index fccf0c995a..402c753a7c 100644 --- a/scripts/agent-evals/coding-side.test.ts +++ b/scripts/agent-evals/coding-side.test.ts @@ -176,3 +176,34 @@ describe("coding side trust gate", () => { }, ) }) + +describe("coding side isolation gate", () => { + it("rejects --allow-unisolated-grading before any run or artifact exists", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + // The argument gate runs before the trust policy loads, so no + // fixture commit is needed and no credential is read. + const trustFile = await writePolicy(dir, []) + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--allow-unisolated-grading", + ], + minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /--allow-unisolated-grading is not accepted here/) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + // The rejection happens before any directory or result file + // exists, so the side can never publish host-graded artifacts. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 891da30572..1746ae172d 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -131,16 +131,14 @@ describe("container grading runtime", () => { assert.equal(runtime.isolation, "container") }) - it("pulls the image outside the grading timeout and runs with a unique name", async () => { + it("skips the pull when the image digest already resolves locally", async () => { const cli = recordingCli() const runtime = createContainerGradingRuntime({ cli }) const verdict = await runtime.run(makeJob()) assert.equal(verdict.kind, "passed") - // The pull happens first, the image identity is resolved from the - // local engine, and the timed run comes after both. - assert.equal(cli.calls[0], `pull ${GRADER_IMAGE}`) - assert.equal(cli.calls.indexOf("run 30000"), 2) - assert.equal(cli.calls[1], "inspect") + // The digest lookup decides, so a cached image never pulls again: the + // inspect comes first and the timed run follows it directly. + assert.deepEqual(cli.calls, ["inspect", "run 30000"]) // The run argv names a unique container. const nameIndex = cli.runArgs[0]!.indexOf("--name") assert.ok(nameIndex > 0) @@ -151,6 +149,45 @@ describe("container grading runtime", () => { assert.match(digest.available ? digest.value : "", /^node:22-bookworm@sha256:[0-9a-f]{64}$/) }) + it("pulls when the local digest is absent, then resolves the identity again", async () => { + let lookups = 0 + const cli = recordingCli({ + imageDigest: async () => { + lookups += 1 + cli.calls.push("inspect") + return lookups === 1 ? undefined : "node:22-bookworm@sha256:" + "a".repeat(64) + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.equal(verdict.kind, "passed") + // Inspect, pull, inspect again, and only then the timed run. + assert.deepEqual(cli.calls, ["inspect", `pull ${GRADER_IMAGE}`, "inspect", "run 30000"]) + const digest = await runtime.imageDigest!() + assert.equal(digest.available, true) + assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "a".repeat(64)) + }) + + it("preserves the resolved digest when a later lookup stays unavailable", async () => { + let lookups = 0 + const cli = recordingCli({ + imageDigest: async () => { + lookups += 1 + cli.calls.push("inspect") + return lookups <= 2 ? "node:22-bookworm@sha256:" + "b".repeat(64) : undefined + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + await runtime.run(makeJob()) + // The second run finds no digest, pulls, and still finds none; the + // runtime keeps the identity the first run resolved and never + // fabricates a new one. + await runtime.run(makeJob()) + const digest = await runtime.imageDigest!() + assert.equal(digest.available, true) + assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "b".repeat(64)) + }) + it("removes the container by name on timeout and verifies the removal before deleting output", async () => { const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-runtime-test-")) try { @@ -215,7 +252,17 @@ describe("container grading runtime", () => { }) it("reports a failed image pull before any container runs", async () => { - const cli = recordingCli({ pull: async () => "failed" }) + const cli = recordingCli({ + // The image is absent locally, so the runtime attempts the pull. + imageDigest: async () => { + cli.calls.push("inspect") + return undefined + }, + pull: async (image) => { + cli.calls.push(`pull ${image}`) + return "failed" + }, + }) const runtime = createContainerGradingRuntime({ cli }) const verdict = await runtime.run(makeJob()) assert.deepEqual(verdict, { @@ -223,6 +270,8 @@ describe("container grading runtime", () => { isolation: "container", reason: "graderImagePullFailed", }) + // The pull failure aborts before any timed run. + assert.deepEqual(cli.calls, ["inspect", `pull ${GRADER_IMAGE}`]) assert.deepEqual( cli.calls.filter((call) => call.startsWith("run")), [], diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 23f981e2ec..1cb5e8dd16 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -223,9 +223,17 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = }) const selected = await resolveCli() if (!selected) return failure("noContainerRuntime") - // The pull stays outside the grading timeout on purpose. - if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") - resolvedDigest = (await selected.imageDigest(job.resources.image)) ?? resolvedDigest + // The digest lookup decides whether a pull is needed, so a cached + // image never pulls again, and the pull stays outside the grading + // timeout on purpose. The identity is resolved again after a pull. + let digest = await selected.imageDigest(job.resources.image) + if (digest === undefined) { + if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") + digest = await selected.imageDigest(job.resources.image) + } + // A lookup that still finds nothing preserves a digest resolved by + // an earlier run of this runtime; it never fabricates one. + resolvedDigest = digest ?? resolvedDigest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) // The container runs as root with all capabilities dropped, so it diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs index 9242597534..f4709f861a 100644 --- a/scripts/agent-evals/grader-driver.mjs +++ b/scripts/agent-evals/grader-driver.mjs @@ -4,6 +4,7 @@ // or a missing result file means the grader itself failed, not the solution. // The driver receives no credential and needs no network. import { spawn } from "node:child_process" +import { randomUUID } from "node:crypto" import fs from "node:fs" import path from "node:path" @@ -14,9 +15,28 @@ if (!outPath) { } const testDir = process.argv[3] ?? "test" +// The result file is the host's only view of the grading outcome, so its +// publication is atomic: the content lands in a unique same-directory temp +// file and is renamed into place. A reader never observes a partial write, +// and a failed write or rename removes the temp file before the error +// propagates. const writeResult = (result) => { fs.mkdirSync(path.dirname(outPath), { recursive: true }) - fs.writeFileSync(outPath, JSON.stringify(result)) + const temp = path.join( + path.dirname(outPath), + `.${path.basename(outPath)}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`, + ) + try { + fs.writeFileSync(temp, JSON.stringify(result)) + fs.renameSync(temp, outPath) + } catch (error) { + try { + fs.rmSync(temp, { force: true }) + } catch { + /* The temp file is already gone. */ + } + throw error + } } const TEST_EXTENSIONS = new Set([".js", ".mjs", ".cjs"]) diff --git a/scripts/agent-evals/grader-driver.test.ts b/scripts/agent-evals/grader-driver.test.ts new file mode 100644 index 0000000000..835d5bd81f --- /dev/null +++ b/scripts/agent-evals/grader-driver.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { driverFilePath } from "./coding-grader.ts" + +// Runs the real driver against a real workspace with the real filesystem. +// The driver is the grading publication boundary, so these checks exercise +// the actual temp-write-and-rename behavior, not a stub. +const runDriver = async (args: readonly string[], cwd: string): Promise => { + // This test file itself runs under the node:test runner, whose + // NODE_TEST_CONTEXT would leak into the driver and then into its inner + // `node --test` children, switching them into runner-child mode with no + // TAP summary. The driver gets an environment without that variable. + const { NODE_TEST_CONTEXT: _runnerContext, ...driverEnv } = process.env + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [driverFilePath(), ...args], { cwd, env: driverEnv, stdio: "ignore" }) + child.once("error", reject) + child.once("close", (code) => resolve(code ?? -1)) + }) +} + +const passingTest = "import assert from 'node:assert/strict'\nassert.equal(1 + 1, 2)\n" + +const writeWorkspace = async (scratch: string): Promise => { + const workspace = path.join(scratch, "workspace") + await fs.mkdir(path.join(workspace, "test"), { recursive: true }) + await fs.writeFile(path.join(workspace, "test", "passing.test.mjs"), passingTest) + return workspace +} + +describe("grader driver result publication", () => { + it( + "publishes the completed result through a same-directory rename with no temp residue", + { timeout: 60_000 }, + async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "grader-driver-test-")) + try { + const workspace = await writeWorkspace(scratch) + const outDir = path.join(scratch, "out") + const exit = await runDriver([path.join(outDir, "result.json"), "test"], workspace) + assert.equal(exit, 0) + const result = JSON.parse(await fs.readFile(path.join(outDir, "result.json"), "utf8")) as { + kind: string + passed: boolean + passedCount: number + } + assert.equal(result.kind, "completed") + assert.equal(result.passed, true) + assert.equal(result.passedCount, 1) + // The atomic publication leaves exactly the result file behind. + assert.deepEqual((await fs.readdir(outDir)).sort(), ["result.json"]) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }, + ) + + it("removes the temp file when the rename into place fails", { timeout: 60_000 }, async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "grader-driver-test-")) + try { + const workspace = await writeWorkspace(scratch) + // A directory at the target path makes the rename fail after the + // temp write succeeded, so the cleanup path is the one under test. + const blocked = path.join(scratch, "blocked") + await fs.mkdir(blocked) + const exit = await runDriver([blocked, "test"], workspace) + assert.notEqual(exit, 0) + // Both publication attempts cleaned their temp files. + const residue = (await fs.readdir(scratch)).filter((name) => name.includes(".tmp")) + assert.deepEqual(residue, []) + // The blocked target is untouched. + assert.equal((await fs.stat(blocked)).isDirectory(), true) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/side-process-guard.test.ts b/scripts/agent-evals/side-process-guard.test.ts index adba5a2b64..c5620d49bc 100644 --- a/scripts/agent-evals/side-process-guard.test.ts +++ b/scripts/agent-evals/side-process-guard.test.ts @@ -5,7 +5,7 @@ import os from "node:os" import path from "node:path" import { describe, it } from "node:test" import { runBoundedPreparation } from "./coding-compare.ts" -import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" +import { createInterruptFlag, createSideProcessGuard, groupAlive, type SideKillOutcome } from "./side-process-guard.ts" // Credential-free process fixtures. Each child is a bare node process in its // own detached group, the same topology the comparison driver creates. @@ -279,3 +279,62 @@ describe("bounded preparation children", () => { } }) }) + +describe("group liveness probe", () => { + it("reports a live process group as alive", { timeout: 30_000 }, async () => { + const { child, spawned } = spawnDetachedChild("setInterval(() => {}, 1000000)") + try { + await spawned + assert.ok(child.pid !== undefined) + assert.equal(groupAlive(child.pid), true) + } finally { + child.kill("SIGKILL") + } + }) + + it("reports a gone process group as gone", { timeout: 30_000 }, async () => { + const { child, spawned, exited } = spawnDetachedChild("process.exit(0)") + await spawned + await exited + assert.ok(child.pid !== undefined) + assert.equal(groupAlive(child.pid), false) + }) + + it("treats a refused probe as an alive group so the escalation continues", () => { + const throwCode = (code: string) => () => { + throw Object.assign(new Error(`probe refused: ${code}`), { code }) + } + // EPERM means the group exists but the probe was not allowed. + assert.equal(groupAlive(123_456, throwCode("EPERM")), true) + // Any other error is not proof of absence either. + assert.equal( + groupAlive(123_456, () => { + throw new Error("probe failed") + }), + true, + ) + // Only ESRCH proves the group is gone. + assert.equal(groupAlive(123_456, throwCode("ESRCH")), false) + }) +}) + +describe("side interrupt flag", () => { + it("arms at most once and keeps the first signal", () => { + const flag = createInterruptFlag() + assert.equal(flag.isInterrupted(), false) + assert.equal(flag.exitCode(), 1) + flag.signal("SIGINT") + // A later signal must not un-arm or replace the first one. + flag.signal("SIGTERM") + assert.equal(flag.isInterrupted(), true) + assert.equal(flag.name(), "SIGINT") + assert.equal(flag.exitCode(), 130) + }) + + it("maps SIGTERM to its conventional exit code", () => { + const flag = createInterruptFlag() + flag.signal("SIGTERM") + assert.equal(flag.name(), "SIGTERM") + assert.equal(flag.exitCode(), 143) + }) +}) diff --git a/scripts/agent-evals/side-process-guard.ts b/scripts/agent-evals/side-process-guard.ts index 8745273b8e..cd7897bf48 100644 --- a/scripts/agent-evals/side-process-guard.ts +++ b/scripts/agent-evals/side-process-guard.ts @@ -17,12 +17,46 @@ export interface SideProcessGuard { interrupt(): Promise } -const groupAlive = (pid: number): boolean => { +// Probes one process group. Only ESRCH proves the group is gone. EPERM and +// any other error mean the probe was refused for a group that still exists, +// so the group counts as alive and the SIGKILL escalation still fires. The +// probe parameter is a test seam; production callers use process.kill. +export const groupAlive = ( + pid: number, + probe: (pid: number, signal?: NodeJS.Signals | number) => void = process.kill, +): boolean => { try { - process.kill(-pid, 0) + probe(-pid, 0) return true - } catch { - return false + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ESRCH" + } +} + +// One side-level interrupt flag. The handler arms it at most once, so +// repeated signals cannot un-arm or re-arm it. Listeners that share the +// process, the substrate's host teardown for example, observe their own +// handlers and never consume this flag. +export interface InterruptFlag { + // Arms the flag. Later calls are no-ops. + signal(name: NodeJS.Signals): void + // True once any signal arrived. + isInterrupted(): boolean + // The observed signal name, for diagnostics. + name(): NodeJS.Signals | undefined + // The conventional shell exit code for the observed signal. + exitCode(): number +} + +export const createInterruptFlag = (): InterruptFlag => { + let signal: NodeJS.Signals | undefined + return { + signal: (name) => { + if (signal === undefined) signal = name + }, + isInterrupted: () => signal !== undefined, + name: () => signal, + exitCode: () => (signal === "SIGINT" ? 130 : signal === "SIGTERM" ? 143 : 1), } } From 4f0e44ea3149194d637723402367f0618d1730d1 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 26 Oct 2026 02:55:24 +0000 Subject: [PATCH 06/22] fix(agent-evals): reject dirty checkouts and clear stale grader digests --- scripts/agent-evals/README.md | 2 +- .../agent-evals/coding-compare-script.test.ts | 43 ++++++ scripts/agent-evals/coding-side-flow.test.ts | 56 ++++++++ scripts/agent-evals/coding-side-flow.ts | 15 ++- scripts/agent-evals/coding-side.mts | 16 ++- scripts/agent-evals/coding-side.test.ts | 42 ++++++ scripts/agent-evals/coding-trust.test.ts | 124 ++++++++++++++++++ scripts/agent-evals/coding-trust.ts | 34 +++++ scripts/agent-evals/container-runtime.test.ts | 23 ++-- scripts/agent-evals/container-runtime.ts | 11 +- 10 files changed, 349 insertions(+), 17 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 570c1744b2..1a6ef27435 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -156,7 +156,7 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It also rejects an evaluated root whose tracked or untracked files differ from that commit, because the approval describes the commit and not the working tree. The check has one trust limit: ignored files, such as dependency installs and build outputs, stay outside commit identity and outside the check, so the approval never attests them. A recorded comparison closes that gap by loading only a fresh detached worktree built from the approved commit. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. Every driver child, preparation steps and the side alike, runs detached in its own process group and is tracked by one guard. A SIGINT or SIGTERM that reaches only the driver terminates the tracked group through a bounded SIGTERM-then-SIGKILL escalation, and the worktree cleanup waits for that escalation before it removes any tree. diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts index 05b68288d5..ff45865fc4 100644 --- a/scripts/agent-evals/coding-compare-script.test.ts +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -82,4 +82,47 @@ describe("coding compare isolation gate", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it( + "runs past the isolation gate without the flag and fails at the approved-commit check", + { timeout: 180_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-test-")) + try { + const trustFile = path.join(dir, "trust-policy.json") + // The empty policy approves nothing, so the trust gate must reject + // the run. The placeholder credential only proves the failure is + // not the credential check; no real credential enters this test. + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [] })) + const out = path.join(dir, "out") + const env = { ...minimalEnv(), SOLHEIM_API_KEY: "not-a-real-credential" } + const outcome = await runCompareScript( + [ + "--baseline", + "a".repeat(40), + "--candidate", + "b".repeat(40), + "--reps", + "1", + "--out", + out, + "--trust-file", + trustFile, + ], + env, + ) + assert.notEqual(outcome.code, 0) + // The failure is the approved-commit gate, not the argument gate. + assert.doesNotMatch(outcome.stderr, /--allow-unisolated-grading is not accepted here/) + assert.match(outcome.stderr, /trust policy must list at least one approved commit SHA/) + // The rejection happens before any report generation exists, so + // the comparison never publishes anything. + for (const name of ["comparison-report.json", "comparison-report.md", "comparison-report.complete"]) { + await assert.rejects(fs.access(path.join(out, name))) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) }) diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts index 85a13c3f8f..ddb44c5082 100644 --- a/scripts/agent-evals/coding-side-flow.test.ts +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -143,6 +143,62 @@ describe("side execution flow", () => { } }) + it("prints one bounded diagnostic with the fixture name and repetition index when a repetition throws", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + const lines: string[] = [] + const originalError = console.error + console.error = (line: string) => lines.push(String(line)) + try { + const outcome = await flow({ + repetitions: plan(1), + out: dir, + isInterrupted: () => false, + runRepetition: async () => { + throw new Error("wiring broke") + }, + }) + assert.equal(outcome.completed, true) + const failed = outcome.runs[0] + assert.ok(failed) + assert.deepEqual(failed.outcome, { outcome: "infrastructure-failure", reason: "runTraceMissing" }) + assert.equal(lines.length, 1) + assert.match( + lines[0] ?? "", + /^coding-evals repetition failed: fixture clamp-range rep 0: Error: wiring broke$/, + ) + } finally { + console.error = originalError + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("prints a safe fallback for a non-Error throw and never prints the thrown value", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + const lines: string[] = [] + const originalError = console.error + console.error = (line: string) => lines.push(String(line)) + try { + const outcome = await flow({ + repetitions: plan(1), + out: dir, + isInterrupted: () => false, + runRepetition: async () => { + throw { credential: "not-a-real-credential" } + }, + }) + const failed = outcome.runs[0] + assert.ok(failed) + assert.deepEqual(failed.outcome, { outcome: "infrastructure-failure", reason: "runTraceMissing" }) + assert.equal(lines.length, 1) + assert.match(lines[0] ?? "", /non-Error value thrown$/) + // The thrown value itself never reaches the diagnostic. + assert.ok(!lines[0]?.includes("not-a-real-credential")) + } finally { + console.error = originalError + await fs.rm(dir, { recursive: true, force: true }) + } + }) + it("keeps running after a wiring error and records it as an infrastructure failure", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) try { diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts index cba0deaf5a..8c0057b3f7 100644 --- a/scripts/agent-evals/coding-side-flow.ts +++ b/scripts/agent-evals/coding-side-flow.ts @@ -40,6 +40,12 @@ export interface SideFlowOutcome { completed: boolean } +// One bounded diagnostic line per failed repetition. Only the error name and +// message travel to stderr; a thrown non-Error value can hold anything, so it +// is never printed. +const describeRepetitionError = (error: unknown): string => + error instanceof Error ? `${error.name}: ${error.message.slice(0, 500)}` : "non-Error value thrown" + export async function executeSideFlow(inputs: SideFlowInputs): Promise { const runs: CodingRunRecord[] = [] for (const repetition of inputs.repetitions) { @@ -48,9 +54,14 @@ export async function executeSideFlow(inputs: SideFlowInputs): Promise { + it( + "rejects a dirty approved evaluated root before any host launch or publication", + { timeout: 180_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + const trustFile = await writePolicy(dir, [commit]) + // The approved checkout is modified after approval: one staged + // tracked change and one untracked file. The commit identity is + // still approved, but it no longer describes the tree. + await fs.writeFile(path.join(repo, "tracked.txt"), "changed") + await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) + await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") + // The fake credential only proves the failure is not the + // credential check: the cleanliness gate must stop the side first. + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, + ], + minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /tracked or untracked changes/) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + // The rejection happens before the publication point: no side + // results and no artifacts exist. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + await assert.rejects(fs.access(path.join(dir, "out", "artifacts"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) + it("rejects an unapproved evaluated root before any host launch", { timeout: 180_000 }, async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) try { diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index c30f55ab66..9081c034a9 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -1,20 +1,53 @@ import assert from "node:assert/strict" +import { execFile } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" +import { promisify } from "node:util" import { describe, it } from "node:test" import { assertCommitsApproved, assertCredentialEnvAllowed, + assertEvaluatedTreeClean, buildSideEnvironment, buildWorktreeEnvironment, + DirtyEvaluatedTreeError, loadTrustPolicy, UntrustedCommitError, type TrustPolicy, } from "./coding-trust.ts" +const execFileAsync = promisify(execFile) const sha = (c: string): string => c.repeat(40) +// One throwaway repository with one empty commit. It stands in for an +// evaluated root; no code from it ever runs in these checks. +const makeRepo = async (dir: string): Promise => { + const repo = path.join(dir, "repo") + await fs.mkdir(repo) + await execFileAsync("git", ["init", "-q"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + // The operator's git config may force signed commits, and the + // throwaway fixture has no key or tty. + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "-q", + "-m", + "fixture", + ], + { cwd: repo }, + ) + return repo +} + const writePolicy = async (dir: string, content: unknown): Promise => { const file = path.join(dir, "trust-policy.json") await fs.writeFile(file, JSON.stringify(content)) @@ -117,6 +150,97 @@ describe("credential environment allowlist", () => { }) }) +describe("evaluated tree cleanliness gate", () => { + it("accepts a clean tree", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await assert.doesNotReject(assertEvaluatedTreeClean(repo)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("accepts a tree with only ignored build artifacts", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await fs.writeFile(path.join(repo, ".gitignore"), "dist\n") + await execFileAsync("git", ["add", ".gitignore"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "ignore", + ], + { cwd: repo }, + ) + // Ignored files sit outside commit identity: dependency installs + // and build outputs never make the tree dirty, and the approval + // never attests them either. + await fs.mkdir(path.join(repo, "dist")) + await fs.writeFile(path.join(repo, "dist", "run.sh"), "#!/bin/sh\n") + await fs.chmod(path.join(repo, "dist", "run.sh"), 0o755) + await assert.doesNotReject(assertEvaluatedTreeClean(repo)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects a tracked modification after the commit was approved", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await fs.writeFile(path.join(repo, "tracked.txt"), "approved\n") + await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "tracked", + ], + { cwd: repo }, + ) + await fs.writeFile(path.join(repo, "tracked.txt"), "changed\n") + await assert.rejects(assertEvaluatedTreeClean(repo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError) + assert.match(error.message, /tracked or untracked changes/) + assert.ok(error.entries.some((entry) => entry.includes("tracked.txt"))) + return true + }) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects an untracked file", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await fs.writeFile(path.join(repo, "untracked.txt"), "x") + await assert.rejects(assertEvaluatedTreeClean(repo), DirtyEvaluatedTreeError) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) + describe("worktree process environment", () => { it("passes the install and build allowlist and drops secrets", () => { const child = buildWorktreeEnvironment({ diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 6ff10a36e6..1b797c68e0 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -63,6 +63,40 @@ export function assertCommitsApproved(policy: TrustPolicy, commits: readonly str if (unapproved.length > 0) throw new UntrustedCommitError(unapproved) } +// A tree whose tracked or untracked files differ from its commit executes +// content the approved commit does not contain, so the approval would not +// describe the code the extension host loads. +export class DirtyEvaluatedTreeError extends Error { + constructor(readonly entries: string[]) { + const shown = entries.slice(0, 10) + const more = entries.length > shown.length ? `, and ${entries.length - shown.length} more` : "" + super( + `evaluated root has tracked or untracked changes; the approved commit does not describe the files the host would load: ${shown.join(", ")}${more}`, + ) + this.name = "DirtyEvaluatedTreeError" + } +} + +// Gate before any host launch. The evaluated root must match its approved +// commit exactly, so a modified checkout can never run under that commit's +// approval. Ignored files, such as dependency installs and build outputs, +// stay outside commit identity and outside this check: the approval does not +// attest them. The comparison driver closes that gap by loading only a fresh +// detached worktree built from the approved commit. +export async function assertEvaluatedTreeClean(root: string): Promise { + const { execFile } = await import("node:child_process") + const { promisify } = await import("node:util") + const { stdout } = await promisify(execFile)("git", ["status", "--porcelain"], { + cwd: root, + timeout: 30_000, + }) + const entries = stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + if (entries.length > 0) throw new DirtyEvaluatedTreeError(entries) +} + // Provider credential variables this framework supports. The side script and // the comparison driver accept only these names, so the tooling can never be // pointed at an unrelated parent secret by naming an arbitrary variable. diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 1746ae172d..7552474f95 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -168,24 +168,31 @@ describe("container grading runtime", () => { assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "a".repeat(64)) }) - it("preserves the resolved digest when a later lookup stays unavailable", async () => { + it("never reuses a previous digest when the current lookup stays unavailable", async () => { let lookups = 0 const cli = recordingCli({ imageDigest: async () => { lookups += 1 cli.calls.push("inspect") - return lookups <= 2 ? "node:22-bookworm@sha256:" + "b".repeat(64) : undefined + // Only the first run's lookup finds the image; every later + // lookup, before and after a pull, stays unavailable. + return lookups === 1 ? "node:22-bookworm@sha256:" + "b".repeat(64) : undefined }, }) const runtime = createContainerGradingRuntime({ cli }) + // The first run resolves the identity and records it for the pins. await runtime.run(makeJob()) - // The second run finds no digest, pulls, and still finds none; the - // runtime keeps the identity the first run resolved and never - // fabricates a new one. + const first = await runtime.imageDigest!() + assert.equal(first.available, true) + assert.equal(first.available ? first.value : "", "node:22-bookworm@sha256:" + "b".repeat(64)) + // The second run finds no digest before or after the pull. The + // runtime must report the identity as unavailable instead of reusing + // the digest of the image the first run graded with. await runtime.run(makeJob()) - const digest = await runtime.imageDigest!() - assert.equal(digest.available, true) - assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "b".repeat(64)) + // The missed lookups sent the second run to the pull path. + assert.equal(cli.calls.filter((call) => call.startsWith("pull")).length, 1) + const second = await runtime.imageDigest!() + assert.equal(second.available, false) }) it("removes the container by name on timeout and verifies the removal before deleting output", async () => { diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 1cb5e8dd16..b09efef745 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -221,6 +221,10 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = isolation: "container", reason, }) + // Each run records only its own lookup result. A digest an earlier + // run resolved must never stand in for the identity of the image + // this run grades with. + resolvedDigest = undefined const selected = await resolveCli() if (!selected) return failure("noContainerRuntime") // The digest lookup decides whether a pull is needed, so a cached @@ -231,9 +235,10 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") digest = await selected.imageDigest(job.resources.image) } - // A lookup that still finds nothing preserves a digest resolved by - // an earlier run of this runtime; it never fabricates one. - resolvedDigest = digest ?? resolvedDigest + // A lookup that still finds nothing leaves the digest unset, and + // the pins record the identity as unavailable instead of reusing + // one from an earlier run. The runtime never fabricates a value. + resolvedDigest = digest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) // The container runs as root with all capabilities dropped, so it From d26ad555efc9f1843da08846899fd74db204bcc7 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 26 Oct 2026 22:35:58 +0000 Subject: [PATCH 07/22] fix(agent-evals): prevent child startup after interruption and preserve signal exit codes --- .../agent-evals/coding-compare-driver.test.ts | 314 ++++++++++++++++++ .../agent-evals/coding-compare-script.test.ts | 157 ++++++++- scripts/agent-evals/coding-compare.mts | 188 +++-------- scripts/agent-evals/coding-compare.ts | 183 +++++++++- 4 files changed, 703 insertions(+), 139 deletions(-) create mode 100644 scripts/agent-evals/coding-compare-driver.test.ts diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts new file mode 100644 index 0000000000..fcf7a1f28d --- /dev/null +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -0,0 +1,314 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { describe, it } from "node:test" +import { + executeComparisonFlow, + publishedComparisonGeneration, + runCompareSide, + type CompareSideWiring, + type ComparisonFlowWiring, + type SideResultFile, +} from "./coding-compare.ts" +import { buildRunMetrics } from "./coding-metrics.ts" +import type { SidePins } from "./coding-pins.ts" +import { createInterruptFlag, type InterruptFlag } from "./side-process-guard.ts" + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const COMMIT = "b".repeat(40) + +// The guard records every tracked group, so a test can prove the side child +// was never spawned and nothing was left tracked after a real child exited. +const trackingGuard = (tracked: (number | undefined)[]): CompareSideWiring["guard"] => ({ + track: (pid) => { + tracked.push(pid) + }, + interrupt: async () => "none", +}) + +const sideWiring = (inputs: { + flag: InterruptFlag + tracked: (number | undefined)[] + sideScript: string + createSideOutDir: (dir: string) => Promise +}): CompareSideWiring => ({ + guard: trackingGuard(inputs.tracked), + isInterrupted: inputs.flag.isInterrupted, + root: repoRoot, + sideScript: inputs.sideScript, + reps: 1, + trustFile: path.join(repoRoot, "unused-in-unit-tests.json"), + credentialEnv: "SOLHEIM_API_KEY", + sideTimeoutMs: 60_000, + killGraceMs: 100, + prepareWorktree: async () => {}, + createSideOutDir: inputs.createSideOutDir, +}) + +const flowWiring = (inputs: { + flag: InterruptFlag + out: string + scratchDir: string + runSide: (side: "baseline" | "candidate") => Promise +}): ComparisonFlowWiring => ({ + out: inputs.out, + scratchDir: inputs.scratchDir, + root: repoRoot, + reps: 1, + runSide: inputs.runSide, + interruptedSignal: inputs.flag.name, + interruptKill: async () => {}, +}) + +const pins = (side: SidePins["side"], commit: string): SidePins => ({ + side, + commit, + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "test-model", + graderVersion: "coding-grader-v2", + graderImageDigest: "unavailable", + runnerEnvironment: { os: "linux", nodeVersion: process.version, vscodeTarget: "test" }, +}) + +const runMetrics = (fixtureName: string, repIndex: number) => + buildRunMetrics({ + fixtureName, + repIndex, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + modelIdentifier: { available: true, value: "test-model" }, + }) + +const record = (side: SidePins["side"], commit: string): SideResultFile => ({ + pins: pins(side, commit), + runs: [ + { + fixtureName: "clamp-range", + fixtureVersion: "1.0.0", + repIndex: 0, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + trace: undefined, + metrics: runMetrics("clamp-range", 0), + }, + ], +}) + +// A side script fixture that writes one valid result file under --out and +// exits with the given code. The commit travels in the source, because the +// spawn passes no commit argument and the pins must record the approved one. +const sideScriptFixture = (commit: string, exitCode: number): string => + [ + `const fs = require("node:fs")`, + `const path = require("node:path")`, + `const read = (name) => {`, + ` const args = process.argv.slice(2)`, + ` const index = args.indexOf("--" + name)`, + ` return index >= 0 ? args[index + 1] : undefined`, + `}`, + `const pins = {`, + ` side: process.env.CODING_EVALS_SIDE,`, + ` commit: ${JSON.stringify(commit)},`, + ` suiteId: "coding-basic-v1",`, + ` suiteDigest: "d1".repeat(32),`, + ` settingsDigest: "d2".repeat(32),`, + ` modelIdentifier: "fixture-model",`, + ` graderVersion: "fixture-grader-v1",`, + ` graderImageDigest: "unavailable",`, + ` runnerEnvironment: { os: "fixture-os", nodeVersion: process.version, vscodeTarget: "fixture" },`, + `}`, + `const out = read("out")`, + `fs.mkdirSync(out, { recursive: true })`, + `fs.writeFileSync(path.join(out, "side-results.json"), JSON.stringify({`, + ` pins,`, + ` runs: [{ fixtureName: "fixture", fixtureVersion: "1.0.0", repIndex: 0, outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, metrics: {} }],`, + `}))`, + `process.exit(${exitCode})`, + ].join("\n") + +describe("compare side interruption gate", () => { + it("refuses to spawn the side when the interrupt lands during the output directory creation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-side-test-")) + try { + const flag = createInterruptFlag() + const tracked: (number | undefined)[] = [] + const sideOut = path.join(dir, "candidate-out") + const wiring = sideWiring({ + flag, + tracked, + sideScript: "unused-because-no-spawn-may-happen.cjs", + // The signal lands inside the last await before the spawn: the + // window where the guard holds no tracked group and a finished + // escalation would never revisit the new child. + createSideOutDir: async () => { + flag.signal("SIGINT") + }, + }) + await assert.rejects( + runCompareSide("candidate", COMMIT, dir, wiring), + /interrupted before the candidate side started/, + ) + // No child was ever spawned or tracked, so no repetition can be + // running and nothing is left to kill. + assert.deepEqual(tracked, []) + await assert.rejects(fs.access(sideOut)) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("runs the side, tracks it, and reads its result when nothing interrupts", { timeout: 60_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-side-test-")) + try { + const flag = createInterruptFlag() + const tracked: (number | undefined)[] = [] + const fixture = path.join(dir, "fixture-side.cjs") + await fs.writeFile(fixture, sideScriptFixture(COMMIT, 0)) + const wiring = sideWiring({ + flag, + tracked, + sideScript: fixture, + createSideOutDir: async (sideOut) => { + await fs.mkdir(sideOut, { recursive: true }) + }, + }) + const result = await runCompareSide("baseline", COMMIT, dir, wiring) + assert.equal(result.pins.side, "baseline") + assert.equal(result.pins.commit, COMMIT) + assert.equal(result.runs.length, 1) + // The real child was tracked and left nothing tracked behind. + assert.equal(tracked.length >= 1, true) + assert.equal(tracked.at(-1), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("fails the side and untracks it when the side process exits nonzero", { timeout: 60_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-side-test-")) + try { + const flag = createInterruptFlag() + const tracked: (number | undefined)[] = [] + const fixture = path.join(dir, "fixture-side.cjs") + await fs.writeFile(fixture, sideScriptFixture(COMMIT, 3)) + const wiring = sideWiring({ + flag, + tracked, + sideScript: fixture, + createSideOutDir: async (sideOut) => { + await fs.mkdir(sideOut, { recursive: true }) + }, + }) + await assert.rejects( + runCompareSide("baseline", COMMIT, dir, wiring), + /baseline side failed with exit code 3/, + ) + assert.equal(tracked.at(-1), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) + +describe("comparison flow outcome and cleanup", () => { + // Both signals replay the throwing interrupt path: the signal lands + // mid-side and the side still ends in a failure. + const interruptedRun = async (signal: NodeJS.Signals) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-flow-test-")) + try { + const flag = createInterruptFlag() + const sides: ("baseline" | "candidate")[] = [] + const outcome = await executeComparisonFlow( + flowWiring({ + flag, + out: path.join(dir, "out"), + scratchDir: dir, + runSide: async (side) => { + sides.push(side) + if (side === "baseline") { + flag.signal(signal) + throw new Error("baseline side failed with exit code 1") + } + return record(side, COMMIT) + }, + }), + ) + return { outcome, sides, dir } + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + } + + it("reports SIGINT and finishes cleanup when a side fails under the interrupt", async () => { + const { outcome, sides, dir } = await interruptedRun("SIGINT") + assert.deepEqual(outcome, { status: "interrupted", signal: "SIGINT" }) + // The interrupted side stopped the flow before the candidate started. + assert.deepEqual(sides, ["baseline"]) + // The cleanup completed before the outcome resolved. + await assert.rejects(fs.access(dir)) + }) + + it("reports SIGTERM and finishes cleanup when a side fails under the interrupt", async () => { + const { outcome, sides, dir } = await interruptedRun("SIGTERM") + assert.deepEqual(outcome, { status: "interrupted", signal: "SIGTERM" }) + assert.deepEqual(sides, ["baseline"]) + await assert.rejects(fs.access(dir)) + }) + + it("keeps a non-interrupt failure a failure and cleans up anyway", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-flow-test-")) + try { + const flag = createInterruptFlag() + await assert.rejects( + executeComparisonFlow( + flowWiring({ + flag, + out: path.join(dir, "out"), + scratchDir: dir, + runSide: async () => { + throw new Error("candidate side failed with exit code 7") + }, + }), + ), + /candidate side failed with exit code 7/, + ) + assert.equal(flag.isInterrupted(), false) + await assert.rejects(fs.access(dir)) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("completes, publishes, and returns the report when both sides succeed", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-flow-test-")) + try { + const out = path.join(dir, "out") + await fs.mkdir(out, { recursive: true }) + const baselineCommit = "a".repeat(40) + const candidateCommit = "c".repeat(40) + const flag = createInterruptFlag() + const sides: ("baseline" | "candidate")[] = [] + const outcome = await executeComparisonFlow( + flowWiring({ + flag, + out, + scratchDir: path.join(dir, "scratch"), + runSide: async (side) => { + sides.push(side) + return record(side, side === "baseline" ? baselineCommit : candidateCommit) + }, + }), + ) + assert.deepEqual(sides, ["baseline", "candidate"]) + assert.equal(outcome.status, "completed") + if (outcome.status !== "completed") return + assert.equal(outcome.report.pins.baseline.commit, baselineCommit) + assert.equal(outcome.report.pins.candidate.commit, candidateCommit) + assert.ok((await publishedComparisonGeneration(out)) !== undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts index ff45865fc4..411dc20729 100644 --- a/scripts/agent-evals/coding-compare-script.test.ts +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -1,11 +1,12 @@ import assert from "node:assert/strict" -import { execFile } from "node:child_process" +import { execFile, execFileSync, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" import { describe, it } from "node:test" +import { groupAlive } from "./side-process-guard.ts" const execFileAsync = promisify(execFile) const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -126,3 +127,157 @@ describe("coding compare isolation gate", () => { }, ) }) + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +const waitForFile = async (filePath: string, timeoutMs: number): Promise => { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if ( + await fs.access(filePath).then( + () => true, + () => false, + ) + ) + return + await delay(20) + } + throw new Error(`fixture file never appeared: ${filePath}`) +} + +// Polls until the group answers ESRCH, so the test proves the group is gone +// instead of hoping the kill landed. +const assertGroupGone = async (groupPid: number): Promise => { + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + if (!groupAlive(groupPid)) return + await delay(50) + } + throw new Error(`worktree preparation group ${groupPid} survived the driver exit`) +} + +interface SignalRun { + exitCode: number | null + stderr: string + scratchParent: string + worktreeGroupPid: number +} + +// Spawns the real comparison driver with a PATH shim. The shim turns +// `git worktree add` into a detached sleeper that records its group id, so +// the test can aim one signal at the driver alone while a preparation child +// runs. Every other git invocation proxies to the real git, and the trust +// gates pass with the HEAD commit. The credential is a placeholder; nothing +// here contacts a provider. +const runDriverUntilSignal = async (signal: NodeJS.Signals): Promise => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-signal-test-")) + const realGit = execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim() + const shimDir = path.join(dir, "bin") + await fs.mkdir(shimDir) + const gitShim = [ + "#!/bin/sh", + 'if [ "$1" = "worktree" ]; then', + ' if [ "$2" = "add" ]; then', + ' printf \'%s\\n\' "$$" > "$TMPDIR/worktree-add-pid"', + " exec sleep 30", + " fi", + ' if [ "$2" = "remove" ]; then', + " exit 0", + " fi", + "fi", + `exec '${realGit}' "$@"`, + ].join("\n") + await fs.writeFile(path.join(shimDir, "git"), gitShim, { mode: 0o755 }) + const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim() + const trustFile = path.join(dir, "trust-policy.json") + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [head] })) + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), + "--baseline", + head, + "--candidate", + head, + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + ], + { + cwd: repoRoot, + env: { + PATH: `${shimDir}:${process.env["PATH"] ?? ""}`, + HOME: process.env["HOME"] ?? os.homedir(), + TMPDIR: dir, + SOLHEIM_API_KEY: "not-a-real-credential", + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + const pidFile = path.join(dir, "worktree-add-pid") + await waitForFile(pidFile, 60_000) + // One signal aimed at the driver alone. The detached preparation group + // receives nothing from this kill call. + child.kill(signal) + const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { + const timer = setTimeout(() => { + child.kill("SIGKILL") + reject(new Error(`driver did not exit after ${signal}: ${stderr}`)) + }, 60_000) + child.once("exit", (code, exitSignal) => { + clearTimeout(timer) + resolve({ code, signal: exitSignal }) + }) + child.once("error", (error) => { + clearTimeout(timer) + reject(error) + }) + }) + const groupPid = Number(await fs.readFile(pidFile, "utf8")) + return { exitCode: exit.code, stderr, scratchParent: dir, worktreeGroupPid: groupPid } +} + +describe("compare driver signal exit", () => { + const expectSignalExit = async (signal: NodeJS.Signals, expectedCode: number): Promise => { + const run = await runDriverUntilSignal(signal) + try { + // The throwing interrupt path exits with the conventional signal + // code after the cleanup, not with an unhandled-rejection 1. + assert.equal(run.exitCode, expectedCode) + // The detached worktree preparation group did not survive. + await assertGroupGone(run.worktreeGroupPid) + // The cleanup removed the scratch generation before the exit. + const leftovers = (await fs.readdir(run.scratchParent)).filter((entry) => + entry.startsWith("coding-evals-compare-"), + ) + assert.deepEqual(leftovers, []) + } finally { + await fs.rm(run.scratchParent, { recursive: true, force: true }).catch(() => {}) + } + } + + it( + "exits 130 after cleanup when SIGINT reaches the driver during worktree preparation", + { timeout: 180_000 }, + async () => { + await expectSignalExit("SIGINT", 130) + }, + ) + + it( + "exits 143 after cleanup when SIGTERM reaches the driver during worktree preparation", + { timeout: 180_000 }, + async () => { + await expectSignalExit("SIGTERM", 143) + }, + ) +}) diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index e201fbc431..226dccfbc8 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -18,31 +18,14 @@ // SIGTERM aimed at the driver alone terminate the tracked group with a bounded // SIGTERM-then-SIGKILL escalation before the scratch cleanup runs, and an // interrupt leaves no orphan host behind. -import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" -import { promisify } from "node:util" -import { - publishComparisonReport, - runBoundedPreparation, - sideResultFileName, - toSideResult, - validateSideResultFile, - type SideResultFile, -} from "./coding-compare.ts" -import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" -import { - assertCommitsApproved, - assertCredentialEnvAllowed, - buildSideEnvironment, - loadTrustPolicy, - resolveCommit, -} from "./coding-trust.ts" -import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" +import { executeComparisonFlow, runBoundedPreparation, runCompareSide } from "./coding-compare.ts" +import { assertCommitsApproved, assertCredentialEnvAllowed, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" +import { createInterruptFlag, createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" -const execFileAsync = promisify(execFile) const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const INSTALL_TIMEOUT_MS = 10 * 60_000 @@ -52,16 +35,16 @@ const SIDE_KILL_GRACE_MS = 30_000 const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 // Interrupt state. The handlers are registered before any child is spawned. -// A signal terminates the tracked child group at once; the main flow observes -// the child exit, the cleanup awaits the escalation before it removes the -// worktrees, and the driver exits with the conventional signal exit code. +// A signal arms the flag and starts the guard escalation at once; the flow +// observes the flag at its async boundaries, the cleanup awaits the +// escalation before it removes the worktrees, and the driver exits with the +// conventional signal exit code after that cleanup. const sideGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) -let interruptSignal: NodeJS.Signals | undefined +const interruptFlag = createInterruptFlag() let interruptKill: Promise | undefined -const isInterrupted = (): boolean => interruptSignal !== undefined const onInterruptSignal = (signal: NodeJS.Signals): void => { - if (interruptSignal !== undefined) return - interruptSignal = signal + if (interruptFlag.isInterrupted()) return + interruptFlag.signal(signal) interruptKill = sideGuard.interrupt() } process.on("SIGINT", () => onInterruptSignal("SIGINT")) @@ -121,97 +104,27 @@ const readArgs = (): { const prepareWorktree = async (worktree: string): Promise => { await runBoundedPreparation( sideGuard, - isInterrupted, + interruptFlag.isInterrupted, "pnpm", ["install", "--frozen-lockfile"], worktree, INSTALL_TIMEOUT_MS, ) - await runBoundedPreparation(sideGuard, isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) + await runBoundedPreparation(sideGuard, interruptFlag.isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) } -// Runs one trusted side against the evaluated worktree. The side script comes -// from this checkout, receives only the allowlisted environment plus the -// named credential variable, re-checks the worktree commit against the trust -// policy, and must finish inside its own wall clock. -const runSide = async ( - side: "baseline" | "candidate", - commit: string, - inputs: ReturnType, - scratchDir: string, -): Promise => { - if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) - const worktree = path.join(scratchDir, side) +// One side's full preparation: the detached worktree add of that exact +// commit, then the install and build inside it. +const prepareSideWorktree = async (commit: string, worktree: string): Promise => { await runBoundedPreparation( sideGuard, - isInterrupted, + interruptFlag.isInterrupted, "git", ["worktree", "add", "--detach", worktree, commit], root, WORKTREE_TIMEOUT_MS, ) await prepareWorktree(worktree) - if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) - const sideOut = path.join(scratchDir, `${side}-out`) - await fs.mkdir(sideOut, { recursive: true }) - const exit = await new Promise((resolve) => { - const child = spawn( - process.execPath, - [ - "--import", - "tsx", - path.join(root, "scripts/agent-evals/coding-side.mts"), - "--reps", - String(inputs.reps), - "--out", - sideOut, - "--trust-file", - inputs.trustFile, - "--credential-env", - inputs.credentialEnv, - "--evaluated-root", - worktree, - ], - { - cwd: root, - env: buildSideEnvironment(process.env, inputs.credentialEnv, side), - stdio: "inherit", - detached: true, - }, - ) - sideGuard.track(child.pid) - let timedOut = false - const timer = setTimeout(() => { - timedOut = true - try { - if (child.pid) process.kill(-child.pid, "SIGTERM") - } catch { - child.kill("SIGTERM") - } - setTimeout(() => { - try { - if (child.pid) process.kill(-child.pid, "SIGKILL") - } catch { - child.kill("SIGKILL") - } - }, SIDE_KILL_GRACE_MS).unref() - }, inputs.sideTimeoutMs) - child.once("error", () => { - clearTimeout(timer) - sideGuard.track(undefined) - resolve(1) - }) - child.once("exit", (code) => { - clearTimeout(timer) - sideGuard.track(undefined) - resolve(timedOut ? 1 : (code ?? 1)) - }) - }) - if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) - const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) - // The side output must describe exactly this side and its approved commit, - // or the pair is invalid. - return validateSideResultFile(parsed, side, commit) } const inputs = readArgs() @@ -227,36 +140,39 @@ assertCommitsApproved(policy, [baselineCommit, candidateCommit]) const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-compare-")) await fs.mkdir(inputs.out, { recursive: true }) -try { - const report = buildComparisonReport({ - baseline: toSideResult(await runSide("baseline", baselineCommit, inputs, scratchDir)), - candidate: toSideResult(await runSide("candidate", candidateCommit, inputs, scratchDir)), - repetitions: inputs.reps, - }) - // The reports are the comparison's publication point. The pair is - // published as one generation with a completion marker; any failure above - // or during publication leaves no complete report generation behind. - await publishComparisonReport(inputs.out, report) - console.log( - report.compatibility.compatible - ? `comparison complete: compatible pair, report in ${inputs.out}` - : `comparison complete: INCOMPATIBLE pair (${report.compatibility.differences.join(", ")}), report in ${inputs.out}`, - ) -} finally { - // An interrupt escalation must finish before the worktrees go away, so - // no child is still writing while its tree is deleted. - await interruptKill - await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "baseline")], { - cwd: root, - timeout: 30_000, - }).catch(() => {}) - await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "candidate")], { - cwd: root, - timeout: 30_000, - }).catch(() => {}) - await fs.rm(scratchDir, { recursive: true, force: true }).catch(() => {}) -} -// An interrupt terminated the side group above, and the cleanup in the -// finally block has completed. Exit with the conventional signal code. -if (interruptSignal === "SIGINT") process.exit(130) -if (interruptSignal === "SIGTERM") process.exit(143) +// Both sides run inside the flow. The flow cleanup finishes the interrupt +// escalation, removes the worktrees, and removes the scratch directory +// before any exit, so no child is still writing while its tree is deleted. +const outcome = await executeComparisonFlow({ + out: inputs.out, + scratchDir, + root, + reps: inputs.reps, + runSide: (side) => + runCompareSide(side, side === "baseline" ? baselineCommit : candidateCommit, scratchDir, { + guard: sideGuard, + isInterrupted: interruptFlag.isInterrupted, + root, + sideScript: path.join(root, "scripts/agent-evals/coding-side.mts"), + reps: inputs.reps, + trustFile: inputs.trustFile, + credentialEnv: inputs.credentialEnv, + sideTimeoutMs: inputs.sideTimeoutMs, + killGraceMs: SIDE_KILL_GRACE_MS, + prepareWorktree: prepareSideWorktree, + createSideOutDir: async (dir) => { + await fs.mkdir(dir, { recursive: true }) + }, + }), + interruptedSignal: interruptFlag.name, + interruptKill: () => interruptKill ?? Promise.resolve(), +}) +// An interrupt stopped the comparison and the flow cleanup has completed. +// Exit with the conventional signal code on the normal and the throwing +// path alike. +if (outcome.status === "interrupted") process.exit(interruptFlag.exitCode()) +console.log( + outcome.report.compatibility.compatible + ? `comparison complete: compatible pair, report in ${inputs.out}` + : `comparison complete: INCOMPATIBLE pair (${outcome.report.compatibility.differences.join(", ")}), report in ${inputs.out}`, +) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 921b901283..2f8c278d37 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -1,11 +1,12 @@ -import { spawn } from "node:child_process" +import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import path from "node:path" +import { promisify } from "node:util" import type { SidePins } from "./coding-pins.ts" import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" import { DEFAULT_REPETITIONS } from "./coding-stats.ts" import type { CodingRunRecord } from "./coding-run.ts" -import { buildWorktreeEnvironment } from "./coding-trust.ts" +import { buildSideEnvironment, buildWorktreeEnvironment } from "./coding-trust.ts" import type { SideProcessGuard } from "./side-process-guard.ts" import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" @@ -108,6 +109,184 @@ export const runBoundedPreparation = async ( }) } +// The worktree removals in the flow cleanup are best effort and bounded. +const WORKTREE_REMOVE_TIMEOUT_MS = 30_000 + +// Everything one side execution needs from the driver process. The guard and +// the interrupt view are shared with the driver's signal handler; the two +// async steps are seams, so tests can place a signal inside the last boundary +// before the spawn without a repository, a toolchain, or a credential. +export interface CompareSideWiring { + guard: SideProcessGuard + isInterrupted: () => boolean + // Trusted checkout root: the spawn cwd of the side process. + root: string + // The side entry script of this checkout. Production points at + // scripts/agent-evals/coding-side.mts; tests substitute a fixture. + sideScript: string + reps: number + trustFile: string + credentialEnv: string + sideTimeoutMs: number + killGraceMs: number + // Prepares the side's worktree: the detached worktree add of this exact + // commit, the frozen dependency install, and the build of that commit. + prepareWorktree: (commit: string, worktree: string) => Promise + // Creates the side output directory. This is the last async boundary + // before the spawn; tests arm the interrupt inside it. + createSideOutDir: (dir: string) => Promise +} + +// Runs one trusted side against the evaluated worktree. The side script comes +// from this checkout, receives only the allowlisted environment plus the named +// credential variable, re-checks the worktree commit against the trust +// policy, and must finish inside its own wall clock. +export const runCompareSide = async ( + side: "baseline" | "candidate", + commit: string, + scratchDir: string, + wiring: CompareSideWiring, +): Promise => { + if (wiring.isInterrupted()) throw new Error(`interrupted before the ${side} side started`) + const worktree = path.join(scratchDir, side) + await wiring.prepareWorktree(commit, worktree) + if (wiring.isInterrupted()) throw new Error(`interrupted before the ${side} side started`) + const sideOut = path.join(scratchDir, `${side}-out`) + // The directory creation is the last await before the spawn. A signal + // during it arms the guard while nothing is tracked, and the guard never + // revisits a finished escalation, so the side must be refused here rather + // than started into an already-interrupted driver. + await wiring.createSideOutDir(sideOut) + if (wiring.isInterrupted()) throw new Error(`interrupted before the ${side} side started`) + // From this refusal check to the guard track there is no await, so a + // signal cannot slip between them and miss the new child. + const exit = await new Promise((resolve) => { + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + wiring.sideScript, + "--reps", + String(wiring.reps), + "--out", + sideOut, + "--trust-file", + wiring.trustFile, + "--credential-env", + wiring.credentialEnv, + "--evaluated-root", + worktree, + ], + { + cwd: wiring.root, + env: buildSideEnvironment(process.env, wiring.credentialEnv, side), + stdio: "inherit", + detached: true, + }, + ) + wiring.guard.track(child.pid) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGTERM") + } catch { + child.kill("SIGTERM") + } + setTimeout(() => { + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, wiring.killGraceMs).unref() + }, wiring.sideTimeoutMs) + child.once("error", () => { + clearTimeout(timer) + wiring.guard.track(undefined) + resolve(1) + }) + child.once("exit", (code) => { + clearTimeout(timer) + wiring.guard.track(undefined) + resolve(timedOut ? 1 : (code ?? 1)) + }) + }) + if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) + const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) + // The side output must describe exactly this side and its approved commit, + // or the pair is invalid. + return validateSideResultFile(parsed, side, commit) +} + +// The comparison body after the trust gates: both sides, one report +// publication, and the cleanup that must finish before the driver exits. +export interface ComparisonFlowWiring { + out: string + scratchDir: string + root: string + reps: number + // Runs one side to its validated result. The driver wires its own side + // runner; tests substitute synthetic sides. + runSide: (side: "baseline" | "candidate") => Promise + // The observed terminal signal, or undefined. The interrupt outcome and + // the final exit code read this one view. + interruptedSignal: () => NodeJS.Signals | undefined + // The guard's kill escalation, created when the signal arrived. Cleanup + // awaits it so no tracked child outlives the removed worktrees. + interruptKill: () => Promise +} + +export type ComparisonFlowOutcome = + { status: "completed"; report: ComparisonReport } | { status: "interrupted"; signal: "SIGINT" | "SIGTERM" } + +// Runs the comparison and resolves to its outcome after the cleanup. An +// interrupt resolves to its signal no matter how the body ended, so the +// caller exits with the conventional code on the normal and the throwing +// path alike; any other failure stays a thrown failure. +export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promise => { + let completed: ComparisonReport | undefined + let failure: unknown + try { + const report = buildComparisonReport({ + baseline: toSideResult(await wiring.runSide("baseline")), + candidate: toSideResult(await wiring.runSide("candidate")), + repetitions: wiring.reps, + }) + // The reports are the comparison's publication point. The pair is + // published as one generation with a completion marker; any failure + // above or during publication leaves no complete report generation + // behind. + await publishComparisonReport(wiring.out, report) + completed = report + } catch (error) { + // Deferred, not swallowed: the cleanup below runs first, then the + // failure rethrows unless the run was interrupted. + failure = error + } finally { + // An interrupt escalation must finish before the worktrees go away, + // so no child is still writing while its tree is deleted. + await wiring.interruptKill() + await execFileAsync("git", ["worktree", "remove", "--force", path.join(wiring.scratchDir, "baseline")], { + cwd: wiring.root, + timeout: WORKTREE_REMOVE_TIMEOUT_MS, + }).catch(() => {}) + await execFileAsync("git", ["worktree", "remove", "--force", path.join(wiring.scratchDir, "candidate")], { + cwd: wiring.root, + timeout: WORKTREE_REMOVE_TIMEOUT_MS, + }).catch(() => {}) + await fs.rm(wiring.scratchDir, { recursive: true, force: true }).catch(() => {}) + } + const signal = wiring.interruptedSignal() + if (signal === "SIGINT" || signal === "SIGTERM") return { status: "interrupted", signal } + if (failure !== undefined) throw failure + if (completed === undefined) throw new Error("comparison ended without a report or a failure") + return { status: "completed", report: completed } +} + +const execFileAsync = promisify(execFile) + // Serializes a side result for artifact storage. Full run records travel so // each side's traces and grading verdicts stay inspectable. export interface SideResultFile { From 02ce7ffcb2574ba0d685e60d3575e967836eac63 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 27 Oct 2026 00:33:52 +0000 Subject: [PATCH 08/22] fix(agent-evals): detect dirty checkouts despite Git settings that hide changes --- scripts/agent-evals/coding-trust.test.ts | 171 +++++++++++++++++++++-- scripts/agent-evals/coding-trust.ts | 15 +- 2 files changed, 174 insertions(+), 12 deletions(-) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index 9081c034a9..4dae732ec7 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -21,11 +21,25 @@ const execFileAsync = promisify(execFile) const sha = (c: string): string => c.repeat(40) // One throwaway repository with one empty commit. It stands in for an -// evaluated root; no code from it ever runs in these checks. -const makeRepo = async (dir: string): Promise => { - const repo = path.join(dir, "repo") +// evaluated root; no code from it ever runs in these checks. The commit +// identity lives in the repository's local config only. +const makeRepo = async (dir: string, name = "repo"): Promise => { + const repo = path.join(dir, name) await fs.mkdir(repo) await execFileAsync("git", ["init", "-q"], { cwd: repo }) + // The operator's git config may force signed commits, and the + // throwaway fixture has no key or tty. + await execFileAsync("git", ["config", "commit.gpgsign", "false"], { cwd: repo }) + await execFileAsync("git", ["config", "user.email", "fixture@example.test"], { cwd: repo }) + await execFileAsync("git", ["config", "user.name", "fixture"], { cwd: repo }) + await execFileAsync("git", ["commit", "--allow-empty", "-q", "-m", "fixture"], { cwd: repo }) + return repo +} + +const commitAll = async (repo: string, message: string): Promise => { + await execFileAsync("git", ["add", "-A"], { cwd: repo }) + // Submodule checkouts are clones without the fixture's local config, so + // the flags travel with the command instead of any stored config. await execFileAsync( "git", [ @@ -33,19 +47,15 @@ const makeRepo = async (dir: string): Promise => { "user.email=fixture@example.test", "-c", "user.name=fixture", - // The operator's git config may force signed commits, and the - // throwaway fixture has no key or tty. "-c", "commit.gpgsign=false", "commit", - "--allow-empty", "-q", "-m", - "fixture", + message, ], { cwd: repo }, ) - return repo } const writePolicy = async (dir: string, content: unknown): Promise => { @@ -54,6 +64,21 @@ const writePolicy = async (dir: string, content: unknown): Promise => { return file } +// A superproject with one clean submodule checked out from a throwaway local +// origin. Nothing from it runs in these checks. `name` keeps the on-disk +// repositories of separate cases apart inside one temporary directory. +const makeSuperWithSubmodule = async (dir: string, name: string): Promise<{ superRepo: string; sub: string }> => { + const origin = await makeRepo(dir, `${name}-origin`) + await fs.writeFile(path.join(origin, "lib.txt"), "library\n") + await commitAll(origin, "library") + const superRepo = await makeRepo(dir, `${name}-super`) + await execFileAsync("git", ["-c", "protocol.file.allow=always", "submodule", "add", "-q", origin, "sub"], { + cwd: superRepo, + }) + await commitAll(superRepo, "submodule") + return { superRepo, sub: path.join(superRepo, "sub") } +} + describe("trust policy", () => { it("loads a policy of full commit SHAs", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) @@ -239,6 +264,136 @@ describe("evaluated tree cleanliness gate", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it("rejects untracked files when the caller config tells status to hide them", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + // status.showUntrackedFiles=no makes a plain status report an + // empty result; the gate must not trust that setting. + const cases = [ + { + name: "root", + makeDirty: async (repo: string) => { + await fs.writeFile(path.join(repo, "untracked.txt"), "x") + }, + }, + { + name: "nested", + makeDirty: async (repo: string) => { + await fs.mkdir(path.join(repo, "nested")) + await fs.writeFile(path.join(repo, "nested", "untracked.txt"), "x") + }, + }, + ] + for (const [index, { name, makeDirty }] of cases.entries()) { + const repo = await makeRepo(dir, `hide-untracked-${index}`) + await execFileAsync("git", ["config", "status.showUntrackedFiles", "no"], { cwd: repo }) + await makeDirty(repo) + await assert.rejects(assertEvaluatedTreeClean(repo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError) + assert.ok( + error.entries.some((entry) => entry.includes("untracked.txt")), + name, + ) + return true + }) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects staged changes under the same hiding config", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const cases = [ + { + name: "staged new file", + stage: async (repo: string) => { + await fs.writeFile(path.join(repo, "staged.txt"), "x") + await execFileAsync("git", ["add", "staged.txt"], { cwd: repo }) + }, + }, + { + name: "staged modification", + stage: async (repo: string) => { + await fs.writeFile(path.join(repo, "tracked.txt"), "approved\n") + await commitAll(repo, "tracked") + await fs.writeFile(path.join(repo, "tracked.txt"), "changed\n") + await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) + }, + }, + ] + for (const [index, { name, stage }] of cases.entries()) { + const repo = await makeRepo(dir, `staged-${index}`) + await execFileAsync("git", ["config", "status.showUntrackedFiles", "no"], { cwd: repo }) + await stage(repo) + await assert.rejects(assertEvaluatedTreeClean(repo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError, name) + return true + }) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("accepts a clean superproject whose submodule the caller config ignores", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const { superRepo } = await makeSuperWithSubmodule(dir, "clean") + // submodule..ignore=all must not turn the explicit + // --ignore-submodules=none scan into a false rejection. + await execFileAsync("git", ["config", "submodule.sub.ignore", "all"], { cwd: superRepo }) + await assert.doesNotReject(assertEvaluatedTreeClean(superRepo)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects dirty submodules the caller config ignores", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const cases = [ + { + name: "modified submodule content", + makeDirty: async (sub: string) => { + await fs.writeFile(path.join(sub, "lib.txt"), "changed\n") + }, + }, + { + name: "untracked submodule content", + makeDirty: async (sub: string) => { + await fs.writeFile(path.join(sub, "extra.txt"), "x") + }, + }, + { + name: "submodule checked out past the approved commit", + makeDirty: async (sub: string) => { + await fs.writeFile(path.join(sub, "lib.txt"), "advanced\n") + await commitAll(sub, "advance") + }, + }, + ] + for (const [index, { name, makeDirty }] of cases.entries()) { + const { superRepo, sub } = await makeSuperWithSubmodule(dir, `dirty-${index}`) + // ignore=all hides every submodule change from a plain + // status, so only the gate's explicit flag sees the dirt. + await execFileAsync("git", ["config", "submodule.sub.ignore", "all"], { cwd: superRepo }) + await makeDirty(sub) + await assert.rejects(assertEvaluatedTreeClean(superRepo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError) + assert.ok( + error.entries.some((entry) => entry.includes("sub")), + name, + ) + return true + }) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) }) describe("worktree process environment", () => { diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 1b797c68e0..463c8126a1 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -86,10 +86,17 @@ export class DirtyEvaluatedTreeError extends Error { export async function assertEvaluatedTreeClean(root: string): Promise { const { execFile } = await import("node:child_process") const { promisify } = await import("node:util") - const { stdout } = await promisify(execFile)("git", ["status", "--porcelain"], { - cwd: root, - timeout: 30_000, - }) + // The flags override the caller's git configuration: status.showUntrackedFiles + // would otherwise hide untracked files and submodule..ignore would + // otherwise hide submodule dirtiness, so a modified checkout could pass. + const { stdout } = await promisify(execFile)( + "git", + ["status", "--porcelain", "--untracked-files=all", "--ignore-submodules=none"], + { + cwd: root, + timeout: 30_000, + }, + ) const entries = stdout .split("\n") .map((line) => line.trim()) From 980095bdc62783b7e87cfa59f42541885303dbb9 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 27 Oct 2026 01:38:52 +0000 Subject: [PATCH 09/22] fix(agent-evals): isolate side builds and invalidate stale results before artifacts --- scripts/agent-evals/README.md | 10 +- .../agent-evals/coding-compare-driver.test.ts | 19 +- .../agent-evals/coding-compare-script.test.ts | 235 +++++++++---- scripts/agent-evals/coding-compare.mts | 50 ++- scripts/agent-evals/coding-compare.ts | 28 +- scripts/agent-evals/coding-report.test.ts | 54 +++ scripts/agent-evals/coding-side.mts | 222 ++++++++---- scripts/agent-evals/coding-side.test.ts | 331 +++++++++++++++--- scripts/agent-evals/coding-trust.ts | 11 +- 9 files changed, 723 insertions(+), 237 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 1a6ef27435..80064cacd2 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -121,7 +121,7 @@ release process: a maintainer runs it locally or on demand. | `coding-stats.ts` | Three repetitions, success rates with Wilson intervals, paired differences. | | `coding-report.ts` | The self-contained JSON report and the Markdown renderer. | | `coding-compare.ts` | The comparison orchestration and the side result shape. | -| `coding-side.mts` | One comparison side: all fixtures times N repetitions on the evaluated root, graded, with pinned inputs. | +| `coding-side.mts` | One comparison side: rebuilds the evaluated tree, then all fixtures times N repetitions on it, graded, with pinned inputs. | | `coding-compare.mts` | Two-commit comparison driver. Writes `comparison-report.json` and `comparison-report.md`. | | `coding-trust.ts` | The comparison trust policy: approved full commit SHAs, resolution, the approval gate, and the side environment allowlist. | | `fs-atomic.ts` | Atomic file publication through same-directory temp files and renames. | @@ -154,17 +154,17 @@ A comparison runs the extension code of two commits with the provider credential { "version": 1, "approvedCommits": [""] } ``` -The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). +The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. The driver creates each side's fresh detached worktree of the approved commit. Each side then constructs its own fresh detached worktree of that same commit, installs the frozen lockfile dependencies and builds inside it, and launches every host against it, so each side runs only its own compiled extension of the approved commit. The install and the build consume the side's own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It also rejects an evaluated root whose tracked or untracked files differ from that commit, because the approval describes the commit and not the working tree. The check has one trust limit: ignored files, such as dependency installs and build outputs, stay outside commit identity and outside the check, so the approval never attests them. A recorded comparison closes that gap by loading only a fresh detached worktree built from the approved commit. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. -Every driver child, preparation steps and the side alike, runs detached in its own process group and is tracked by one guard. A SIGINT or SIGTERM that reaches only the driver terminates the tracked group through a bounded SIGTERM-then-SIGKILL escalation, and the worktree cleanup waits for that escalation before it removes any tree. +Every driver child, the side included, runs detached in its own process group and is tracked by one guard. The driver's guard tracks the side; the side's guard tracks its own worktree, install, and build. A SIGINT or SIGTERM that reaches only the driver terminates the side group through a bounded SIGTERM-then-SIGKILL escalation. The side starts the same escalation for its preparation children and waits for it before it removes its worktree or exits. The driver's worktree cleanup also waits for the driver's escalation before it removes any tree. ### Output publication contract Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. -A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and the whole `artifacts` tree, so stale results from an earlier rerun can never survive. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. +A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and then the whole `artifacts` tree, so stale results from an earlier rerun can never survive. The authoritative result goes first, so an error or an interruption during invalidation leaves the generation incomplete, and consumers fail closed until the new atomic publication. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. The side process arms one flag when it receives SIGINT or SIGTERM. The flag starts no further repetition and blocks the completion marker, so an interrupted side exits with the conventional signal code and publishes nothing. diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts index fcf7a1f28d..5f05e4ba39 100644 --- a/scripts/agent-evals/coding-compare-driver.test.ts +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -235,26 +235,33 @@ describe("comparison flow outcome and cleanup", () => { }, }), ) - return { outcome, sides, dir } + // Observed before this helper's finally: a removal here would + // mask a flow cleanup that left the scratch directory behind. + const scratchDirExists = await fs.access(dir).then( + () => true, + () => false, + ) + return { outcome, sides, scratchDirExists } } finally { await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) } } it("reports SIGINT and finishes cleanup when a side fails under the interrupt", async () => { - const { outcome, sides, dir } = await interruptedRun("SIGINT") + const { outcome, sides, scratchDirExists } = await interruptedRun("SIGINT") assert.deepEqual(outcome, { status: "interrupted", signal: "SIGINT" }) // The interrupted side stopped the flow before the candidate started. assert.deepEqual(sides, ["baseline"]) - // The cleanup completed before the outcome resolved. - await assert.rejects(fs.access(dir)) + // The observation happened before the helper's own teardown, so a + // false value proves the flow's cleanup removed the directory. + assert.equal(scratchDirExists, false) }) it("reports SIGTERM and finishes cleanup when a side fails under the interrupt", async () => { - const { outcome, sides, dir } = await interruptedRun("SIGTERM") + const { outcome, sides, scratchDirExists } = await interruptedRun("SIGTERM") assert.deepEqual(outcome, { status: "interrupted", signal: "SIGTERM" }) assert.deepEqual(sides, ["baseline"]) - await assert.rejects(fs.access(dir)) + assert.equal(scratchDirExists, false) }) it("keeps a non-interrupt failure a failure and cleans up anyway", async () => { diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts index 411dc20729..6f3217b6d5 100644 --- a/scripts/agent-evals/coding-compare-script.test.ts +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { execFile, execFileSync, spawn } from "node:child_process" +import { execFile, execFileSync, spawn, type ChildProcess } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -163,87 +163,149 @@ interface SignalRun { worktreeGroupPid: number } +interface DriverRunOptions { + // Test seam: bounds the wait for the preparation group's pid file. + pidFileTimeoutMs?: number + // Test seam: bounds the wait for the driver exit after the signal. + exitTimeoutMs?: number + // Test seam: throws after the preparation group recorded itself, so the + // failure cleanup runs with a live driver and a live sleeper group. + failAfterPidFile?: boolean + // Test seam: reports what the failure cleanup observed, so a test can + // prove the group kill and the directory removal without holding state + // the helper already removed. + onFailureCleanup?: (observed: { scratchParent: string; groupPid: number | undefined }) => void +} + +// Reads the group id the shim recorded. Only a plausible group id counts: +// a missing, negative, zero, or tiny value would aim the kill at unrelated +// process groups, so the cleanup skips it instead. +const recordedGroupPid = async (pidFile: string): Promise => { + const raw = await fs.readFile(pidFile, "utf8").catch(() => undefined) + if (raw === undefined) return undefined + const pid = Number(raw.trim()) + return Number.isInteger(pid) && pid > 1 ? pid : undefined +} + +// Terminates the spawned driver and waits for its exit: SIGTERM first, a +// bounded wait, then SIGKILL. The wait proves the process is gone before +// the failure path removes the temporary directory. +const terminateDriver = async (child: ChildProcess): Promise => { + if (child.exitCode !== null || child.signalCode !== null) return + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000) + timer.unref() + child.once("exit", () => { + clearTimeout(timer) + resolve() + }) + if (!child.kill("SIGTERM")) resolve() + }) +} + // Spawns the real comparison driver with a PATH shim. The shim turns // `git worktree add` into a detached sleeper that records its group id, so // the test can aim one signal at the driver alone while a preparation child // runs. Every other git invocation proxies to the real git, and the trust // gates pass with the HEAD commit. The credential is a placeholder; nothing // here contacts a provider. -const runDriverUntilSignal = async (signal: NodeJS.Signals): Promise => { +const runDriverUntilSignal = async (signal: NodeJS.Signals, options: DriverRunOptions = {}): Promise => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-signal-test-")) - const realGit = execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim() - const shimDir = path.join(dir, "bin") - await fs.mkdir(shimDir) - const gitShim = [ - "#!/bin/sh", - 'if [ "$1" = "worktree" ]; then', - ' if [ "$2" = "add" ]; then', - ' printf \'%s\\n\' "$$" > "$TMPDIR/worktree-add-pid"', - " exec sleep 30", - " fi", - ' if [ "$2" = "remove" ]; then', - " exit 0", - " fi", - "fi", - `exec '${realGit}' "$@"`, - ].join("\n") - await fs.writeFile(path.join(shimDir, "git"), gitShim, { mode: 0o755 }) - const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim() - const trustFile = path.join(dir, "trust-policy.json") - await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [head] })) - const child = spawn( - process.execPath, - [ - "--import", - "tsx", - path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), - "--baseline", - head, - "--candidate", - head, - "--reps", - "1", - "--out", - path.join(dir, "out"), - "--trust-file", - trustFile, - ], - { - cwd: repoRoot, - env: { - PATH: `${shimDir}:${process.env["PATH"] ?? ""}`, - HOME: process.env["HOME"] ?? os.homedir(), - TMPDIR: dir, - SOLHEIM_API_KEY: "not-a-real-credential", - }, - stdio: ["ignore", "ignore", "pipe"], - }, - ) - let stderr = "" - child.stderr?.on("data", (chunk: Buffer) => { - stderr += chunk.toString() - }) const pidFile = path.join(dir, "worktree-add-pid") - await waitForFile(pidFile, 60_000) - // One signal aimed at the driver alone. The detached preparation group - // receives nothing from this kill call. - child.kill(signal) - const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { - const timer = setTimeout(() => { - child.kill("SIGKILL") - reject(new Error(`driver did not exit after ${signal}: ${stderr}`)) - }, 60_000) - child.once("exit", (code, exitSignal) => { - clearTimeout(timer) - resolve({ code, signal: exitSignal }) + let child: ChildProcess | undefined + try { + const realGit = execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim() + const shimDir = path.join(dir, "bin") + await fs.mkdir(shimDir) + const gitShim = [ + "#!/bin/sh", + 'if [ "$1" = "worktree" ]; then', + ' if [ "$2" = "add" ]; then', + ' printf \'%s\\n\' "$$" > "$TMPDIR/worktree-add-pid"', + " exec sleep 30", + " fi", + ' if [ "$2" = "remove" ]; then', + " exit 0", + " fi", + "fi", + `exec '${realGit}' "$@"`, + ].join("\n") + await fs.writeFile(path.join(shimDir, "git"), gitShim, { mode: 0o755 }) + const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim() + const trustFile = path.join(dir, "trust-policy.json") + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [head] })) + child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), + "--baseline", + head, + "--candidate", + head, + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + ], + { + cwd: repoRoot, + env: { + PATH: `${shimDir}:${process.env["PATH"] ?? ""}`, + HOME: process.env["HOME"] ?? os.homedir(), + TMPDIR: dir, + SOLHEIM_API_KEY: "not-a-real-credential", + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() }) - child.once("error", (error) => { - clearTimeout(timer) - reject(error) + await waitForFile(pidFile, options.pidFileTimeoutMs ?? 60_000) + if (options.failAfterPidFile) throw new Error("injected failure after the preparation group recorded itself") + // One signal aimed at the driver alone. The detached preparation group + // receives nothing from this kill call. + child.kill(signal) + const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { + const timer = setTimeout(() => { + child?.kill("SIGKILL") + reject(new Error(`driver did not exit after ${signal}: ${stderr}`)) + }, options.exitTimeoutMs ?? 60_000) + child?.once("exit", (code, exitSignal) => { + clearTimeout(timer) + resolve({ code, signal: exitSignal }) + }) + child?.once("error", (error) => { + clearTimeout(timer) + reject(error) + }) }) - }) - const groupPid = Number(await fs.readFile(pidFile, "utf8")) - return { exitCode: exit.code, stderr, scratchParent: dir, worktreeGroupPid: groupPid } + const groupPid = Number(await fs.readFile(pidFile, "utf8")) + return { exitCode: exit.code, stderr, scratchParent: dir, worktreeGroupPid: groupPid } + } catch (error) { + // A failure here used to leak both the detached preparation group and + // the temporary directory. The recorded group dies first, but only + // when the shim discovered it: an unknown or implausible pid must + // never aim a negative-pid kill at unrelated groups. The driver is + // then terminated and waited for, and the directory is removed. + const groupPid = await recordedGroupPid(pidFile) + if (groupPid !== undefined) { + try { + process.kill(-groupPid, "SIGKILL") + } catch { + /* The group already exited. */ + } + } + if (child) await terminateDriver(child) + options.onFailureCleanup?.({ scratchParent: dir, groupPid }) + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + throw error + } } describe("compare driver signal exit", () => { @@ -280,4 +342,33 @@ describe("compare driver signal exit", () => { await expectSignalExit("SIGTERM", 143) }, ) + + it( + "kills the preparation group and removes the scratch directory when a run fails mid-wait", + { timeout: 180_000 }, + async () => { + let observed: { scratchParent: string; groupPid: number | undefined } | undefined + await assert.rejects( + runDriverUntilSignal("SIGINT", { + failAfterPidFile: true, + onFailureCleanup: (info) => { + observed = info + }, + }), + /injected failure after the preparation group recorded itself/, + ) + assert.ok(observed) + // The shim had discovered its group, so the cleanup had a real + // group id and the sleeper cannot have survived. + assert.equal(typeof observed.groupPid, "number") + await assertGroupGone(observed.groupPid as number) + // The driver was terminated and waited for before the scratch + // directory was removed. + const scratchParentExists = await fs.access(observed.scratchParent).then( + () => true, + () => false, + ) + assert.equal(scratchParentExists, false) + }, + ) }) diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index 226dccfbc8..ccb01f450f 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -1,8 +1,10 @@ // Paired comparison driver. Verifies both commits against an explicit trust -// policy, checks out each approved commit into a temporary worktree, prepares -// pinned dependencies and builds that exact commit, runs one trusted coding -// side per commit against its worktree with identical repetition counts, then -// writes the JSON and Markdown comparison reports. +// policy, checks out each approved commit into a fresh detached temporary +// worktree, and runs one trusted coding side per commit with identical +// repetition counts. Each side then constructs its own fresh detached +// worktree of that exact commit, installs the frozen dependencies, and +// builds inside it before any host launch. The driver writes the JSON and +// Markdown comparison reports. // // Trust model. The side script, fixtures, grader, and report builder all run // from this trusted checkout. Only the extension host loads the evaluated @@ -10,14 +12,17 @@ // the one named provider variable. A commit runs only when its full SHA is // listed in the operator's trust policy; refs are resolved and checked // against the policy before any worktree or process exists. The side process -// re-checks the root it was handed against the same policy. +// re-checks the commit it was handed against the same policy, constructs a +// fresh detached worktree of it, and installs and builds inside that +// worktree before any launch. // -// Process containment. Install, build, worktree, and side children receive -// only an allowlisted environment, so the driver's secrets never reach them. -// Each runs detached in its own process group under one guard, so SIGINT and -// SIGTERM aimed at the driver alone terminate the tracked group with a bounded -// SIGTERM-then-SIGKILL escalation before the scratch cleanup runs, and an -// interrupt leaves no orphan host behind. +// Process containment. Worktree, side, and the sides' install and build +// children receive only an allowlisted environment, so the driver's secrets +// never reach them. Each runs detached in its own process group under a +// guard, so SIGINT and SIGTERM aimed at the driver alone terminate the side +// group with a bounded SIGTERM-then-SIGKILL escalation before the scratch +// cleanup runs, the side runs the same escalation for its own children, and +// an interrupt leaves no orphan host behind. import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -28,8 +33,6 @@ import { createInterruptFlag, createSideProcessGuard, type SideKillOutcome } fro const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") -const INSTALL_TIMEOUT_MS = 10 * 60_000 -const BUILD_TIMEOUT_MS = 30 * 60_000 const WORKTREE_TIMEOUT_MS = 120_000 const SIDE_KILL_GRACE_MS = 30_000 const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 @@ -98,23 +101,11 @@ const readArgs = (): { } } -// Prepares the worktree of one evaluated commit. Dependencies install from -// the worktree's own frozen lockfile, and that exact commit's extension and -// webview build from its own sources. No artifact crosses sides. -const prepareWorktree = async (worktree: string): Promise => { - await runBoundedPreparation( - sideGuard, - interruptFlag.isInterrupted, - "pnpm", - ["install", "--frozen-lockfile"], - worktree, - INSTALL_TIMEOUT_MS, - ) - await runBoundedPreparation(sideGuard, interruptFlag.isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) -} - // One side's full preparation: the detached worktree add of that exact -// commit, then the install and build inside it. +// commit. The side derives its own fresh worktree of this commit, installs +// the frozen dependencies, and builds there before any host launch, so the +// executed build output is always the side's own product of the approved +// commit. const prepareSideWorktree = async (commit: string, worktree: string): Promise => { await runBoundedPreparation( sideGuard, @@ -124,7 +115,6 @@ const prepareSideWorktree = async (commit: string, worktree: string): Promise Promise // Creates the side output directory. This is the last async boundary // before the spawn; tests arm the interrupt inside it. @@ -420,12 +421,19 @@ export async function readPublishedComparisonReport(outDir: string): Promise { - await fs.rm(path.join(outDir, "artifacts"), { recursive: true, force: true }) - await fs.rm(path.join(outDir, sideResultFileName()), { force: true }) +// Removes the previous side generation before a side executes. The +// authoritative result is removed first, so every stop after that point, an +// error or an interruption alike, leaves the generation incomplete: consumers +// require side-results.json, and it exists only after the new atomic +// publication completes. A result that cannot be removed, or an artifacts +// tree that cannot be removed after it, fails the side before any run spends +// budget. +export type SideArtifactRemover = (target: string, options: { force?: boolean; recursive?: boolean }) => Promise + +export async function invalidateSideArtifacts( + outDir: string, + remove: SideArtifactRemover = (target, options) => fs.rm(target, options), +): Promise { + await remove(path.join(outDir, sideResultFileName()), { force: true }) + await remove(path.join(outDir, "artifacts"), { recursive: true, force: true }) } diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 7dee41eba0..13f1e3c96a 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -11,6 +11,7 @@ import { readPublishedComparisonReport, runComparison, validateSideResultFile, + type SideArtifactRemover, type SideExecutor, } from "./coding-compare.ts" import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" @@ -489,4 +490,57 @@ describe("side artifact invalidation", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it("removes the authoritative side result before the artifacts tree", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + await fs.mkdir(path.join(dir, "artifacts", "clamp-range-rep4"), { recursive: true }) + const removed: string[] = [] + const remove: SideArtifactRemover = async (target, options) => { + removed.push(path.relative(dir, target)) + await fs.rm(target, options) + } + await invalidateSideArtifacts(dir, remove) + assert.deepEqual(removed, ["side-results.json", "artifacts"]) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("fails closed when artifact removal fails after the result is gone", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + const remove: SideArtifactRemover = (target, options) => + target === path.join(dir, "artifacts") + ? Promise.reject(new Error("artifact removal blocked")) + : fs.rm(target, options) + await assert.rejects(invalidateSideArtifacts(dir, remove), /artifact removal blocked/) + // The authoritative result is already gone, so the generation is + // incomplete and every consumer fails closed until the new atomic + // publication. + await assert.rejects(fs.access(path.join(dir, "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("leaves publication working after invalidation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + await fs.mkdir(path.join(dir, "artifacts", "old"), { recursive: true }) + await invalidateSideArtifacts(dir) + assert.deepEqual(await fs.readdir(dir), []) + // The new generation publishes atomically into the invalidated + // directory and is complete again. + await writeJsonAtomic(path.join(dir, "side-results.json"), { pins: { fresh: true } }) + const parsed = JSON.parse(await fs.readFile(path.join(dir, "side-results.json"), "utf8")) as unknown + assert.deepEqual(parsed, { pins: { fresh: true } }) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) }) diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 52b53747f7..07326eaffb 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -2,12 +2,22 @@ // commit through the substrate, grades each final workspace, and writes one // side-results file with the pinned inputs. Trusted repo code only: the // provider credential stays with this process and the extension host; the -// grader never receives it. The extension under test comes from the evaluated -// root, which must be an approved commit and a tree that matches that commit -// exactly before any host is created or any credential is read. +// grader never receives it. +// +// The extension under test is never a caller-prepared tree. The side resolves +// the evaluated root's HEAD commit, requires its approval, and then +// constructs a fresh detached worktree of that exact commit in private +// scratch space. The frozen install, the build, and every host launch run +// inside that worktree, so the executed content is only this side's own +// product of the approved commit. Dirty files, untracked files, ignored +// dependencies, stale build output, and symlinks in the caller's tree can +// never reach it. +import { execFile } from "node:child_process" import fs from "node:fs/promises" +import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" +import { promisify } from "node:util" import { createHostRuntime } from "./launcher.ts" import { createContainerGradingRuntime } from "./container-runtime.ts" import { runCodingFixtureOnce } from "./coding-run.ts" @@ -16,7 +26,7 @@ import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from import { GRADER_VERSION } from "./coding-grader.ts" import { digestValue } from "./coding-digest.ts" import type { SidePins } from "./coding-pins.ts" -import { invalidateSideArtifacts } from "./coding-compare.ts" +import { invalidateSideArtifacts, runBoundedPreparation } from "./coding-compare.ts" import { assertCommitsApproved, assertCredentialEnvAllowed, @@ -25,10 +35,21 @@ import { resolveCommit, } from "./coding-trust.ts" import { executeSideFlow, type SideRepetition } from "./coding-side-flow.ts" -import { createInterruptFlag } from "./side-process-guard.ts" +import { createInterruptFlag, createSideProcessGuard } from "./side-process-guard.ts" + +const execFileAsync = promisify(execFile) const thisRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +// The side owns the worktree construction, the frozen install, and the build +// of the approved commit. All steps consume the side's own wall clock, and a +// step past its bound is killed as a group like an expired run. +const WORKTREE_TIMEOUT_MS = 120_000 +const INSTALL_TIMEOUT_MS = 10 * 60_000 +const BUILD_TIMEOUT_MS = 30 * 60_000 +const WORKTREE_REMOVE_TIMEOUT_MS = 30_000 +const SIDE_KILL_GRACE_MS = 30_000 + const readArgs = (): { reps: number out: string @@ -59,8 +80,9 @@ const readArgs = (): { reps: Number(read("reps") ?? "3"), out: path.resolve(out), credentialEnv, - // Repository root of the extension under test. The trusted harness - // code drives the run; only the extension host loads this tree. + // Repository whose HEAD commit this side evaluates. The trusted + // harness code drives the run; only the extension host loads the + // fresh worktree this side builds from the approved commit. evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), trustFile: path.resolve(trustFile), } @@ -92,19 +114,15 @@ const readVscodeTarget = async (root: string): Promise => { const { reps, out, credentialEnv, evaluatedRoot, trustFile } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") const sideLabel = readSideLabel() -// Trust gate. The evaluated root is loaded by the extension host and receives -// the credential, so its resolved full commit must be approved before any -// directory is prepared, any host is created, or the credential is read. The -// comparison driver checks its resolved commits first; this second check binds -// the root that was actually passed in. The commit of an unapproved root is -// unreadable or foreign, and both fail closed here. +// Trust gate. The evaluated root names the repository whose HEAD commit this +// side evaluates, so that commit must be approved before any directory is +// prepared, any host is created, or the credential is read. The comparison +// driver checks its resolved commits first; this second check binds the root +// that was actually passed in. The commit of an unapproved root is unreadable +// or foreign, and both fail closed here. const policy = await loadTrustPolicy(trustFile) const evaluatedCommit = await resolveCommit("HEAD", evaluatedRoot) assertCommitsApproved(policy, [evaluatedCommit]) -// The approval covers the commit only, so the tree the host loads must also -// match that commit. A tracked or untracked change makes the executed files -// differ from the approved content, and the side stops here. -await assertEvaluatedTreeClean(evaluatedRoot) // Fail before any directory, run, or result file exists. An empty credential // would otherwise produce a plausible file of infrastructure failures. const credential = process.env[credentialEnv] @@ -114,14 +132,21 @@ if (credential === undefined || credential.length === 0) { ensureCodingSuiteRegistered() // Side-level interrupt. A SIGINT or SIGTERM that reaches this process arms -// one flag at most once. The flow checks it before each repetition and -// before publication, so an interrupted side starts no new repetition and -// never publishes the completion marker. The substrate registers its own -// host teardown listener per run; the two listeners stay independent and -// the flag never consumes their signals. +// one flag at most once and starts the preparation guard's escalation, so +// a bounded install or build never outlives the side. The flow checks the +// flag before each repetition and before publication, so an interrupted +// side starts no new repetition and never publishes the completion marker. +// The substrate registers its own host teardown listener per run; the +// listeners stay independent and the flag never consumes their signals. const interrupt = createInterruptFlag() -process.on("SIGINT", () => interrupt.signal("SIGINT")) -process.on("SIGTERM", () => interrupt.signal("SIGTERM")) +const prepGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) +let prepSettled: Promise = Promise.resolve() +const onSideInterrupt = (signal: NodeJS.Signals): void => { + interrupt.signal(signal) + prepSettled = prepGuard.interrupt() +} +process.on("SIGINT", () => onSideInterrupt("SIGINT")) +process.on("SIGTERM", () => onSideInterrupt("SIGTERM")) // Live sides always grade in the container runtime. The local subprocess // runtime stays available to unit tests only. @@ -133,6 +158,25 @@ const gradingRuntime = createContainerGradingRuntime() // completes. await invalidateSideArtifacts(out) +// Private scratch space for the fresh evaluated tree. The caller's root +// contributes only the approved commit object; nothing from its working +// tree, its ignored dependencies, or its build output is copied, linked, or +// executed. +const sideScratch = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-side-")) +const evaluatedWorktree = path.join(sideScratch, "worktree") + +// Removal is best effort and bounded, like the comparison flow cleanup. The +// escalation of an in-flight preparation child settles before any tree is +// removed, so no child is still writing while its tree is deleted. +const removePreparedTree = async (): Promise => { + await prepSettled + await execFileAsync("git", ["worktree", "remove", "--force", evaluatedWorktree], { + cwd: evaluatedRoot, + timeout: WORKTREE_REMOVE_TIMEOUT_MS, + }).catch(() => {}) + await fs.rm(sideScratch, { recursive: true, force: true }).catch(() => {}) +} + const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} const repetitions: SideRepetition[] = CODING_FIXTURE_BUNDLES.flatMap((bundle) => @@ -160,44 +204,102 @@ const buildPins = async (): Promise => { runnerEnvironment: { os: process.platform, nodeVersion: process.version, - vscodeTarget: await readVscodeTarget(evaluatedRoot), + vscodeTarget: await readVscodeTarget(evaluatedWorktree), }, } } -const { runs, completed } = await executeSideFlow({ - repetitions, - out, - isInterrupted: interrupt.isInterrupted, - runRepetition: async ({ fixtureName, repIndex }) => { - const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) - await fs.mkdir(artifactsDir, { recursive: true }) - return runCodingFixtureOnce({ - fixtureName, - repIndex, - credential, - runtime: createHostRuntime({ - root: evaluatedRoot, - artifactsDir, - vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), - }), - gradingRuntime, - scratchDir: artifactsDir, - }) - }, - // Publication point and completion marker. side-results.json appears - // atomically only after every repetition of this invocation finished, so - // a reader that sees the file sees a complete side of the current run. A - // missing file means an interrupted side; consumers must treat it as a - // failed side and never pair partial output. - buildResult: async (runs) => ({ pins: await buildPins(), runs }), -}) - -if (!completed) { - // An interrupted side exits with the conventional signal code and without - // side-results.json, so consumers treat it as a failed side. - console.error(`coding-evals side interrupted by ${interrupt.name() ?? "signal"}; side-results.json not published`) - process.exit(interrupt.exitCode()) +// Everything after the gates runs under one cleanup: the prepared tree is +// removed on completion, on failure, and on interruption alike, and an +// interrupted side exits with the conventional signal code after that +// removal. A thrown failure is deferred until the removal finished, so no +// error path leaks the worktree. +let interruptedExitCode: number | undefined +let failure: unknown +try { + await runBoundedPreparation( + prepGuard, + interrupt.isInterrupted, + "git", + ["worktree", "add", "--detach", evaluatedWorktree, evaluatedCommit], + evaluatedRoot, + WORKTREE_TIMEOUT_MS, + ) + // The approval names one commit, so the constructed tree must resolve to + // exactly that commit and hold no other content. + const preparedCommit = await resolveCommit("HEAD", evaluatedWorktree) + if (preparedCommit !== evaluatedCommit) { + throw new Error(`prepared evaluated worktree is at ${preparedCommit}, not the approved ${evaluatedCommit}`) + } + await assertEvaluatedTreeClean(evaluatedWorktree) + // The install and the build run inside the fresh worktree, so the entry + // the host loads is always this side's own product of the approved + // commit. The preparation children receive only the allowlisted + // environment, so the credential never reaches them. + await runBoundedPreparation( + prepGuard, + interrupt.isInterrupted, + "pnpm", + ["install", "--frozen-lockfile"], + evaluatedWorktree, + INSTALL_TIMEOUT_MS, + ) + await runBoundedPreparation( + prepGuard, + interrupt.isInterrupted, + "pnpm", + ["build"], + evaluatedWorktree, + BUILD_TIMEOUT_MS, + ) + + const { runs, completed } = await executeSideFlow({ + repetitions, + out, + isInterrupted: interrupt.isInterrupted, + runRepetition: async ({ fixtureName, repIndex }) => { + const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) + await fs.mkdir(artifactsDir, { recursive: true }) + return runCodingFixtureOnce({ + fixtureName, + repIndex, + credential, + runtime: createHostRuntime({ + root: evaluatedWorktree, + artifactsDir, + vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), + }), + gradingRuntime, + scratchDir: artifactsDir, + }) + }, + // Publication point and completion marker. side-results.json appears + // atomically only after every repetition of this invocation finished, + // so a reader that sees the file sees a complete side of the current + // run. A missing file means an interrupted side; consumers must treat + // it as a failed side and never pair partial output. + buildResult: async (runs) => ({ pins: await buildPins(), runs }), + }) + + if (!completed) { + console.error( + `coding-evals side interrupted by ${interrupt.name() ?? "signal"}; side-results.json not published`, + ) + interruptedExitCode = interrupt.exitCode() + } else { + const correct = runs.filter((record) => record.outcome.outcome === "correct").length + console.log( + `coding-evals side complete: ${correct}/${runs.length} correct, suite ${codingSuiteDigest().slice(0, 12)}`, + ) + } +} catch (error) { + if (interrupt.isInterrupted()) { + console.error("coding-evals side interrupted during preparation; side-results.json not published") + interruptedExitCode = interrupt.exitCode() + } else { + failure = error + } } -const correct = runs.filter((record) => record.outcome.outcome === "correct").length -console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${codingSuiteDigest().slice(0, 12)}`) +await removePreparedTree() +if (failure !== undefined) throw failure +if (interruptedExitCode !== undefined) process.exit(interruptedExitCode) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts index 418e79f4b0..f00b5d47d8 100644 --- a/scripts/agent-evals/coding-side.test.ts +++ b/scripts/agent-evals/coding-side.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { execFile } from "node:child_process" +import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -56,12 +56,21 @@ const writePolicy = async (dir: string, approvedCommits: string[]): Promise => { +// One fixture commit in a throwaway repository. It stands in for an +// evaluated root; no fixture code from it ever runs in these checks. The +// optional files become part of the committed tree. +const makeFixtureCommit = async ( + dir: string, + files: Record = {}, +): Promise<{ repo: string; commit: string }> => { const repo = path.join(dir, "fixture-repo") await fs.mkdir(repo) + for (const [name, content] of Object.entries(files)) { + await fs.mkdir(path.dirname(path.join(repo, name)), { recursive: true }) + await fs.writeFile(path.join(repo, name), content) + } await execFileAsync("git", ["init", "-q"], { cwd: repo }) + await execFileAsync("git", ["add", "-A"], { cwd: repo }) await execFileAsync( "git", [ @@ -86,48 +95,6 @@ const makeFixtureCommit = async (dir: string): Promise<{ repo: string; commit: s } describe("coding side trust gate", () => { - it( - "rejects a dirty approved evaluated root before any host launch or publication", - { timeout: 180_000 }, - async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) - try { - const { repo, commit } = await makeFixtureCommit(dir) - const trustFile = await writePolicy(dir, [commit]) - // The approved checkout is modified after approval: one staged - // tracked change and one untracked file. The commit identity is - // still approved, but it no longer describes the tree. - await fs.writeFile(path.join(repo, "tracked.txt"), "changed") - await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) - await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") - // The fake credential only proves the failure is not the - // credential check: the cleanliness gate must stop the side first. - const outcome = await runSideScript( - [ - "--reps", - "1", - "--out", - path.join(dir, "out"), - "--trust-file", - trustFile, - "--evaluated-root", - repo, - ], - minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), - ) - assert.notEqual(outcome.code, 0) - assert.match(outcome.stderr, /tracked or untracked changes/) - assert.ok(!outcome.stdout.includes("coding-evals side complete")) - // The rejection happens before the publication point: no side - // results and no artifacts exist. - await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) - await assert.rejects(fs.access(path.join(dir, "out", "artifacts"))) - } finally { - await fs.rm(dir, { recursive: true, force: true }) - } - }, - ) - it("rejects an unapproved evaluated root before any host launch", { timeout: 180_000 }, async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) try { @@ -190,8 +157,8 @@ describe("coding side trust gate", () => { const { repo, commit } = await makeFixtureCommit(dir) const trustFile = await writePolicy(dir, [commit]) // No credential variable is set. The gate must pass and the run - // must fail closed at the credential check, before any host or - // container work. + // must fail closed at the credential check, before any worktree + // or container work. const outcome = await runSideScript( [ "--reps", @@ -249,3 +216,271 @@ describe("coding side isolation gate", () => { } }) }) + +// The install behavior of the shimmed toolchain: "succeed" runs the benign +// build, "fail" exits nonzero after the install starts, and "hang" blocks so +// the test can interrupt the side mid-preparation. +type InstallBehavior = "succeed" | "fail" | "hang" + +// A pnpm shim that records each invocation, its working directory, and its +// environment, so a test can prove the side prepared and built a fresh tree +// without the credential. The build step writes the benign entry the real +// toolchain would produce, relative to the tree it actually ran in. +const writePnpmShim = async (dir: string, install: InstallBehavior): Promise => { + const shimDir = path.join(dir, "bin") + await fs.mkdir(shimDir) + const log = (name: string): string => JSON.stringify(path.join(dir, name)) + const lines = [ + "#!/bin/sh", + `printf '%s\\n' "$*" >> ${log("pnpm-args.log")}`, + 'case "$1" in', + " install)", + ` pwd > ${log("pnpm-install-cwd.log")}`, + ` env > ${log("pnpm-env.log")}`, + ] as string[] + if (install === "hang") lines.push(" sleep 30") + if (install === "fail") lines.push(" exit 1") + lines.push( + " exit 0", + " ;;", + " build)", + ` pwd > ${log("pnpm-build-cwd.log")}`, + ` git rev-parse HEAD > ${log("pnpm-build-head.log")}`, + " mkdir -p src/dist", + " printf 'benign extension entry\\n' > src/dist/extension.js", + " exit 0", + " ;;", + "esac", + "exit 1", + ) + await fs.writeFile(path.join(shimDir, "pnpm"), lines.join("\n"), { mode: 0o755 }) + return shimDir +} + +const preparedEnv = (dir: string, shimDir: string): NodeJS.ProcessEnv => + minimalEnv({ SOLHEIM_API_KEY: "sk-test", PATH: `${shimDir}:${process.env["PATH"] ?? ""}` }) + +const preparedArgs = (dir: string, repo: string, trustFile: string): string[] => [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, +] + +const runPreparedSide = async (dir: string, repo: string, trustFile: string): Promise => { + const shimDir = await writePnpmShim(dir, "succeed") + return runSideScript(preparedArgs(dir, repo, trustFile), preparedEnv(dir, shimDir)) +} + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +const waitForFile = async (filePath: string, timeoutMs: number): Promise => { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if ( + await fs.access(filePath).then( + () => true, + () => false, + ) + ) + return + await delay(20) + } + throw new Error(`fixture file never appeared: ${filePath}`) +} + +// The repository must hold no linked worktree: its only entry is the main +// tree itself. +const assertSingleWorktree = async (repo: string): Promise => { + const { stdout } = await execFileAsync("git", ["worktree", "list", "--porcelain"], { cwd: repo }) + const listed = stdout.split("\n").filter((line) => line.startsWith("worktree ")) + assert.equal(listed.length, 1) +} + +// The install and the build must have run in one fresh worktree of the +// approved commit, never in the caller tree. Returns the worktree path, so +// the caller can prove its removal. +const assertPreparationRanInFreshTree = async (dir: string, repo: string, commit: string): Promise => { + const args = (await fs.readFile(path.join(dir, "pnpm-args.log"), "utf8")).trim().split("\n") + assert.deepEqual(args, ["install --frozen-lockfile", "build"]) + const installCwd = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + const buildCwd = (await fs.readFile(path.join(dir, "pnpm-build-cwd.log"), "utf8")).trim() + const buildHead = (await fs.readFile(path.join(dir, "pnpm-build-head.log"), "utf8")).trim() + assert.equal(installCwd, buildCwd) + assert.notEqual(installCwd, repo) + assert.ok(path.basename(path.dirname(installCwd)).startsWith("coding-evals-side-")) + assert.equal(buildHead, commit) + return installCwd +} + +interface SideResultsFile { + pins: { commit: string } + runs: { outcome: { outcome: string } }[] +} + +// The launch fails deterministically because the fixture has no host entry, +// so every repetition is a non-correct record and the side still publishes. +const assertPublishedFailedRuns = async (dir: string, commit: string): Promise => { + const results = JSON.parse(await fs.readFile(path.join(dir, "out", "side-results.json"), "utf8")) as SideResultsFile + assert.equal(results.pins.commit, commit) + assert.ok(results.runs.length >= 1) + for (const run of results.runs) assert.notEqual(run.outcome.outcome, "correct") +} + +describe("coding side evaluated tree preparation", () => { + it( + "builds a fresh detached worktree of the approved commit and never loads the caller tree", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { + ".gitignore": "src/dist\nnode_modules\n", + "tracked.txt": "committed", + }) + const trustFile = await writePolicy(dir, [commit]) + // Everything planted below stays outside the approval. The + // tracked change and the untracked file make the caller tree + // dirty. The ignored entry and the ignored dependency bin are + // executable artifacts a real build may never overwrite. If + // any of them ran, its marker file would exist. + await fs.writeFile(path.join(repo, "tracked.txt"), "tampered") + await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") + const plantedEntry = `require("node:fs").writeFileSync(${JSON.stringify(path.join(repo, "planted-entry-marker"))}, "executed")\n` + await fs.mkdir(path.join(repo, "src/dist"), { recursive: true }) + await fs.writeFile(path.join(repo, "src/dist/extension.js"), plantedEntry) + await fs.mkdir(path.join(repo, "node_modules/.bin"), { recursive: true }) + await fs.writeFile( + path.join(repo, "node_modules/.bin/evil"), + `#!/bin/sh\nprintf executed > ${JSON.stringify(path.join(repo, "planted-bin-marker"))}\n`, + { mode: 0o755 }, + ) + const outcome = await runPreparedSide(dir, repo, trustFile) + assert.equal(outcome.code, 0) + assert.ok(outcome.stdout.includes("coding-evals side complete")) + // No planted artifact executed, and the side never wrote into + // the caller tree. + await assert.rejects(fs.access(path.join(repo, "planted-entry-marker"))) + await assert.rejects(fs.access(path.join(repo, "planted-bin-marker"))) + assert.equal(await fs.readFile(path.join(repo, "src/dist/extension.js"), "utf8"), plantedEntry) + await fs.access(path.join(repo, "node_modules/.bin/evil")) + // The install and the build ran in one fresh worktree of the + // approved commit, never in the caller tree. + const worktree = await assertPreparationRanInFreshTree(dir, repo, commit) + // Cleanup on the normal path: the worktree and its scratch are + // gone, and the caller repository holds no linked worktree. + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // The pins record the approved commit, and the preparation + // children never saw the credential. + await assertPublishedFailedRuns(dir, commit) + const envLog = await fs.readFile(path.join(dir, "pnpm-env.log"), "utf8") + assert.ok(!envLog.includes("SOLHEIM_API_KEY=")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) + + it( + "accepts a clean approved checkout and builds it in a fresh worktree before the first run", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\n" }) + const trustFile = await writePolicy(dir, [commit]) + const outcome = await runPreparedSide(dir, repo, trustFile) + assert.equal(outcome.code, 0) + assert.ok(outcome.stdout.includes("coding-evals side complete")) + const worktree = await assertPreparationRanInFreshTree(dir, repo, commit) + await assert.rejects(fs.access(worktree)) + await assertSingleWorktree(repo) + await assertPublishedFailedRuns(dir, commit) + const envLog = await fs.readFile(path.join(dir, "pnpm-env.log"), "utf8") + assert.ok(!envLog.includes("SOLHEIM_API_KEY=")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) + + it("removes its worktree when preparation fails", { timeout: 240_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n" }) + const trustFile = await writePolicy(dir, [commit]) + const shimDir = await writePnpmShim(dir, "fail") + const outcome = await runSideScript(preparedArgs(dir, repo, trustFile), preparedEnv(dir, shimDir)) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /pnpm failed: code 1/) + // The failing install names the worktree it ran in; that tree and + // its scratch must be gone, and the repository must hold no linked + // worktree. + const worktree = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // The build never ran and no results exist. + await assert.rejects(fs.access(path.join(repo, "src/dist"))) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it( + "removes its worktree when interrupted during preparation and exits with the signal code", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n" }) + const trustFile = await writePolicy(dir, [commit]) + const shimDir = await writePnpmShim(dir, "hang") + let stderr = "" + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-side.mts"), + ...preparedArgs(dir, repo, trustFile), + ], + { + cwd: repoRoot, + env: preparedEnv(dir, shimDir), + stdio: ["ignore", "ignore", "pipe"], + }, + ) + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + const exitCode = new Promise((resolve, reject) => { + child.once("error", reject) + child.once("exit", (code) => resolve(code ?? 1)) + }) + // The install is in flight when the signal arrives. A SIGINT that + // reaches only the side must stop the preparation group, remove + // the worktree, and exit with the conventional code. + await waitForFile(path.join(dir, "pnpm-install-cwd.log"), 60_000) + child.kill("SIGINT") + assert.equal(await exitCode, 130) + assert.match(stderr, /interrupted during preparation/) + const worktree = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // An interrupted side never publishes the completion marker. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 463c8126a1..fb55cccfd8 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -77,12 +77,11 @@ export class DirtyEvaluatedTreeError extends Error { } } -// Gate before any host launch. The evaluated root must match its approved -// commit exactly, so a modified checkout can never run under that commit's -// approval. Ignored files, such as dependency installs and build outputs, -// stay outside commit identity and outside this check: the approval does not -// attest them. The comparison driver closes that gap by loading only a fresh -// detached worktree built from the approved commit. +// Gate before any host launch. The executed tree must match its approved +// commit exactly, so modified or planted content can never run under that +// commit's approval. The side runs this check against the fresh detached +// worktree it constructed from the approved commit. The caller's own tree is +// never executed at all, so its ignored files need no approval. export async function assertEvaluatedTreeClean(root: string): Promise { const { execFile } = await import("node:child_process") const { promisify } = await import("node:util") From 8210012ef842669e5eb860e59e6a554142f12b8b Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 30 Oct 2026 02:50:07 +0000 Subject: [PATCH 10/22] fix(evals): wait for preparation cleanup after repeated interrupts --- scripts/agent-evals/coding-side.mts | 3 +- scripts/agent-evals/coding-side.test.ts | 139 +++++++++++++++--------- 2 files changed, 91 insertions(+), 51 deletions(-) diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 07326eaffb..90e73cbc51 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -142,8 +142,9 @@ const interrupt = createInterruptFlag() const prepGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) let prepSettled: Promise = Promise.resolve() const onSideInterrupt = (signal: NodeJS.Signals): void => { + const first = !interrupt.isInterrupted() interrupt.signal(signal) - prepSettled = prepGuard.interrupt() + if (first) prepSettled = prepGuard.interrupt() } process.on("SIGINT", () => onSideInterrupt("SIGINT")) process.on("SIGTERM", () => onSideInterrupt("SIGTERM")) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts index f00b5d47d8..239a3b2a5b 100644 --- a/scripts/agent-evals/coding-side.test.ts +++ b/scripts/agent-evals/coding-side.test.ts @@ -220,7 +220,7 @@ describe("coding side isolation gate", () => { // The install behavior of the shimmed toolchain: "succeed" runs the benign // build, "fail" exits nonzero after the install starts, and "hang" blocks so // the test can interrupt the side mid-preparation. -type InstallBehavior = "succeed" | "fail" | "hang" +type InstallBehavior = "succeed" | "fail" | "hang" | "ignore-term" // A pnpm shim that records each invocation, its working directory, and its // environment, so a test can prove the side prepared and built a fresh tree @@ -238,6 +238,19 @@ const writePnpmShim = async (dir: string, install: InstallBehavior): Promise ${log("pnpm-install-cwd.log")}`, ` env > ${log("pnpm-env.log")}`, ] as string[] + if (install === "ignore-term") { + const script = path.join(dir, "stubborn-install.cjs") + await fs.writeFile( + script, + [ + 'const fs = require("node:fs")', + `process.on("SIGTERM", () => fs.writeFileSync(${log("install-term")}, "received"))`, + `fs.writeFileSync(${log("install-pid")}, String(process.pid))`, + "setInterval(() => {}, 1000)", + ].join("\n"), + ) + lines.push(` exec ${JSON.stringify(process.execPath)} ${JSON.stringify(script)}`) + } if (install === "hang") lines.push(" sleep 30") if (install === "fail") lines.push(" exit 1") lines.push( @@ -434,53 +447,79 @@ describe("coding side evaluated tree preparation", () => { } }) - it( - "removes its worktree when interrupted during preparation and exits with the signal code", - { timeout: 240_000 }, - async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) - try { - const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n" }) - const trustFile = await writePolicy(dir, [commit]) - const shimDir = await writePnpmShim(dir, "hang") - let stderr = "" - const child = spawn( - process.execPath, - [ - "--import", - "tsx", - path.join(repoRoot, "scripts/agent-evals/coding-side.mts"), - ...preparedArgs(dir, repo, trustFile), - ], - { - cwd: repoRoot, - env: preparedEnv(dir, shimDir), - stdio: ["ignore", "ignore", "pipe"], - }, - ) - child.stderr?.on("data", (chunk: Buffer) => { - stderr += chunk.toString() - }) - const exitCode = new Promise((resolve, reject) => { - child.once("error", reject) - child.once("exit", (code) => resolve(code ?? 1)) - }) - // The install is in flight when the signal arrives. A SIGINT that - // reaches only the side must stop the preparation group, remove - // the worktree, and exit with the conventional code. - await waitForFile(path.join(dir, "pnpm-install-cwd.log"), 60_000) - child.kill("SIGINT") - assert.equal(await exitCode, 130) - assert.match(stderr, /interrupted during preparation/) - const worktree = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() - await assert.rejects(fs.access(worktree)) - await assert.rejects(fs.access(path.dirname(worktree))) - await assertSingleWorktree(repo) - // An interrupted side never publishes the completion marker. - await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) - } finally { - await fs.rm(dir, { recursive: true, force: true }) - } - }, - ) + for (const repeated of [false, true]) + it( + `removes its worktree after ${repeated ? "repeated interrupts" : "an interrupt"} during preparation`, + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + let stopChild: (() => Promise) | undefined + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n" }) + const trustFile = await writePolicy(dir, [commit]) + const shimDir = await writePnpmShim(dir, repeated ? "ignore-term" : "hang") + let stderr = "" + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-side.mts"), + ...preparedArgs(dir, repo, trustFile), + ], + { + cwd: repoRoot, + env: preparedEnv(dir, shimDir), + stdio: ["ignore", "ignore", "pipe"], + }, + ) + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + const exitCode = new Promise((resolve, reject) => { + child.once("error", reject) + child.once("exit", (code) => resolve(code ?? 1)) + }) + stopChild = async () => { + const pid = Number(await fs.readFile(path.join(dir, "install-pid"), "utf8").catch(() => "")) + if (Number.isInteger(pid) && pid > 0) { + try { + process.kill(-pid, "SIGKILL") + } catch { + // The preparation group already exited. + } + } + child.kill("SIGKILL") + await exitCode.catch(() => {}) + } + // The install is in flight when the signal arrives. A SIGINT that + // reaches only the side must stop the preparation group, remove + // the worktree, and exit with the conventional code. + await waitForFile(path.join(dir, "pnpm-install-cwd.log"), 60_000) + if (repeated) await waitForFile(path.join(dir, "install-pid"), 60_000) + child.kill("SIGINT") + if (repeated) { + // Wait for the first escalation to reach the child before + // delivering another signal during its grace period. + await waitForFile(path.join(dir, "install-term"), 60_000) + child.kill("SIGINT") + } + assert.equal(await exitCode, 130) + if (repeated) { + const pid = Number(await fs.readFile(path.join(dir, "install-pid"), "utf8")) + assert.throws(() => process.kill(-pid, 0), { code: "ESRCH" }) + } + assert.match(stderr, /interrupted during preparation/) + const worktree = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // An interrupted side never publishes the completion marker. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await stopChild?.() + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) }) From 9028fde750a6eb850d68492a27499f71db6bc480 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 2 Nov 2026 01:44:44 +0000 Subject: [PATCH 11/22] fix(evals): address review feedback on isolation, cleanup, and reports --- scripts/agent-evals/README.md | 4 +- scripts/agent-evals/coding-compare.ts | 81 ++++++++++--------- scripts/agent-evals/coding-fixtures.test.ts | 2 +- scripts/agent-evals/coding-grader.test.ts | 43 ++++++++++ scripts/agent-evals/coding-grader.ts | 28 +++++-- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/coding-report.test.ts | 10 +++ scripts/agent-evals/coding-report.ts | 16 +++- scripts/agent-evals/coding-side-flow.test.ts | 34 ++++++++ scripts/agent-evals/coding-side-flow.ts | 12 ++- scripts/agent-evals/coding-side.mts | 8 +- scripts/agent-evals/coding-side.test.ts | 60 +++++++++++--- scripts/agent-evals/coding-stats.test.ts | 14 ++++ scripts/agent-evals/coding-stats.ts | 21 ++++- scripts/agent-evals/coding-trust.test.ts | 42 ++++++++-- scripts/agent-evals/coding-trust.ts | 68 ++++++++++++---- scripts/agent-evals/launcher.test.ts | 34 +++++++- scripts/agent-evals/launcher.ts | 2 +- .../preparation-environment.test.ts | 63 +++++++++++++++ 19 files changed, 456 insertions(+), 88 deletions(-) create mode 100644 scripts/agent-evals/preparation-environment.test.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 80064cacd2..50edd5ccb6 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -144,6 +144,8 @@ A paired comparison pins both commit SHAs, the suite digest, the settings digest The default is three repetitions per fixture per commit. Reports show success rates with Wilson 95% intervals and paired differences. They make no significance claim: three repetitions can show large regressions, not small improvements. +Paired outcome, duration, and cost deltas report the sample standard error of the mean candidate-minus-baseline difference. This estimate is unavailable with fewer than two measured pairs. With small samples it can be unstable, including zero for identical deltas. It is not a significance test. + Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct, when a correct run lacks complete cost data, or when any attempt, including a failed one, lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. ### Trust policy for comparisons @@ -156,7 +158,7 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. The driver creates each side's fresh detached worktree of the approved commit. Each side then constructs its own fresh detached worktree of that same commit, installs the frozen lockfile dependencies and builds inside it, and launches every host against it, so each side runs only its own compiled extension of the approved commit. The install and the build consume the side's own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children receive an allowlisted environment and a temporary HOME with isolated XDG and npm configuration. Parent package-manager configuration and registry URLs are not forwarded. The prepared tree is checked against the raw bytes in the approved commit, and tracked symlinks are rejected. Every driver child, the side included, runs detached in its own process group and is tracked by one guard. The driver's guard tracks the side; the side's guard tracks its own worktree, install, and build. A SIGINT or SIGTERM that reaches only the driver terminates the side group through a bounded SIGTERM-then-SIGKILL escalation. The side starts the same escalation for its preparation children and waits for it before it removes its worktree or exits. The driver's worktree cleanup also waits for the driver's escalation before it removes any tree. diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 3bb750869e..abe654794f 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -1,6 +1,7 @@ import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import path from "node:path" +import os from "node:os" import { promisify } from "node:util" import type { SidePins } from "./coding-pins.ts" import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" @@ -69,44 +70,50 @@ export const runBoundedPreparation = async ( timeoutMs: number, ): Promise => { if (isInterrupted()) throw new Error(`interrupted before ${command} started`) - await new Promise((resolve, reject) => { - const child = spawn(command, [...args], { - cwd, - stdio: ["ignore", "ignore", "pipe"], - detached: true, - env: buildWorktreeEnvironment(process.env), - }) - guard.track(child.pid) - let stderr = "" - child.stderr?.on("data", (chunk: Buffer) => { - if (stderr.length >= STDERR_LIMIT_BYTES) return - stderr += chunk.toString() - }) - let timedOut = false - const timer = setTimeout(() => { - timedOut = true - try { - if (child.pid) process.kill(-child.pid, "SIGKILL") - } catch { - child.kill("SIGKILL") - } - }, timeoutMs) - child.once("error", (error) => { - clearTimeout(timer) - guard.track(undefined) - reject(error) - }) - child.once("exit", (code, signal) => { - clearTimeout(timer) - guard.track(undefined) - if (code === 0) { - resolve() - return - } - const reason = timedOut ? "its time bound expired" : code === null ? `signal ${signal}` : `code ${code}` - reject(new Error([`${command} failed: ${reason}`, stderr].filter(Boolean).join("\n"))) + const preparationHome = await fs.mkdtemp(path.join(os.tmpdir(), "coding-preparation-home-")) + try { + if (isInterrupted()) throw new Error(`interrupted before ${command} started`) + await new Promise((resolve, reject) => { + const child = spawn(command, [...args], { + cwd, + stdio: ["ignore", "ignore", "pipe"], + detached: true, + env: buildWorktreeEnvironment(process.env, preparationHome), + }) + guard.track(child.pid) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + if (stderr.length >= STDERR_LIMIT_BYTES) return + stderr += chunk.toString() + }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, timeoutMs) + child.once("error", (error) => { + clearTimeout(timer) + guard.track(undefined) + reject(error) + }) + child.once("exit", (code, signal) => { + clearTimeout(timer) + guard.track(undefined) + if (code === 0) { + resolve() + return + } + const reason = timedOut ? "its time bound expired" : code === null ? `signal ${signal}` : `code ${code}` + reject(new Error([`${command} failed: ${reason}`, stderr].filter(Boolean).join("\n"))) + }) }) - }) + } finally { + await fs.rm(preparationHome, { recursive: true, force: true }) + } } // The worktree removals in the flow cleanup are best effort and bounded. diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index 66f2daafa3..df1d601bf5 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -91,6 +91,6 @@ describe("coding fixture suite", () => { assert.throws(() => codingBundle("unknown-fixture")) assert.ok(codingSuiteDigest().length === 64) assert.equal(CODING_SUITE_ID, "coding-basic-v1") - assert.equal(GRADER_VERSION, "coding-grader-v3") + assert.equal(GRADER_VERSION, "coding-grader-v4") }) }) diff --git a/scripts/agent-evals/coding-grader.test.ts b/scripts/agent-evals/coding-grader.test.ts index ec2e946c13..9773409e6e 100644 --- a/scripts/agent-evals/coding-grader.test.ts +++ b/scripts/agent-evals/coding-grader.test.ts @@ -214,3 +214,46 @@ describe("local subprocess runtime failure paths", () => { } }) }) + +it("kills a stubborn descendant before returning a local grader timeout", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + // Both processes share the driver's group. The test child ignores + // SIGTERM, so only group escalation can stop its heartbeat. + const descendant = path.join(scratch, "descendant.cjs") + await fs.writeFile( + descendant, + ` + const fs = require("node:fs") + process.on("SIGTERM", () => {}) + fs.writeFileSync(process.argv[2], "started") + setInterval(() => fs.appendFileSync(process.argv[2], "."), 10) + `, + ) + const heartbeat = path.join(scratch, "heartbeat") + const driverPath = await writeDriver( + scratch, + ` + import { spawn } from "node:child_process" + import path from "node:path" + const scratch = path.dirname(process.argv[2]) + "/.." + spawn(process.execPath, [path.join(scratch, "descendant.cjs"), path.join(scratch, "heartbeat")], { stdio: "ignore" }) + setInterval(() => {}, 1000) + `, + ) + const verdict = await createLocalSubprocessGradingRuntime().run({ + ...base, + driverPath, + resources: { ...base.resources, timeoutMs: 1000 }, + }) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "local-subprocess", reason: "graderTimeout" }) + const before = await fs.readFile(heartbeat, "utf8") + assert.ok(before.startsWith("started")) + await new Promise((resolve) => setTimeout(resolve, 100)) + assert.equal(await fs.readFile(heartbeat, "utf8"), before) + await assert.rejects(fs.access(path.join(scratch, "grading-out"))) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts index 33039f4426..cee3b5adc4 100644 --- a/scripts/agent-evals/coding-grader.ts +++ b/scripts/agent-evals/coding-grader.ts @@ -1,3 +1,4 @@ +import { createSideProcessGuard } from "./side-process-guard.ts" import { spawn } from "node:child_process" import { fileURLToPath } from "node:url" import fs from "node:fs/promises" @@ -9,7 +10,7 @@ import type { Measured } from "./trace.ts" // The grader version is a pinned comparison input. Any change to the driver, // the image, or the verdict rules must bump it. -export const GRADER_VERSION = "coding-grader-v3" +export const GRADER_VERSION = "coding-grader-v4" export interface GraderResources { timeoutMs: number @@ -121,15 +122,32 @@ export function createLocalSubprocessGradingRuntime(nodeExecutable = process.exe const value = process.env[name] if (value !== undefined) env[name] = value } + const guard = createSideProcessGuard(100) + let timedOut = false + let cleanup: Promise | undefined const child = spawn(nodeExecutable, [job.driverPath, resultPath, "test"], { cwd: job.gradingDir, env, stdio: "ignore", - timeout: job.resources.timeoutMs, - killSignal: "SIGKILL", + detached: true, + }) + guard.track(child.pid) + const timer = setTimeout(() => { + timedOut = true + cleanup = guard.interrupt() + }, job.resources.timeoutMs) + child.once("error", () => { + clearTimeout(timer) + resolve("error") + }) + child.once("close", (code) => { + clearTimeout(timer) + // Reap leftover test descendants even when the driver exits first. + void (async () => { + await (cleanup ?? guard.interrupt()) + resolve(timedOut ? "timeout" : (code ?? "error")) + })() }) - child.once("error", () => resolve("error")) - child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) }) if (exit === "timeout") return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderTimeout" } diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index 1593b4c245..b893830f39 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -66,7 +66,7 @@ describe("coding graders against reference solutions", () => { } it("pins one grader version across the suite", () => { - assert.equal(GRADER_VERSION, "coding-grader-v3") + assert.equal(GRADER_VERSION, "coding-grader-v4") assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) }) }) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 13f1e3c96a..a81eefe689 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -100,6 +100,16 @@ describe("comparison report", () => { // The report serializes cleanly: a self-contained result. const parsed = JSON.parse(JSON.stringify(report)) as typeof report assert.equal(parsed.fixtures.length, report.fixtures.length) + assert.equal(parsed.fixtures[0]?.pairedDurations.standardError, 0) + assert.equal(parsed.fixtures[0]?.pairedCosts.standardError, 0) + assert.ok((parsed.fixtures[0]?.pairedOutcomes.standardError ?? 0) > 0) + const markdown = renderMarkdownReport(report) + assert.match(markdown, /Outcome delta \(SE\)/) + assert.match(markdown, /Duration delta ms \(SE\)/) + assert.match(markdown, /Cost delta USD \(SE\)/) + assert.ok(markdown.includes("-0.05000 (SE 0.000)")) + assert.match(markdown, /unavailable/) + assert.match(markdown, /sample standard error/) }) it("flags incompatible pairs when pinned inputs differ", () => { diff --git a/scripts/agent-evals/coding-report.ts b/scripts/agent-evals/coding-report.ts index d4a8f80948..366cda4d98 100644 --- a/scripts/agent-evals/coding-report.ts +++ b/scripts/agent-evals/coding-report.ts @@ -138,6 +138,11 @@ const costCell = (cost: CostPerSuccessfulTask): string => ? `$${cost.usd?.toFixed(4)} over ${cost.correctRuns} correct run(s), ${cost.excludedRuns} excluded` : `unavailable (${cost.reason ?? "unavailable"}; ${cost.excludedRuns} excluded)` +const deltaCell = (diff: { meanDelta?: number; standardError?: number }): string => + diff.meanDelta === undefined + ? "unavailable" + : `${diff.meanDelta.toPrecision(4)} (SE ${diff.standardError === undefined ? "unavailable: fewer than two pairs" : diff.standardError.toPrecision(4)})` + // Self-contained Markdown rendering of the JSON report. export function renderMarkdownReport(report: ComparisonReport): string { const lines: string[] = [] @@ -172,6 +177,9 @@ export function renderMarkdownReport(report: ComparisonReport): string { `- ${report.repetitions} repetitions per fixture per commit (default ${STATISTICAL_POLICY.repetitionsDefault}).`, ) lines.push(`- ${report.statisticalPolicy.method}.`) + lines.push( + "- SE is the sample standard error of the mean candidate-minus-baseline paired delta; unavailable with fewer than two measured pairs. Small samples can give unstable estimates, including zero SE.", + ) lines.push(`- ${report.statisticalPolicy.significanceClaim}`) lines.push( `- Partial-data policy: ${report.partialDataPolicy}. Runs with incomplete cost accounting are flagged and excluded from cost aggregates.`, @@ -179,11 +187,13 @@ export function renderMarkdownReport(report: ComparisonReport): string { lines.push("") lines.push("## Results per fixture") lines.push("") - lines.push("| Fixture | Baseline | Candidate | Paired (improved/regressed/unchanged) |") - lines.push("| --- | --- | --- | --- |") + lines.push( + "| Fixture | Baseline | Candidate | Paired (improved/regressed/unchanged) | Outcome delta (SE) | Duration delta ms (SE) | Cost delta USD (SE) |", + ) + lines.push("| --- | --- | --- | --- | --- | --- | --- |") for (const row of report.fixtures) lines.push( - `| ${row.fixtureName} | ${rateCell(row.baseline)} | ${rateCell(row.candidate)} | ${row.pairedOutcomes.improved}/${row.pairedOutcomes.regressed}/${row.pairedOutcomes.unchanged} |`, + `| ${row.fixtureName} | ${rateCell(row.baseline)} | ${rateCell(row.candidate)} | ${row.pairedOutcomes.improved}/${row.pairedOutcomes.regressed}/${row.pairedOutcomes.unchanged} | ${deltaCell(row.pairedOutcomes)} | ${deltaCell(row.pairedDurations)} | ${deltaCell(row.pairedCosts)} |`, ) lines.push("") lines.push("## Cost per successful task") diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts index ddb44c5082..82e3c77557 100644 --- a/scripts/agent-evals/coding-side-flow.test.ts +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict" import fs from "node:fs/promises" +import { existsSync } from "node:fs" import os from "node:os" import path from "node:path" import { describe, it } from "node:test" @@ -232,3 +233,36 @@ describe("side execution flow", () => { } }) }) + +// Inject the interrupt at each publication boundary, including after the +// marker write. The observed filesystem state proves each await has finished. +for (const boundary of ["directory", "record", "pins", "marker"] as const) { + it(`removes partial publication when interrupted after ${boundary}`, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-publication-")) + let interrupted = false + try { + const exists = (relative: string): boolean => existsSync(path.join(dir, relative)) + const outcome = await executeSideFlow({ + repetitions: plan(1), + out: dir, + isInterrupted: () => { + if (boundary === "directory" && exists("artifacts/clamp-range-rep0")) interrupted = true + if (boundary === "record" && exists("artifacts/clamp-range-rep0/run-record.json")) + interrupted = true + if (boundary === "marker" && exists("side-results.json")) interrupted = true + return interrupted + }, + runRepetition: async (repetition) => record(repetition, true), + buildResult: async (runs) => { + if (boundary === "pins") interrupted = true + return { pins: pins(), runs } + }, + }) + assert.equal(interrupted, true) + assert.equal(outcome.completed, false) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts index 8c0057b3f7..1de68ba6fd 100644 --- a/scripts/agent-evals/coding-side-flow.ts +++ b/scripts/agent-evals/coding-side-flow.ts @@ -80,13 +80,23 @@ export async function executeSideFlow(inputs: SideFlowInputs): Promise => { + await fs.rm(path.join(inputs.out, "side-results.json"), { force: true }) + await fs.rm(path.join(inputs.out, "artifacts"), { recursive: true, force: true }) + return { runs, completed: false } + } // Per-run artifacts are published atomically as each record exists. for (const record of runs) { if (!record.trace) continue const artifactDir = path.join(inputs.out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) await fs.mkdir(artifactDir, { recursive: true }) + if (inputs.isInterrupted()) return abortPublication() await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) + if (inputs.isInterrupted()) return abortPublication() } - await writeJsonAtomic(path.join(inputs.out, "side-results.json"), await inputs.buildResult(runs)) + const result = await inputs.buildResult(runs) + if (inputs.isInterrupted()) return abortPublication() + await writeJsonAtomic(path.join(inputs.out, "side-results.json"), result) + if (inputs.isInterrupted()) return abortPublication() return { runs, completed: true } } diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 90e73cbc51..d6f647da72 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -174,8 +174,12 @@ const removePreparedTree = async (): Promise => { await execFileAsync("git", ["worktree", "remove", "--force", evaluatedWorktree], { cwd: evaluatedRoot, timeout: WORKTREE_REMOVE_TIMEOUT_MS, - }).catch(() => {}) - await fs.rm(sideScratch, { recursive: true, force: true }).catch(() => {}) + }).catch((error: unknown) => { + console.error(`coding-evals side: failed to remove worktree ${evaluatedWorktree}: ${String(error)}`) + }) + await fs.rm(sideScratch, { recursive: true, force: true }).catch((error: unknown) => { + console.error(`coding-evals side: failed to remove scratch directory ${sideScratch}: ${String(error)}`) + }) } const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts index 239a3b2a5b..9b3a7cd38d 100644 --- a/scripts/agent-evals/coding-side.test.ts +++ b/scripts/agent-evals/coding-side.test.ts @@ -7,6 +7,10 @@ import { fileURLToPath } from "node:url" import { promisify } from "node:util" import { describe, it } from "node:test" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" + +const shellQuote = (value: string): string => "'" + value.replace(/'/g, "'\\''") + "'" + const execFileAsync = promisify(execFile) const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -229,7 +233,7 @@ type InstallBehavior = "succeed" | "fail" | "hang" | "ignore-term" const writePnpmShim = async (dir: string, install: InstallBehavior): Promise => { const shimDir = path.join(dir, "bin") await fs.mkdir(shimDir) - const log = (name: string): string => JSON.stringify(path.join(dir, name)) + const log = (name: string): string => shellQuote(path.join(dir, name)) const lines = [ "#!/bin/sh", `printf '%s\\n' "$*" >> ${log("pnpm-args.log")}`, @@ -244,12 +248,12 @@ const writePnpmShim = async (dir: string, install: InstallBehavior): Promise fs.writeFileSync(${log("install-term")}, "received"))`, - `fs.writeFileSync(${log("install-pid")}, String(process.pid))`, + 'process.on("SIGTERM", () => fs.writeFileSync(require("node:path").join(process.argv[2], "install-term"), "received"))', + 'fs.writeFileSync(require("node:path").join(process.argv[2], "install-pid"), String(process.pid))', "setInterval(() => {}, 1000)", ].join("\n"), ) - lines.push(` exec ${JSON.stringify(process.execPath)} ${JSON.stringify(script)}`) + lines.push(` exec ${shellQuote(process.execPath)} ${shellQuote(script)} ${shellQuote(dir)}`) } if (install === "hang") lines.push(" sleep 30") if (install === "fail") lines.push(" exit 1") @@ -273,9 +277,9 @@ const writePnpmShim = async (dir: string, install: InstallBehavior): Promise minimalEnv({ SOLHEIM_API_KEY: "sk-test", PATH: `${shimDir}:${process.env["PATH"] ?? ""}` }) -const preparedArgs = (dir: string, repo: string, trustFile: string): string[] => [ +const preparedArgs = (dir: string, repo: string, trustFile: string, reps = 1): string[] => [ "--reps", - "1", + String(reps), "--out", path.join(dir, "out"), "--trust-file", @@ -284,9 +288,9 @@ const preparedArgs = (dir: string, repo: string, trustFile: string): string[] => repo, ] -const runPreparedSide = async (dir: string, repo: string, trustFile: string): Promise => { +const runPreparedSide = async (dir: string, repo: string, trustFile: string, reps = 1): Promise => { const shimDir = await writePnpmShim(dir, "succeed") - return runSideScript(preparedArgs(dir, repo, trustFile), preparedEnv(dir, shimDir)) + return runSideScript(preparedArgs(dir, repo, trustFile, reps), preparedEnv(dir, shimDir)) } const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) @@ -332,15 +336,20 @@ const assertPreparationRanInFreshTree = async (dir: string, repo: string, commit interface SideResultsFile { pins: { commit: string } - runs: { outcome: { outcome: string } }[] + runs: { fixtureName: string; repIndex: number; outcome: { outcome: string } }[] } // The launch fails deterministically because the fixture has no host entry, // so every repetition is a non-correct record and the side still publishes. -const assertPublishedFailedRuns = async (dir: string, commit: string): Promise => { +const assertPublishedFailedRuns = async (dir: string, commit: string, reps = 1): Promise => { const results = JSON.parse(await fs.readFile(path.join(dir, "out", "side-results.json"), "utf8")) as SideResultsFile assert.equal(results.pins.commit, commit) - assert.ok(results.runs.length >= 1) + assert.deepEqual( + results.runs.map((run) => `${run.fixtureName}#${run.repIndex}`).sort(), + CODING_FIXTURE_BUNDLES.flatMap((bundle) => + Array.from({ length: reps }, (_, repIndex) => `${bundle.fixture.identity.name}#${repIndex}`), + ).sort(), + ) for (const run of results.runs) assert.notEqual(run.outcome.outcome, "correct") } @@ -354,6 +363,7 @@ describe("coding side evaluated tree preparation", () => { const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n", "tracked.txt": "committed", + "src/package.json": JSON.stringify({ version: "1.2.3" }), }) const trustFile = await writePolicy(dir, [commit]) // Everything planted below stays outside the approval. The @@ -362,6 +372,7 @@ describe("coding side evaluated tree preparation", () => { // executable artifacts a real build may never overwrite. If // any of them ran, its marker file would exist. await fs.writeFile(path.join(repo, "tracked.txt"), "tampered") + await fs.writeFile(path.join(repo, "src/package.json"), JSON.stringify({ version: "9.9.9" })) await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") const plantedEntry = `require("node:fs").writeFileSync(${JSON.stringify(path.join(repo, "planted-entry-marker"))}, "executed")\n` await fs.mkdir(path.join(repo, "src/dist"), { recursive: true }) @@ -369,7 +380,7 @@ describe("coding side evaluated tree preparation", () => { await fs.mkdir(path.join(repo, "node_modules/.bin"), { recursive: true }) await fs.writeFile( path.join(repo, "node_modules/.bin/evil"), - `#!/bin/sh\nprintf executed > ${JSON.stringify(path.join(repo, "planted-bin-marker"))}\n`, + `#!/bin/sh\nprintf executed > ${shellQuote(path.join(repo, "planted-bin-marker"))}\n`, { mode: 0o755 }, ) const outcome = await runPreparedSide(dir, repo, trustFile) @@ -392,6 +403,13 @@ describe("coding side evaluated tree preparation", () => { // The pins record the approved commit, and the preparation // children never saw the credential. await assertPublishedFailedRuns(dir, commit) + const published = JSON.parse(await fs.readFile(path.join(dir, "out/side-results.json"), "utf8")) as { + runs: { trace: { run: { identity: { extensionVersion: unknown } } } }[] + } + for (const run of published.runs) { + assert.deepEqual(run.trace.run.identity.extensionVersion, { available: true, value: "1.2.3" }) + } + const envLog = await fs.readFile(path.join(dir, "pnpm-env.log"), "utf8") assert.ok(!envLog.includes("SOLHEIM_API_KEY=")) } finally { @@ -523,3 +541,21 @@ describe("coding side evaluated tree preparation", () => { }, ) }) + +it( + "preserves literal shell metacharacters in shim paths and publishes every repetition", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-$HOME-`pwd`-'-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\n" }) + const trustFile = await writePolicy(dir, [commit]) + const outcome = await runPreparedSide(dir, repo, trustFile, 2) + assert.equal(outcome.code, 0, outcome.stderr) + await assertPreparationRanInFreshTree(dir, repo, commit) + await assertPublishedFailedRuns(dir, commit, 2) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, +) diff --git a/scripts/agent-evals/coding-stats.test.ts b/scripts/agent-evals/coding-stats.test.ts index c7e86dcd47..9d851084d6 100644 --- a/scripts/agent-evals/coding-stats.test.ts +++ b/scripts/agent-evals/coding-stats.test.ts @@ -3,6 +3,7 @@ import { describe, it } from "node:test" import { DEFAULT_REPETITIONS, STATISTICAL_POLICY, + pairedStandardError, pairedNumericDiffs, pairedOutcomeDiffs, successRate, @@ -60,3 +61,16 @@ describe("statistics", () => { assert.equal(empty.meanDelta, undefined) }) }) + +it("reports paired sample standard errors, including insufficient samples", () => { + assert.equal(pairedStandardError([]), undefined) + assert.equal(pairedStandardError([2]), undefined) + assert.equal(pairedStandardError([2, 4]), 1) + assert.equal(pairedNumericDiffs([1, 2], [3, 6]).standardError, 1) + assert.equal(pairedNumericDiffs([1, undefined], [3, 6]).standardError, undefined) + const outcomes = pairedOutcomeDiffs(["correct", "incorrect"], ["incorrect", "correct"]) + assert.equal(outcomes.meanDelta, 0) + assert.equal(outcomes.standardError, 1) + assert.equal(pairedOutcomeDiffs([], []).standardError, undefined) + assert.equal(pairedOutcomeDiffs(["correct"], ["correct"]).standardError, undefined) +}) diff --git a/scripts/agent-evals/coding-stats.ts b/scripts/agent-evals/coding-stats.ts index 5ac1befefd..375957dd80 100644 --- a/scripts/agent-evals/coding-stats.ts +++ b/scripts/agent-evals/coding-stats.ts @@ -7,7 +7,7 @@ export const UNCERTAINTY_Z = 1.96 export const DEFAULT_REPETITIONS = 3 export const STATISTICAL_POLICY = { - method: "success-rate-with-wilson-95-percent-interval-and-paired-deltas", + method: "success-rate-with-wilson-95-percent-interval-and-paired-deltas-with-sample-standard-errors", significanceClaim: "none. Three repetitions can show large regressions. They cannot show small improvements.", repetitionsDefault: DEFAULT_REPETITIONS, } as const @@ -45,7 +45,17 @@ export function successRate(outcomes: readonly CodingRunOutcome[]): SuccessRate } } +// Sample standard error of the mean paired delta. Undefined for fewer than +// two observations; this descriptive estimate makes no significance claim. +export function pairedStandardError(deltas: readonly number[]): number | undefined { + if (deltas.length < 2) return undefined + const mean = deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length + return Math.sqrt(deltas.reduce((sum, delta) => sum + (delta - mean) ** 2, 0) / (deltas.length - 1) / deltas.length) +} + export interface PairedOutcomeDiff { + meanDelta?: number + standardError?: number pairs: number improved: number regressed: number @@ -72,6 +82,13 @@ export function pairedOutcomeDiffs( else regressed++ } return { + meanDelta: pairs ? (improved - regressed) / pairs : undefined, + standardError: pairedStandardError( + Array.from( + { length: pairs }, + (_, index) => Number(candidate[index] === "correct") - Number(baseline[index] === "correct"), + ), + ), pairs, improved, regressed, @@ -81,6 +98,7 @@ export function pairedOutcomeDiffs( } export interface PairedNumericDiff { + standardError?: number pairs: { baseline: number; candidate: number; delta: number }[] missingPairs: number meanDelta?: number @@ -111,6 +129,7 @@ export function pairedNumericDiffs( return { pairs, missingPairs, + standardError: pairedStandardError(deltas), meanDelta: deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length, minDelta: Math.min(...deltas), maxDelta: Math.max(...deltas), diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index 4dae732ec7..bc1e22fc73 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -409,13 +409,7 @@ describe("worktree process environment", () => { OTHER_SECRET: "leak-me-too", NODE_OPTIONS: "--require /tmp/never", }) - assert.deepEqual(child, { - PATH: "/usr/bin", - HOME: "/home/dev", - PNPM_HOME: "/home/dev/.local/share/pnpm", - XDG_DATA_HOME: "/home/dev/.local/share", - npm_config_registry: "https://registry.example", - }) + assert.deepEqual(child, { PATH: "/usr/bin" }) }) it("never carries the provider credential into install or build", () => { @@ -423,3 +417,37 @@ describe("worktree process environment", () => { assert.equal(child["SOLHEIM_API_KEY"], undefined) }) }) + +for (const kind of ["symlink", "filter"] as const) { + it(`rejects unpinned ${kind} content even when git status is clean`, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-bytes-")) + try { + const repo = await makeRepo(dir) + if (kind === "symlink") { + await fs.writeFile(path.join(dir, "outside"), "unapproved") + await fs.symlink(path.join(dir, "outside"), path.join(repo, "entry")) + } else { + await fs.writeFile(path.join(repo, "entry"), "approved\n") + await fs.writeFile(path.join(repo, ".gitattributes"), "entry filter=fixture\n") + } + await commitAll(repo, "entry") + let checked = repo + if (kind === "filter") { + await execFileAsync("git", ["config", "filter.fixture.smudge", "sed s/approved/unpinned/"], { + cwd: repo, + }) + await execFileAsync("git", ["config", "filter.fixture.clean", "sed s/unpinned/approved/"], { + cwd: repo, + }) + checked = path.join(dir, "checked") + await execFileAsync("git", ["worktree", "add", "--detach", checked, "HEAD"], { cwd: repo }) + assert.equal(await fs.readFile(path.join(checked, "entry"), "utf8"), "unpinned\n") + } + const { stdout } = await execFileAsync("git", ["status", "--porcelain"], { cwd: checked }) + assert.equal(stdout, "") + await assert.rejects(assertEvaluatedTreeClean(checked), DirtyEvaluatedTreeError) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index fb55cccfd8..627ecd5729 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -101,6 +101,36 @@ export async function assertEvaluatedTreeClean(root: string): Promise { .map((line) => line.trim()) .filter((line) => line.length > 0) if (entries.length > 0) throw new DirtyEvaluatedTreeError(entries) + // Compare raw bytes to HEAD, bypassing both the index stat cache and + // clean filters. A symlink can reference unapproved bytes outside this tree. + const { stdout: tree } = await promisify(execFile)("git", ["ls-tree", "-rz", "HEAD"], { + cwd: root, + maxBuffer: 16 * 1024 * 1024, + }) + const blobs: { file: string; object: string }[] = [] + for (const entry of tree.split("\0").filter(Boolean)) { + const tab = entry.indexOf("\t") + const [mode, , object] = entry.slice(0, tab).split(" ") + const file = entry.slice(tab + 1) + if (mode === "120000") throw new DirtyEvaluatedTreeError([`symlink: ${file}`]) + if (mode === "160000") continue // Submodule dirtiness is checked above. + if (!object) throw new Error("invalid approved tree entry") + blobs.push({ file, object }) + } + // Bounded batches avoid one Git process per source file and argv limits. + for (let offset = 0; offset < blobs.length; offset += 128) { + const batch = blobs.slice(offset, offset + 128) + const { stdout: hashes } = await promisify(execFile)( + "git", + ["hash-object", "--no-filters", "--", ...batch.map((blob) => blob.file)], + { cwd: root, timeout: 30_000 }, + ) + const actual = hashes.trim().split("\n") + for (const [index, blob] of batch.entries()) { + if (actual[index] !== blob.object) + throw new DirtyEvaluatedTreeError([`content differs from approved tree: ${blob.file}`]) + } + } } // Provider credential variables this framework supports. The side script and @@ -155,24 +185,32 @@ export function buildSideEnvironment( // Environment for the worktree install and build processes. These children // inherit neither the driver's secrets nor the provider credential. They get -// the shared process locations plus the explicitly approved pnpm variables a -// frozen-lockfile install and a build need to locate their store, cache, and -// registry. -const WORKTREE_ENV_ALLOWLIST = [ - ...PROCESS_ENV_ALLOWLIST, - "PNPM_HOME", - "XDG_DATA_HOME", - "XDG_CACHE_HOME", - "XDG_CONFIG_HOME", - "XDG_STATE_HOME", - "npm_config_registry", -] as const - -export function buildWorktreeEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { +// process executable locations and an isolated home. Parent package-manager +// configuration and registry URLs never cross this boundary. +export function buildWorktreeEnvironment(env: NodeJS.ProcessEnv, home?: string): NodeJS.ProcessEnv { const child: NodeJS.ProcessEnv = {} - for (const name of WORKTREE_ENV_ALLOWLIST) { + for (const name of [ + "PATH", + "LANG", + "LC_ALL", + "TMPDIR", + "TMP", + "TEMP", + "SYSTEMROOT", + "WINDIR", + "COMSPEC", + "PATHEXT", + ] as const) { const value = env[name] if (value !== undefined) child[name] = value } + if (home) { + child["HOME"] = home + child["USERPROFILE"] = home + for (const name of ["XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_STATE_HOME"]) + child[name] = `${home}/${name.toLowerCase()}` + child["npm_config_userconfig"] = `${home}/.npmrc` + child["npm_config_globalconfig"] = `${home}/global-npmrc` + } return child } diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts index 905ea6d1de..de1f5e9396 100644 --- a/scripts/agent-evals/launcher.test.ts +++ b/scripts/agent-evals/launcher.test.ts @@ -4,7 +4,7 @@ import os from "node:os" import path from "node:path" import { describe, it } from "node:test" import { fileURLToPath } from "node:url" -import { createHostRuntime } from "./launcher.ts" +import { createHostRuntime, readWorkspaceFiles } from "./launcher.ts" const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -63,3 +63,35 @@ describe("host runtime", () => { } }) }) + +it("rejects final workspace capture before launch", async () => { + const runtime = createHostRuntime({ root: repoRoot, artifactsDir: os.tmpdir() }) + assert.ok(runtime.readFinalWorkspace) + await assert.rejects(runtime.readFinalWorkspace(), /workspace not created/) +}) + +for (const scenario of ["nested", "oversized", "total", "count"] as const) { + it(`captures bounded final workspace files: ${scenario}`, async () => { + const dir = await tempDir("workspace-capture-") + try { + if (scenario === "nested") { + for (const name of ["nested", "node_modules", ".git"]) { + await fs.mkdir(path.join(dir, name)) + await fs.writeFile(path.join(dir, name, "file.js"), name) + } + } else { + const count = scenario === "count" ? 4097 : scenario === "total" ? 9 : 1 + const content = scenario === "count" ? "x" : "x".repeat(scenario === "oversized" ? 1048577 : 1048576) + await Promise.all( + Array.from({ length: count }, (_, index) => fs.writeFile(path.join(dir, `${index}.js`), content)), + ) + } + const result = await readWorkspaceFiles(dir) + assert.equal(result.truncated, scenario !== "nested") + if (scenario === "nested") assert.deepEqual(result.files, [{ path: "nested/file.js", content: "nested" }]) + else assert.equal(result.files.length, scenario === "count" ? 4096 : scenario === "total" ? 8 : 0) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 66ef28b270..64dfc95252 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -67,7 +67,7 @@ const FINAL_WORKSPACE_SKIPPED_DIRS = new Set(["node_modules", ".git"]) // directories, caps file count and size, and reports truncation instead of // reading unbounded agent-written trees. Paths stay relative and forward // slashed, matching the fixture workspace file shape. -async function readWorkspaceFiles(workspaceRoot: string): Promise { +export async function readWorkspaceFiles(workspaceRoot: string): Promise { const files: AgentFixtureWorkspaceFile[] = [] let truncated = false let totalBytes = 0 diff --git a/scripts/agent-evals/preparation-environment.test.ts b/scripts/agent-evals/preparation-environment.test.ts new file mode 100644 index 0000000000..f5a97b904b --- /dev/null +++ b/scripts/agent-evals/preparation-environment.test.ts @@ -0,0 +1,63 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { it } from "node:test" +import { runBoundedPreparation } from "./coding-compare.ts" +import { createSideProcessGuard } from "./side-process-guard.ts" + +it("preparation children use isolated home and npm configuration without registry credentials", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "preparation-env-")) + const names = ["HOME", "XDG_CONFIG_HOME", "npm_config_registry"] as const + const saved = names.map((name) => process.env[name]) + try { + await fs.writeFile(path.join(dir, ".npmrc"), "//registry.example/:_authToken=private-token") + process.env["HOME"] = dir + process.env["XDG_CONFIG_HOME"] = dir + process.env["npm_config_registry"] = "https://user:private-token@registry.example/" + await runBoundedPreparation( + createSideProcessGuard(100), + () => false, + process.execPath, + [ + "-e", + ` + const fs = require("node:fs") + const path = require("node:path") + fs.writeFileSync("observed.json", JSON.stringify({ + home: process.env.HOME, + config: process.env.XDG_CONFIG_HOME, + registry: process.env.npm_config_registry ?? null, + npmrcExists: fs.existsSync(path.join(process.env.HOME, ".npmrc")), + userconfig: process.env.npm_config_userconfig, + globalconfig: process.env.npm_config_globalconfig, + })) + `, + ], + dir, + 5000, + ) + const observed = JSON.parse(await fs.readFile(path.join(dir, "observed.json"), "utf8")) as { + home: string + config: string + registry: unknown + npmrcExists: boolean + userconfig: string + globalconfig: string + } + assert.notEqual(observed.home, dir) + assert.ok(observed.config.startsWith(observed.home + "/")) + assert.equal(observed.registry, null) + assert.equal(observed.npmrcExists, false) + assert.equal(observed.userconfig, path.join(observed.home, ".npmrc")) + assert.equal(observed.globalconfig, path.join(observed.home, "global-npmrc")) + await assert.rejects(fs.access(observed.home)) + } finally { + for (const [index, name] of names.entries()) { + const value = saved[index] + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + await fs.rm(dir, { recursive: true, force: true }) + } +}) From 9fd42e9746d055e46d923fa5368d069d9272d197 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 2 Nov 2026 01:44:44 +0000 Subject: [PATCH 12/22] fix(evals): address review feedback on isolation, cleanup, and reports --- scripts/agent-evals/runner.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index 5d780473c5..d40db265df 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" +import { providerIdentifiers } from "@roo-code/types/provider-identifiers" import { registerFixture } from "./registry.ts" -import { runAgentFixture, type AgentRuntime, type AgentRunResult } from "./runner.ts" +import { runAgentFixture, type AgentRuntime } from "./runner.ts" import { digestProjection, type AgentTrace } from "./trace.ts" import type { AgentFixture } from "./contract.ts" @@ -42,7 +43,7 @@ function registerTestFixture(name: string, overrides: Partial = {} }, settingsProjection: { provider: "test", model: `model-${name}` }, buildConfiguration: () => ({ - apiProvider: "openai", + apiProvider: providerIdentifiers.openai, openAiApiKey: CREDENTIAL, mode: "test", customModes: [{ slug: "test", name: "Test", roleDefinition: "Run one synthetic task.", groups: [] }], From fceb8a7c78f629f4f77417618d814ed5480cfee1 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 2 Nov 2026 12:59:58 +0000 Subject: [PATCH 13/22] fix(evals): close review gaps in cleanup and run accounting --- scripts/agent-evals/README.md | 2 +- .../agent-evals/coding-compare-driver.test.ts | 49 +++++++++++++++++ scripts/agent-evals/coding-compare.ts | 35 +++++++++++-- scripts/agent-evals/coding-pins.test.ts | 18 ++++++- scripts/agent-evals/coding-pins.ts | 7 +++ scripts/agent-evals/coding-report.test.ts | 52 +++++++++++++++++++ scripts/agent-evals/coding-report.ts | 15 ++++-- scripts/agent-evals/coding-run.test.ts | 6 ++- scripts/agent-evals/coding-run.ts | 15 +++++- scripts/agent-evals/coding-side-flow.test.ts | 4 ++ scripts/agent-evals/coding-side-flow.ts | 15 +++--- scripts/agent-evals/coding-side.mts | 11 ++-- scripts/agent-evals/launcher.test.ts | 40 ++++++++++++++ scripts/agent-evals/launcher.ts | 15 ++++-- .../preparation-environment.test.ts | 42 +++++++++++++++ 15 files changed, 298 insertions(+), 28 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 50edd5ccb6..fecdc42f27 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -131,7 +131,7 @@ release process: a maintainer runs it locally or on demand. - The grader is model-free and credential-free. It judges the final workspace state only. - The final workspace is captured as bounded file data and rebuilt as a fresh copy. Hidden acceptance tests enter only that copy, so the agent can never read or change them, and one run can never inherit another run's files. - The default runtime executes the driver in a uniquely named disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. -- The grader has its own wall-clock limit. On expiry the runtime removes the container by name and verifies the removal before it deletes any output. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. The image pull happens before the timed run and never consumes grading time. The runtime resolves the immutable grader image digest from the local engine and the side records it in the comparison pins; it never fabricates one. +- The grader has its own wall-clock limit. On expiry the runtime removes the container by name and verifies the removal before it deletes any output. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. The image pull happens before the timed run and never consumes grading time. The runtime resolves the immutable grader image digest from the local engine, and each graded run records its own identity. The side records that identity in the comparison pins and refuses to publish if run identities differ; it never fabricates a digest. - The driver's result file is untrusted data: it is written inside the container where agent-written test code executes. The host reads it with a size bound and validates shape, bounds, and internal consistency. An invalid or inconsistent result is a `grading failure`, never a verdict. - Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. - `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts index 5f05e4ba39..cc9700a384 100644 --- a/scripts/agent-evals/coding-compare-driver.test.ts +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -319,3 +319,52 @@ describe("comparison flow outcome and cleanup", () => { } }) }) + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + it(`does not publish a report when ${signal} arrives as the candidate returns`, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-final-interrupt-")) + try { + const flag = createInterruptFlag() + const out = path.join(dir, "out") + await fs.mkdir(out) + const outcome = await executeComparisonFlow( + flowWiring({ + flag, + out, + scratchDir: path.join(dir, "scratch"), + runSide: async (side) => { + if (side === "candidate") flag.signal(signal) + return record(side, COMMIT) + }, + }), + ) + assert.deepEqual(outcome, { status: "interrupted", signal }) + assert.deepEqual(await fs.readdir(out), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} + +it("removes a published generation when interrupted during final cleanup", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-cleanup-interrupt-")) + try { + const flag = createInterruptFlag() + const out = path.join(dir, "out") + await fs.mkdir(out) + const wiring = flowWiring({ + flag, + out, + scratchDir: path.join(dir, "scratch"), + runSide: async (side) => record(side, COMMIT), + }) + wiring.interruptKill = async () => { + assert.ok(await publishedComparisonGeneration(out)) + flag.signal("SIGINT") + } + assert.deepEqual(await executeComparisonFlow(wiring), { status: "interrupted", signal: "SIGINT" }) + assert.deepEqual(await fs.readdir(out), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index abe654794f..2ad6806dce 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -102,6 +102,14 @@ export const runBoundedPreparation = async ( }) child.once("exit", (code, signal) => { clearTimeout(timer) + // The group may still contain workers after its leader exits. + if (child.pid) { + try { + process.kill(-child.pid, "SIGKILL") + } catch { + // The group has already exited. + } + } guard.track(undefined) if (code === 0) { resolve() @@ -266,7 +274,9 @@ export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promi // published as one generation with a completion marker; any failure // above or during publication leaves no complete report generation // behind. - await publishComparisonReport(wiring.out, report) + if (!wiring.interruptedSignal()) { + await publishComparisonReport(wiring.out, report, () => wiring.interruptedSignal() !== undefined) + } completed = report } catch (error) { // Deferred, not swallowed: the cleanup below runs first, then the @@ -287,7 +297,10 @@ export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promi await fs.rm(wiring.scratchDir, { recursive: true, force: true }).catch(() => {}) } const signal = wiring.interruptedSignal() - if (signal === "SIGINT" || signal === "SIGTERM") return { status: "interrupted", signal } + if (signal === "SIGINT" || signal === "SIGTERM") { + await removeComparisonReport(wiring.out) + return { status: "interrupted", signal } + } if (failure !== undefined) throw failure if (completed === undefined) throw new Error("comparison ended without a report or a failure") return { status: "completed", report: completed } @@ -380,16 +393,32 @@ const COMPARISON_REPORT_COMPLETE = "comparison-report.complete" // instead of a mixed one. A thrown failure rolls both files back, so a failed // comparison leaves no report files behind. Consumers must require the // completion marker and match its generation against the report files. -export async function publishComparisonReport(outDir: string, report: ComparisonReport): Promise { +export async function removeComparisonReport(outDir: string): Promise { + for (const name of [COMPARISON_REPORT_COMPLETE, COMPARISON_REPORT_JSON, COMPARISON_REPORT_MD]) + await fs.rm(path.join(outDir, name), { force: true }) +} + +export async function publishComparisonReport( + outDir: string, + report: ComparisonReport, + isInterrupted: () => boolean = () => false, +): Promise { const generation = report.generatedAt const jsonPath = path.join(outDir, COMPARISON_REPORT_JSON) const markdownPath = path.join(outDir, COMPARISON_REPORT_MD) const markerPath = path.join(outDir, COMPARISON_REPORT_COMPLETE) await fs.rm(markerPath, { force: true }) + const checkInterrupted = (): void => { + if (isInterrupted()) throw new Error("comparison report publication interrupted") + } try { + checkInterrupted() await writeJsonAtomic(jsonPath, { ...report, generation }) + checkInterrupted() await writeFileAtomic(markdownPath, renderMarkdownReport(report)) + checkInterrupted() await writeFileAtomic(markerPath, `${generation}\n`) + checkInterrupted() } catch (error) { await fs.rm(jsonPath, { force: true }).catch(() => {}) await fs.rm(markdownPath, { force: true }).catch(() => {}) diff --git a/scripts/agent-evals/coding-pins.test.ts b/scripts/agent-evals/coding-pins.test.ts index 88e9ed6ddf..7957a0650b 100644 --- a/scripts/agent-evals/coding-pins.test.ts +++ b/scripts/agent-evals/coding-pins.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" -import { comparePins, type SidePins } from "./coding-pins.ts" +import { sideGraderImageDigest, comparePins, type SidePins } from "./coding-pins.ts" const pins = (side: "baseline" | "candidate", overrides: Partial = {}): SidePins => ({ side, @@ -46,3 +46,19 @@ describe("pinned comparison inputs", () => { assert.equal(result.compatible, true) }) }) + +it("pins one recorded image and rejects image changes within a side", () => { + assert.equal(sideGraderImageDigest([]), "unavailable") + assert.equal( + sideGraderImageDigest([{ graderImageDigest: "sha256:a" }, {}, { graderImageDigest: "sha256:a" }]), + "sha256:a", + ) + assert.throws( + () => sideGraderImageDigest([{ graderImageDigest: "sha256:a" }, { graderImageDigest: "sha256:b" }]), + /changed within the side/, + ) + assert.throws( + () => sideGraderImageDigest([{ graderImageDigest: "sha256:a" }, { graderImageDigest: "unavailable" }]), + /changed within the side/, + ) +}) diff --git a/scripts/agent-evals/coding-pins.ts b/scripts/agent-evals/coding-pins.ts index 7449e0dff7..cbd0ea0cd0 100644 --- a/scripts/agent-evals/coding-pins.ts +++ b/scripts/agent-evals/coding-pins.ts @@ -43,3 +43,10 @@ export function comparePins(baseline: SidePins, candidate: SidePins): PinCompari differences.push(`runnerEnvironment.${field}`) return { compatible: differences.length === 0, differences } } + +// A side cannot publish a single image pin for runs graded with different images. +export function sideGraderImageDigest(runs: readonly { graderImageDigest?: string }[]): string { + const digests = new Set(runs.flatMap((run) => (run.graderImageDigest === undefined ? [] : [run.graderImageDigest]))) + if (digests.size > 1) throw new Error("grader image digest changed within the side") + return digests.values().next().value ?? "unavailable" +} diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index a81eefe689..850ddd42d8 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict" import fs from "node:fs/promises" +import { existsSync } from "node:fs" import os from "node:os" import path from "node:path" import { describe, it } from "node:test" @@ -554,3 +555,54 @@ describe("side artifact invalidation", () => { } }) }) + +for (const incompleteSide of ["baseline", "candidate"] as const) { + it(`excludes an available estimate with incomplete ${incompleteSide} accounting`, () => { + const baseline = metrics("clamp-range", 0, "correct", 0.1) + const candidate = metrics("clamp-range", 0, "correct", 0.05) + const incomplete = incompleteSide === "baseline" ? baseline : candidate + incomplete.cost.accountingComplete = false + const report = buildComparisonReport({ + baseline: sideResult("baseline", "a".repeat(40), [baseline]), + candidate: sideResult("candidate", "b".repeat(40), [candidate]), + }) + const row = report.fixtures.find((fixture) => fixture.fixtureName === "clamp-range")! + assert.equal(row.pairedCosts.missingPairs, 1) + assert.deepEqual(row.pairedCosts.pairs, []) + const markdown = renderMarkdownReport(report) + assert.ok( + markdown.includes( + `Grader image digest: \`${report.pins.baseline.graderImageDigest}\` / \`${report.pins.candidate.graderImageDigest}\``, + ), + ) + assert.match(markdown, /identifier and date are recorded per run in the JSON report/) + }) +} + +for (const boundary of ["before", "json", "markdown", "marker"] as const) { + it(`removes report publication interrupted at ${boundary}`, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-publication-")) + try { + const report = buildComparisonReport({ + baseline: sideResult("baseline", "a".repeat(40), []), + candidate: sideResult("candidate", "b".repeat(40), []), + }) + const names = { + json: "comparison-report.json", + markdown: "comparison-report.md", + marker: "comparison-report.complete", + } + await assert.rejects( + publishComparisonReport( + dir, + report, + () => boundary === "before" || existsSync(path.join(dir, names[boundary])), + ), + /publication interrupted/, + ) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/coding-report.ts b/scripts/agent-evals/coding-report.ts index 366cda4d98..c51dc72ceb 100644 --- a/scripts/agent-evals/coding-report.ts +++ b/scripts/agent-evals/coding-report.ts @@ -88,8 +88,12 @@ export function buildComparisonReport(inputs: ComparisonInputs): ComparisonRepor candidateRuns.map((run) => run.durationMs), ), pairedCosts: pairedNumericDiffs( - baselineRuns.map((run) => (run.cost.estimated.available ? run.cost.estimated.usd : undefined)), - candidateRuns.map((run) => (run.cost.estimated.available ? run.cost.estimated.usd : undefined)), + baselineRuns.map((run) => + run.cost.accountingComplete && run.cost.estimated.available ? run.cost.estimated.usd : undefined, + ), + candidateRuns.map((run) => + run.cost.accountingComplete && run.cost.estimated.available ? run.cost.estimated.usd : undefined, + ), ), }) } @@ -161,6 +165,9 @@ export function renderMarkdownReport(report: ComparisonReport): string { ) lines.push(`- Model: ${report.pins.baseline.modelIdentifier} / ${report.pins.candidate.modelIdentifier}`) lines.push(`- Grader version: ${report.pins.baseline.graderVersion} / ${report.pins.candidate.graderVersion}`) + lines.push( + `- Grader image digest: \`${report.pins.baseline.graderImageDigest}\` / \`${report.pins.candidate.graderImageDigest}\``, + ) lines.push( `- Runner environment: ${report.pins.baseline.runnerEnvironment.os}, node ${report.pins.baseline.runnerEnvironment.nodeVersion}, vscode ${report.pins.baseline.runnerEnvironment.vscodeTarget} / ${report.pins.candidate.runnerEnvironment.os}, node ${report.pins.candidate.runnerEnvironment.nodeVersion}, vscode ${report.pins.candidate.runnerEnvironment.vscodeTarget}`, ) @@ -200,7 +207,9 @@ export function renderMarkdownReport(report: ComparisonReport): string { lines.push("") lines.push(`- Baseline: ${costCell(report.costSummary.baseline)}`) lines.push(`- Candidate: ${costCell(report.costSummary.candidate)}`) - lines.push(`- Price table: recorded per run with identifier and date; cache reads and writes priced separately.`) + lines.push( + `- Price table: identifier and date are recorded per run in the JSON report; cache reads and writes priced separately.`, + ) lines.push("") lines.push("## Unavailable telemetry") lines.push("") diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts index cbef1e6956..d07d260739 100644 --- a/scripts/agent-evals/coding-run.test.ts +++ b/scripts/agent-evals/coding-run.test.ts @@ -141,12 +141,16 @@ describe("coding run orchestration", () => { repIndex: 0, credential: "sk-test", runtime, - gradingRuntime: createLocalSubprocessGradingRuntime(), + gradingRuntime: { + ...createLocalSubprocessGradingRuntime(), + imageDigest: async () => ({ available: true, value: "sha256:run-image" }), + }, scratchDir: await scratch(), }) assert.equal(record.outcome.outcome, "correct") assert.equal(record.executionCode, "COMPLETED") assert.equal(record.grading?.kind, "passed") + assert.equal(record.graderImageDigest, "sha256:run-image") assert.equal(record.metrics.outcome.outcome, "correct") // Usage lacks cache counts, so cost accounting stays incomplete. assert.equal(record.metrics.cost.accountingComplete, false) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts index 07914237d2..22c4bafcd9 100644 --- a/scripts/agent-evals/coding-run.ts +++ b/scripts/agent-evals/coding-run.ts @@ -20,6 +20,7 @@ export interface CodingRunRecord { outcome: ClassifiedOutcome executionCode?: string trace?: AgentTrace + graderImageDigest?: string grading?: { kind: GraderVerdict["kind"] isolation?: string @@ -131,6 +132,7 @@ export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { + try { + return await inputs.gradingRuntime.run(job) + } finally { + const digest = await inputs.gradingRuntime.imageDigest?.() + graderImageDigest = digest?.available ? digest.value : "unavailable" + } + }, + }, scratchDir: inputs.scratchDir, }) } @@ -167,6 +179,7 @@ export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { isInterrupted: () => interrupted, runRepetition: async (repetition) => { executed.push(repetition.repIndex) + // The live substrate writes artifacts during the repetition. + const artifactDir = path.join(dir, "artifacts", "clamp-range-rep0") + await fs.mkdir(artifactDir, { recursive: true }) + await fs.writeFile(path.join(artifactDir, "trace.json"), "{}") // The signal lands during the first repetition. interrupted = true return record(repetition, true) diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts index 1de68ba6fd..8d2a463cd5 100644 --- a/scripts/agent-evals/coding-side-flow.ts +++ b/scripts/agent-evals/coding-side-flow.ts @@ -48,10 +48,16 @@ const describeRepetitionError = (error: unknown): string => export async function executeSideFlow(inputs: SideFlowInputs): Promise { const runs: CodingRunRecord[] = [] + const abortPublication = async (): Promise => { + await fs.rm(path.join(inputs.out, "side-results.json"), { force: true }) + await fs.rm(path.join(inputs.out, "artifacts"), { recursive: true, force: true }) + return { runs, completed: false } + } + for (const repetition of inputs.repetitions) { // An interrupt stops the loop before the next repetition starts. The // in-flight repetition is left to the substrate's own teardown. - if (inputs.isInterrupted()) return { runs, completed: false } + if (inputs.isInterrupted()) return abortPublication() try { runs.push(await inputs.runRepetition(repetition)) } catch (error) { @@ -79,12 +85,7 @@ export async function executeSideFlow(inputs: SideFlowInputs): Promise => { - await fs.rm(path.join(inputs.out, "side-results.json"), { force: true }) - await fs.rm(path.join(inputs.out, "artifacts"), { recursive: true, force: true }) - return { runs, completed: false } - } + if (inputs.isInterrupted()) return abortPublication() // Per-run artifacts are published atomically as each record exists. for (const record of runs) { if (!record.trace) continue diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index d6f647da72..9dd2509e67 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -20,12 +20,12 @@ import { fileURLToPath } from "node:url" import { promisify } from "node:util" import { createHostRuntime } from "./launcher.ts" import { createContainerGradingRuntime } from "./container-runtime.ts" -import { runCodingFixtureOnce } from "./coding-run.ts" +import { runCodingFixtureOnce, type CodingRunRecord } from "./coding-run.ts" import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from "./coding-suite.ts" import { GRADER_VERSION } from "./coding-grader.ts" import { digestValue } from "./coding-digest.ts" -import type { SidePins } from "./coding-pins.ts" +import { sideGraderImageDigest, type SidePins } from "./coding-pins.ts" import { invalidateSideArtifacts, runBoundedPreparation } from "./coding-compare.ts" import { assertCommitsApproved, @@ -195,8 +195,7 @@ const repetitions: SideRepetition[] = CODING_FIXTURE_BUNDLES.flatMap((bundle) => // Pins are recorded when the side publishes, so they describe what actually // executed: the approved evaluated commit and, when the container runtime // resolved one, the immutable grader image identity. -const buildPins = async (): Promise => { - const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined +const buildPins = async (runs: CodingRunRecord[]): Promise => { return { side: sideLabel, commit: evaluatedCommit, @@ -205,7 +204,7 @@ const buildPins = async (): Promise => { settingsDigest: digestValue(projection), modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", graderVersion: GRADER_VERSION, - graderImageDigest: digest?.available === true ? digest.value : "unavailable", + graderImageDigest: sideGraderImageDigest(runs), runnerEnvironment: { os: process.platform, nodeVersion: process.version, @@ -283,7 +282,7 @@ try { // so a reader that sees the file sees a complete side of the current // run. A missing file means an interrupted side; consumers must treat // it as a failed side and never pair partial output. - buildResult: async (runs) => ({ pins: await buildPins(), runs }), + buildResult: async (runs) => ({ pins: await buildPins(runs), runs }), }) if (!completed) { diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts index de1f5e9396..8539d179af 100644 --- a/scripts/agent-evals/launcher.test.ts +++ b/scripts/agent-evals/launcher.test.ts @@ -1,3 +1,5 @@ +import { execFile } from "node:child_process" +import { promisify } from "node:util" import assert from "node:assert/strict" import fs from "node:fs/promises" import os from "node:os" @@ -95,3 +97,41 @@ for (const scenario of ["nested", "oversized", "total", "count"] as const) { } }) } + +it("reads commit identity from the evaluated root despite an inherited GITHUB_SHA", async () => { + const dir = await tempDir("identity-ci-") + const saved = process.env.GITHUB_SHA + try { + process.env.GITHUB_SHA = "f".repeat(40) + const { stdout } = await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: repoRoot }) + const identity = await createHostRuntime({ root: repoRoot, artifactsDir: dir }).readIdentity?.() + assert.deepEqual(identity?.commit, { available: true, value: stdout.trim() }) + } finally { + if (saved === undefined) delete process.env.GITHUB_SHA + else process.env.GITHUB_SHA = saved + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +for (const scenario of ["file", "total"] as const) { + it(`checks actual workspace bytes after stale ${scenario} stat results`, async (t) => { + const dir = await tempDir("workspace-growth-") + const stat = fs.stat + try { + const count = scenario === "file" ? 1 : 9 + for (let index = 0; index < count; index++) + await fs.writeFile(path.join(dir, `${index}.js`), "x".repeat(scenario === "file" ? 1048577 : 1048576)) + t.mock.method(fs, "stat", async (file: string) => { + const entry = await stat(file) + entry.size = 0 + return entry + }) + const snapshot = await readWorkspaceFiles(dir) + assert.equal(snapshot.truncated, true) + assert.equal(snapshot.files.length, scenario === "file" ? 0 : 8) + } finally { + t.mock.restoreAll() + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 64dfc95252..2e3afced56 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -33,8 +33,6 @@ export interface HostRuntimeOptions { const readCommit = async (root: string): Promise => { try { - const fromCi = process.env.GITHUB_SHA - if (fromCi && /^[0-9a-f]{40}$/i.test(fromCi)) return { available: true, value: fromCi } const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) const sha = stdout.trim() return /^[0-9a-f]{40}$/i.test(sha) @@ -94,9 +92,16 @@ export async function readWorkspaceFiles(workspaceRoot: string): Promise FINAL_WORKSPACE_MAX_FILE_BYTES || + totalBytes + bytes.byteLength > FINAL_WORKSPACE_MAX_TOTAL_BYTES + ) { + truncated = true + continue + } + totalBytes += bytes.byteLength + files.push({ path: relativePath, content: bytes.toString("utf8") }) } } await visit(workspaceRoot, "") diff --git a/scripts/agent-evals/preparation-environment.test.ts b/scripts/agent-evals/preparation-environment.test.ts index f5a97b904b..fca1770132 100644 --- a/scripts/agent-evals/preparation-environment.test.ts +++ b/scripts/agent-evals/preparation-environment.test.ts @@ -61,3 +61,45 @@ it("preparation children use isolated home and npm configuration without registr await fs.rm(dir, { recursive: true, force: true }) } }) + +it("kills background preparation workers after their parent succeeds", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "preparation-worker-")) + const heartbeat = path.join(dir, "heartbeat") + const pidFile = path.join(dir, "worker-pid") + try { + const worker = + "const fs = require('node:fs'); const file = process.argv[1]; fs.writeFileSync(file, 'x'); process.stdout.write('ready'); setInterval(() => fs.appendFileSync(file, 'x'), 20)" + await runBoundedPreparation( + createSideProcessGuard(100), + () => false, + process.execPath, + [ + "-e", + `const { spawn } = require('node:child_process'); + const fs = require('node:fs'); + const [worker, heartbeat, pidFile] = process.argv.slice(1); + const child = spawn(process.execPath, ['-e', worker, heartbeat], { stdio: ['ignore', 'pipe', 'ignore'] }); + fs.writeFileSync(pidFile, String(child.pid)); + child.stdout.once('data', () => process.exit(0));`, + worker, + heartbeat, + pidFile, + ], + dir, + 5000, + ) + const before = (await fs.stat(heartbeat)).size + await new Promise((resolve) => setTimeout(resolve, 200)) + assert.equal((await fs.stat(heartbeat)).size, before) + } finally { + const pid = await fs.readFile(pidFile, "utf8").then(Number, () => undefined) + if (pid !== undefined) { + try { + process.kill(pid, "SIGKILL") + } catch { + /* Already exited. */ + } + } + await fs.rm(dir, { recursive: true, force: true }) + } +}) From 6f381c80f25af6b6761e931b988322e200bd2d3c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 3 Nov 2026 00:46:53 +0000 Subject: [PATCH 14/22] fix(evals): bind grader images and isolate host caches --- scripts/agent-evals/README.md | 6 +- scripts/agent-evals/coding-fixtures.test.ts | 2 +- scripts/agent-evals/coding-grader.ts | 2 +- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/coding-side-flow.test.ts | 63 +++++++++++++++++++ scripts/agent-evals/coding-side-flow.ts | 11 ++++ scripts/agent-evals/coding-side.mts | 1 - scripts/agent-evals/container-runtime.test.ts | 39 +++++++++++- scripts/agent-evals/container-runtime.ts | 12 ++-- scripts/agent-evals/launcher.test.ts | 53 ++++++++++++++++ scripts/agent-evals/launcher.ts | 10 ++- 11 files changed, 180 insertions(+), 21 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index fecdc42f27..b4ece269cc 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -131,7 +131,7 @@ release process: a maintainer runs it locally or on demand. - The grader is model-free and credential-free. It judges the final workspace state only. - The final workspace is captured as bounded file data and rebuilt as a fresh copy. Hidden acceptance tests enter only that copy, so the agent can never read or change them, and one run can never inherit another run's files. - The default runtime executes the driver in a uniquely named disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. -- The grader has its own wall-clock limit. On expiry the runtime removes the container by name and verifies the removal before it deletes any output. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. The image pull happens before the timed run and never consumes grading time. The runtime resolves the immutable grader image digest from the local engine, and each graded run records its own identity. The side records that identity in the comparison pins and refuses to publish if run identities differ; it never fabricates a digest. +- The grader has its own wall-clock limit. On expiry the runtime removes the container by name and verifies the removal before it deletes any output. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. The image pull happens before the timed run and never consumes grading time. The runtime resolves the immutable grader image digest from the local engine, and each graded run records its own identity. The container launches by that resolved identity rather than the mutable tag, and grading fails if no immutable identity is available. The side records that identity in the comparison pins and refuses to publish if run identities differ; it never fabricates a digest. - The driver's result file is untrusted data: it is written inside the container where agent-written test code executes. The host reads it with a size bound and validates shape, bounds, and internal consistency. An invalid or inconsistent result is a `grading failure`, never a verdict. - Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. - `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. @@ -158,7 +158,7 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. The driver creates each side's fresh detached worktree of the approved commit. Each side then constructs its own fresh detached worktree of that same commit, installs the frozen lockfile dependencies and builds inside it, and launches every host against it, so each side runs only its own compiled extension of the approved commit. The install and the build consume the side's own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children receive an allowlisted environment and a temporary HOME with isolated XDG and npm configuration. Parent package-manager configuration and registry URLs are not forwarded. The prepared tree is checked against the raw bytes in the approved commit, and tracked symlinks are rejected. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The launcher downloads VS Code into fresh private per-run scratch space and removes the cache during cleanup. It never executes a cached VS Code binary from the caller checkout. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children receive an allowlisted environment and a temporary HOME with isolated XDG and npm configuration. Parent package-manager configuration and registry URLs are not forwarded. The prepared tree is checked against the raw bytes in the approved commit, and tracked symlinks are rejected. Every driver child, the side included, runs detached in its own process group and is tracked by one guard. The driver's guard tracks the side; the side's guard tracks its own worktree, install, and build. A SIGINT or SIGTERM that reaches only the driver terminates the side group through a bounded SIGTERM-then-SIGKILL escalation. The side starts the same escalation for its preparation children and waits for it before it removes its worktree or exits. The driver's worktree cleanup also waits for the driver's escalation before it removes any tree. @@ -166,7 +166,7 @@ Every driver child, the side included, runs detached in its own process group an Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. -A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and then the whole `artifacts` tree, so stale results from an earlier rerun can never survive. The authoritative result goes first, so an error or an interruption during invalidation leaves the generation incomplete, and consumers fail closed until the new atomic publication. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. +A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and then the whole `artifacts` tree, so stale results from an earlier rerun can never survive. The authoritative result goes first, so an error or an interruption during invalidation leaves the generation incomplete, and consumers fail closed until the new atomic publication. A repetition that throws has its partial artifact directory removed before the next repetition starts. If that cleanup fails, the side removes its completion marker and artifact tree and fails without publishing. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. The side process arms one flag when it receives SIGINT or SIGTERM. The flag starts no further repetition and blocks the completion marker, so an interrupted side exits with the conventional signal code and publishes nothing. diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index df1d601bf5..cf9b207b33 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -91,6 +91,6 @@ describe("coding fixture suite", () => { assert.throws(() => codingBundle("unknown-fixture")) assert.ok(codingSuiteDigest().length === 64) assert.equal(CODING_SUITE_ID, "coding-basic-v1") - assert.equal(GRADER_VERSION, "coding-grader-v4") + assert.equal(GRADER_VERSION, "coding-grader-v5") }) }) diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts index cee3b5adc4..4280b999ad 100644 --- a/scripts/agent-evals/coding-grader.ts +++ b/scripts/agent-evals/coding-grader.ts @@ -10,7 +10,7 @@ import type { Measured } from "./trace.ts" // The grader version is a pinned comparison input. Any change to the driver, // the image, or the verdict rules must bump it. -export const GRADER_VERSION = "coding-grader-v4" +export const GRADER_VERSION = "coding-grader-v5" export interface GraderResources { timeoutMs: number diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index b893830f39..54d8e6799a 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -66,7 +66,7 @@ describe("coding graders against reference solutions", () => { } it("pins one grader version across the suite", () => { - assert.equal(GRADER_VERSION, "coding-grader-v4") + assert.equal(GRADER_VERSION, "coding-grader-v5") assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) }) }) diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts index 5acc83482d..1af7299611 100644 --- a/scripts/agent-evals/coding-side-flow.test.ts +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -270,3 +270,66 @@ for (const boundary of ["directory", "record", "pins", "marker"] as const) { } }) } + +it("removes a failed repetition's partial artifacts before continuing", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-failed-artifacts-")) + const partial = path.join(dir, "artifacts/clamp-range-rep0") + try { + const outcome = await flow({ + repetitions: plan(2), + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + if (repetition.repIndex === 0) { + await fs.mkdir(partial, { recursive: true }) + await fs.writeFile(path.join(partial, "trace.json"), "partial") + throw new Error("failed after writing trace") + } + await assert.rejects(fs.access(partial)) + return record(repetition, true) + }, + }) + assert.equal(outcome.completed, true) + assert.equal(outcome.runs[0]?.outcome.outcome, "infrastructure-failure") + assert.deepEqual(await fs.readdir(path.join(dir, "artifacts")), ["clamp-range-rep1"]) + await fs.access(path.join(dir, "side-results.json")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +it("aborts publication and removes all artifacts if failed-repetition cleanup fails", async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-cleanup-failure-")) + const partial = path.join(dir, "artifacts/clamp-range-rep0") + const remove = fs.rm + const lines: string[] = [] + t.mock.method(console, "error", (line: string) => lines.push(line)) + try { + const executed: number[] = [] + await assert.rejects( + flow({ + repetitions: plan(2), + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + executed.push(repetition.repIndex) + await fs.mkdir(partial, { recursive: true }) + await fs.writeFile(path.join(partial, "trace.json"), "partial") + await fs.writeFile(path.join(dir, "side-results.json"), "stale") + t.mock.method(fs, "rm", async (target: string, options: Parameters[1]) => { + if (target === partial) throw new Error("cleanup denied") + await remove(target, options) + }) + throw new Error("repetition failed") + }, + }), + /repetition artifact cleanup failed/, + ) + assert.deepEqual(executed, [0]) + assert.deepEqual(await fs.readdir(dir), []) + assert.match(lines[0] ?? "", /repetition cleanup failed: Error: cleanup denied/) + } finally { + t.mock.restoreAll() + await remove(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts index 8d2a463cd5..a6ae0240ba 100644 --- a/scripts/agent-evals/coding-side-flow.ts +++ b/scripts/agent-evals/coding-side-flow.ts @@ -61,6 +61,17 @@ export async function executeSideFlow(inputs: SideFlowInputs): Promise { // The second run finds no digest before or after the pull. The // runtime must report the identity as unavailable instead of reusing // the digest of the image the first run graded with. - await runtime.run(makeJob()) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderImageDigestUnavailable", + }) + assert.equal(cli.runArgs.length, 1) // The missed lookups sent the second run to the pull path. assert.equal(cli.calls.filter((call) => call.startsWith("pull")).length, 1) const second = await runtime.imageDigest!() @@ -402,3 +408,34 @@ describe("bounded cli output capture", () => { assert.equal(outcome.stderr, "no such container: gone") }) }) + +for (const identity of ["node@sha256:" + "c".repeat(64), "sha256:" + "d".repeat(64)]) { + it(`launches the exact inspected image identity ${identity}`, async () => { + const cli = recordingCli({ imageDigest: async () => identity }) + const runtime = createContainerGradingRuntime({ cli }) + assert.equal((await runtime.run(makeJob())).kind, "passed") + const args = cli.runArgs[0]! + assert.deepEqual(args.slice(args.indexOf(identity)), [ + identity, + "node", + "/driver/grader-driver.mjs", + "/out/result.json", + "test", + ]) + assert.equal(args.includes(GRADER_IMAGE), false) + assert.deepEqual(await runtime.imageDigest!(), { available: true, value: identity }) + }) +} + +it("refuses to run when inspection remains unavailable after a successful pull", async () => { + const cli = recordingCli({ imageDigest: async () => undefined }) + const runtime = createContainerGradingRuntime({ cli }) + assert.deepEqual(await runtime.run(makeJob()), { + kind: "grading-failure", + isolation: "container", + reason: "graderImageDigestUnavailable", + }) + assert.deepEqual(cli.calls, [`pull ${GRADER_IMAGE}`]) + assert.deepEqual(cli.runArgs, []) + assert.deepEqual(await runtime.imageDigest!(), { available: false, reason: "digestUnavailable" }) +}) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index b09efef745..58ac8a0aa1 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -5,9 +5,8 @@ import path from "node:path" import type { GraderJob, GraderVerdict, GradingRuntime, DriverResult } from "./coding-grader.ts" import type { Measured } from "./trace.ts" -// Pinned grader image tag. The digest resolved from the local engine travels -// in the comparison pins, so the recorded identity is immutable even though -// the tag alone is not. +// Grader image tag used only for lookup and pull. Containers launch by the +// resolved immutable identity, which is also recorded in the comparison pins. export const GRADER_IMAGE = "node:22-bookworm" export type ContainerExecutable = "docker" | "podman" @@ -235,9 +234,8 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") digest = await selected.imageDigest(job.resources.image) } - // A lookup that still finds nothing leaves the digest unset, and - // the pins record the identity as unavailable instead of reusing - // one from an earlier run. The runtime never fabricates a value. + // Never launch a mutable tag when its immutable identity is unavailable. + if (digest === undefined) return failure("graderImageDigestUnavailable") resolvedDigest = digest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) @@ -246,7 +244,7 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = // write to any uid. await fs.chmod(outDir, 0o777) const name = containerNameFor() - const args = buildContainerArgs(job, outDir, name) + const args = buildContainerArgs({ ...job, resources: { ...job.resources, image: digest } }, outDir, name) let exit: number | "timeout" | "error" try { exit = await selected.run(args, job.resources.timeoutMs) diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts index 8539d179af..8218646bd3 100644 --- a/scripts/agent-evals/launcher.test.ts +++ b/scripts/agent-evals/launcher.test.ts @@ -135,3 +135,56 @@ for (const scenario of ["file", "total"] as const) { } }) } + +it("ignores poisoned caller VS Code caches and removes its fresh private cache", async () => { + const dir = await tempDir("launcher-cache-poison-") + const cache = path.join(dir, "apps/vscode-e2e/.vscode-test") + const marker = path.join(dir, "poison-executed") + let selectedCache: string | undefined + const runtime = createHostRuntime({ + root: dir, + artifactsDir: dir, + downloadVSCode: async (options) => { + assert.ok(typeof options === "object" && options !== null) + selectedCache = options.cachePath + assert.ok(selectedCache) + // Mimic a cache hit: a regression selecting the caller's cache + // executes the planted binary instead of attempting a download. + const executable = path.join(selectedCache, "code") + if ( + await fs.access(executable).then( + () => true, + () => false, + ) + ) { + await promisify(execFile)(process.execPath, [executable, marker]) + return executable + } + assert.notEqual(selectedCache, cache) + assert.equal(path.basename(selectedCache), "vscode-cache") + assert.ok(path.basename(path.dirname(selectedCache)).startsWith("agent-evals-")) + throw new Error("fresh cache requires download") + }, + }) + try { + await fs.mkdir(path.join(dir, "src")) + await fs.writeFile(path.join(dir, "src/package.json"), "{}") + await fs.mkdir(cache, { recursive: true }) + await fs.writeFile(path.join(dir, ".gitignore"), "apps/vscode-e2e/.vscode-test/\n") + await fs.writeFile( + path.join(dir, "apps/vscode-e2e/package.json"), + JSON.stringify({ devDependencies: { "@types/vscode": "1.104.0" } }), + ) + await fs.writeFile(path.join(cache, "code"), "require('node:fs').writeFileSync(process.argv[2], 'executed')") + await assert.rejects(runtime.launch([]), /fresh cache requires download/) + await assert.rejects(fs.access(marker)) + await fs.access(path.join(cache, "code")) + assert.ok(selectedCache) + await fs.mkdir(selectedCache) + await runtime.cleanup() + await assert.rejects(fs.access(selectedCache)) + } finally { + await runtime.cleanup() + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 2e3afced56..f69c6106b2 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -18,10 +18,8 @@ export interface HostRuntimeOptions { // commit identity, and the extension source all stay inside it. root: string artifactsDir: string - // Optional trusted cache for the downloaded VS Code build. Defaults to - // the evaluated root, so callers driving historical worktrees can keep - // one cache in the harness checkout. - vscodeCachePath?: string + // Test seam for the download boundary; caches always live in private scratch. + downloadVSCode?: typeof downloadAndUnzipVSCode now?: () => number pause?: (ms: number) => Promise // Optional --extensionTestsPath module. The module's run() must never @@ -137,9 +135,9 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { devDependencies: Record } - const executable = await downloadAndUnzipVSCode({ + const executable = await (options.downloadVSCode ?? downloadAndUnzipVSCode)({ version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", - cachePath: options.vscodeCachePath ?? path.join(root, "apps/vscode-e2e/.vscode-test"), + cachePath: path.join(temp, "vscode-cache"), }) const args = buildLaunchArgs( workspace, From 87901d14587f1726b9c6e29267e9ac19e533c83c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 3 Nov 2026 01:42:06 +0000 Subject: [PATCH 15/22] fix(evals): clean up partial side publication on failure --- scripts/agent-evals/coding-side-flow.test.ts | 88 ++++++++++++++++++++ scripts/agent-evals/coding-side-flow.ts | 47 ++++++++--- 2 files changed, 122 insertions(+), 13 deletions(-) diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts index 1af7299611..be1e4ac499 100644 --- a/scripts/agent-evals/coding-side-flow.test.ts +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -333,3 +333,91 @@ it("aborts publication and removes all artifacts if failed-repetition cleanup fa await remove(dir, { recursive: true, force: true }) } }) + +for (const boundary of ["later artifact", "result construction", "result write"] as const) { + it(`removes all publication output when ${boundary} fails`, async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-publication-failure-")) + const failure = new Error(`${boundary} failed`) + const rename = fs.rename + try { + // Include an existing marker and substrate artifacts in the rollback. + await fs.writeFile(path.join(dir, "side-results.json"), "stale") + t.mock.method(fs, "rename", async (...args: Parameters) => { + const target = String(args[1]) + if ( + (boundary === "later artifact" && target.endsWith("clamp-range-rep1/run-record.json")) || + (boundary === "result write" && target === path.join(dir, "side-results.json")) + ) { + await fs.access(path.join(dir, "artifacts/clamp-range-rep0/run-record.json")) + throw failure + } + await rename(...args) + }) + await assert.rejects( + executeSideFlow({ + repetitions: plan(2), + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + const artifactDir = path.join(dir, "artifacts", `clamp-range-rep${repetition.repIndex}`) + await fs.mkdir(artifactDir, { recursive: true }) + await fs.writeFile(path.join(artifactDir, "trace.json"), "{}") + return record(repetition, true) + }, + buildResult: async (runs) => { + await fs.access(path.join(dir, "artifacts/clamp-range-rep1/run-record.json")) + if (boundary === "result construction") throw failure + return { pins: pins(), runs } + }, + }), + (error: unknown) => error === failure, + ) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + t.mock.restoreAll() + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} + +for (const failedRemoval of ["side-results.json", "artifacts"] as const) { + it(`reports failed ${failedRemoval} cleanup and still attempts the other removal`, async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-publication-cleanup-")) + const remove = fs.rm + const failure = new Error("result construction failed") + const cleanupFailure = new Error("cleanup denied") + const lines: string[] = [] + t.mock.method(console, "error", (line: string) => lines.push(line)) + try { + await fs.writeFile(path.join(dir, "side-results.json"), "stale") + t.mock.method(fs, "rm", async (target: string, options: Parameters[1]) => { + if (target === path.join(dir, failedRemoval)) throw cleanupFailure + await remove(target, options) + }) + await assert.rejects( + executeSideFlow({ + repetitions: plan(1), + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => record(repetition, true), + buildResult: () => { + throw failure + }, + }), + (error: unknown) => { + assert.ok(error instanceof AggregateError) + assert.equal(error.errors[0], failure) + const cleanup = error.errors[1] + assert.ok(cleanup instanceof AggregateError) + assert.deepEqual(cleanup.errors, [cleanupFailure]) + return true + }, + ) + assert.deepEqual(await fs.readdir(dir), [failedRemoval]) + assert.deepEqual(lines, ["coding-evals publication cleanup failed: Error: cleanup denied"]) + } finally { + t.mock.restoreAll() + await remove(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts index a6ae0240ba..242e00633f 100644 --- a/scripts/agent-evals/coding-side-flow.ts +++ b/scripts/agent-evals/coding-side-flow.ts @@ -49,8 +49,20 @@ const describeRepetitionError = (error: unknown): string => export async function executeSideFlow(inputs: SideFlowInputs): Promise { const runs: CodingRunRecord[] = [] const abortPublication = async (): Promise => { - await fs.rm(path.join(inputs.out, "side-results.json"), { force: true }) - await fs.rm(path.join(inputs.out, "artifacts"), { recursive: true, force: true }) + // Attempt both removals even if one fails. Never report completion when + // publication or its cleanup failed. + const cleanup = await Promise.allSettled([ + fs.rm(path.join(inputs.out, "side-results.json"), { force: true }), + fs.rm(path.join(inputs.out, "artifacts"), { recursive: true, force: true }), + ]) + const errors: unknown[] = [] + for (const result of cleanup) { + if (result.status === "rejected") { + errors.push(result.reason) + console.error(`coding-evals publication cleanup failed: ${describeRepetitionError(result.reason)}`) + } + } + if (errors.length > 0) throw new AggregateError(errors, "publication cleanup failed") return { runs, completed: false } } @@ -97,18 +109,27 @@ export async function executeSideFlow(inputs: SideFlowInputs): Promise Date: Tue, 3 Nov 2026 02:52:42 +0000 Subject: [PATCH 16/22] fix(agent-evals): prevent credential leaks to grading subprocesses --- scripts/agent-evals/coding-compare.ts | 2 + scripts/agent-evals/coding-side.mts | 2 + scripts/agent-evals/coding-trust.ts | 5 +- scripts/agent-evals/container-runtime.test.ts | 31 ++++++++++++ scripts/agent-evals/container-runtime.ts | 50 ++++++++++++++----- scripts/agent-evals/launcher.ts | 7 ++- 6 files changed, 83 insertions(+), 14 deletions(-) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 2ad6806dce..c42cdd9e99 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -289,10 +289,12 @@ export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promi await execFileAsync("git", ["worktree", "remove", "--force", path.join(wiring.scratchDir, "baseline")], { cwd: wiring.root, timeout: WORKTREE_REMOVE_TIMEOUT_MS, + env: buildWorktreeEnvironment(process.env), }).catch(() => {}) await execFileAsync("git", ["worktree", "remove", "--force", path.join(wiring.scratchDir, "candidate")], { cwd: wiring.root, timeout: WORKTREE_REMOVE_TIMEOUT_MS, + env: buildWorktreeEnvironment(process.env), }).catch(() => {}) await fs.rm(wiring.scratchDir, { recursive: true, force: true }).catch(() => {}) } diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 475d7e22aa..ecc059c78e 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -29,6 +29,7 @@ import { sideGraderImageDigest, type SidePins } from "./coding-pins.ts" import { invalidateSideArtifacts, runBoundedPreparation } from "./coding-compare.ts" import { assertCommitsApproved, + buildWorktreeEnvironment, assertCredentialEnvAllowed, assertEvaluatedTreeClean, loadTrustPolicy, @@ -174,6 +175,7 @@ const removePreparedTree = async (): Promise => { await execFileAsync("git", ["worktree", "remove", "--force", evaluatedWorktree], { cwd: evaluatedRoot, timeout: WORKTREE_REMOVE_TIMEOUT_MS, + env: buildWorktreeEnvironment(process.env), }).catch((error: unknown) => { console.error(`coding-evals side: failed to remove worktree ${evaluatedWorktree}: ${String(error)}`) }) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 627ecd5729..111087fe22 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -50,6 +50,7 @@ export const resolveCommit = async (ref: string, root: string): Promise const { promisify } = await import("node:util") const { stdout } = await promisify(execFile)("git", ["rev-parse", `${ref}^{commit}`], { cwd: root, + env: buildWorktreeEnvironment(process.env), timeout: 30_000, }) const sha = stdout.trim().toLowerCase() @@ -93,6 +94,7 @@ export async function assertEvaluatedTreeClean(root: string): Promise { ["status", "--porcelain", "--untracked-files=all", "--ignore-submodules=none"], { cwd: root, + env: buildWorktreeEnvironment(process.env), timeout: 30_000, }, ) @@ -105,6 +107,7 @@ export async function assertEvaluatedTreeClean(root: string): Promise { // clean filters. A symlink can reference unapproved bytes outside this tree. const { stdout: tree } = await promisify(execFile)("git", ["ls-tree", "-rz", "HEAD"], { cwd: root, + env: buildWorktreeEnvironment(process.env), maxBuffer: 16 * 1024 * 1024, }) const blobs: { file: string; object: string }[] = [] @@ -123,7 +126,7 @@ export async function assertEvaluatedTreeClean(root: string): Promise { const { stdout: hashes } = await promisify(execFile)( "git", ["hash-object", "--no-filters", "--", ...batch.map((blob) => blob.file)], - { cwd: root, timeout: 30_000 }, + { cwd: root, env: buildWorktreeEnvironment(process.env), timeout: 30_000 }, ) const actual = hashes.trim().split("\n") for (const [index, blob] of batch.entries()) { diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 2b6449d2e7..ae3da13ed6 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -1,4 +1,6 @@ import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { promisify } from "node:util" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -14,6 +16,35 @@ import { } from "./container-runtime.ts" import type { GraderJob } from "./coding-grader.ts" +it("sanitizes PATH children and ignores PATH for container detection", async () => { + const temp = await fs.mkdtemp(path.join(os.tmpdir(), "container-path-boundary-")) + try { + const marker = path.join(temp, "observed") + for (const name of ["docker", "podman"]) { + await fs.writeFile( + path.join(temp, name), + `#!/bin/sh\nprintf '%s:%s' "$SOLHEIM_API_KEY" "$UNRELATED_SECRET" > '${marker}'\nprintf '%s:%s' "$SOLHEIM_API_KEY" "$UNRELATED_SECRET"\n`, + { mode: 0o700 }, + ) + } + const moduleUrl = new URL("./container-runtime.ts", import.meta.url).href + const script = ` + const { exec, createContainerGradingRuntime } = await import(${JSON.stringify(moduleUrl)}); + const result = await exec("docker", ["--version"], 5000); + if (result.code !== 0 || result.stdout !== ":") throw new Error("child received secrets"); + await (await import("node:fs/promises")).unlink(${JSON.stringify(marker)}); + await createContainerGradingRuntime().imageDigest(); + ` + await promisify(execFile)(process.execPath, ["--import", "tsx", "--input-type=module", "-e", script], { + env: { ...process.env, PATH: temp, SOLHEIM_API_KEY: "provider-secret", UNRELATED_SECRET: "other-secret" }, + timeout: 15_000, + }) + await assert.rejects(fs.access(marker), { code: "ENOENT" }) + } finally { + await fs.rm(temp, { recursive: true, force: true }) + } +}) + const makeJob = (): GraderJob => ({ gradingDir: "/tmp/evals/grading-copy", driverPath: "/repo/scripts/agent-evals/grader-driver.mjs", diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 58ac8a0aa1..6b228fd484 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process" +import { buildWorktreeEnvironment } from "./coding-trust.ts" import { randomUUID } from "node:crypto" import fs from "node:fs/promises" import path from "node:path" @@ -42,10 +43,34 @@ const defaultDetect = async (): Promise => { return undefined } -const probeExecutable = (candidate: ContainerExecutable): Promise => +// Never search the caller's PATH while the side holds a provider credential. +// These are administrator-controlled installation locations. +const containerPath = async (candidate: ContainerExecutable): Promise => { + for (const directory of ["/usr/bin", "/usr/local/bin", "/bin"]) { + const executable = path.join(directory, candidate) + try { + await fs.access(executable, 1) + return executable + } catch { + // Try the next trusted installation location. + } + } + return undefined +} + +const probeExecutable = async (candidate: ContainerExecutable): Promise => { + const executable = await containerPath(candidate) + if (!executable) return false + return probePath(executable) +} + +const probePath = (candidate: string): Promise => new Promise((resolve) => { try { - const child = spawn(candidate, ["--version"], { stdio: "ignore" }) + const child = spawn(candidate, ["--version"], { + stdio: "ignore", + env: buildWorktreeEnvironment(process.env), + }) const timer = setTimeout(() => { child.kill("SIGKILL") resolve(false) @@ -81,6 +106,7 @@ export const exec = ( try { const child = spawn(file, [...args], { stdio: ["ignore", "pipe", "pipe"], + env: buildWorktreeEnvironment(process.env), timeout: timeoutMs, killSignal: "SIGKILL", }) @@ -108,23 +134,27 @@ const IMAGE_INSPECT_TIMEOUT_MS = 30_000 const containerGone = (stderr: string): boolean => /no such container/i.test(stderr) export function createDefaultContainerCli(executable: ContainerExecutable): ContainerCli { + const resolved = containerPath(executable) + const invoke = async (args: readonly string[], timeoutMs: number) => { + const file = await resolved + return file ? exec(file, args, timeoutMs) : { code: "error" as const, stdout: "", stderr: "" } + } return { async pull(image) { - const outcome = await exec(executable, ["pull", "-q", image], IMAGE_PULL_TIMEOUT_MS) + const outcome = await invoke(["pull", "-q", image], IMAGE_PULL_TIMEOUT_MS) return outcome.code === 0 ? "ok" : "failed" }, run(args, timeoutMs) { - return exec(executable, args, timeoutMs).then((outcome) => outcome.code) + return invoke(args, timeoutMs).then((outcome) => outcome.code) }, async remove(name) { - const outcome = await exec(executable, ["rm", "--force", name], CONTAINER_REMOVE_TIMEOUT_MS) + const outcome = await invoke(["rm", "--force", name], CONTAINER_REMOVE_TIMEOUT_MS) if (outcome.code === 0) return "removed" if (outcome.code !== "timeout" && containerGone(outcome.stderr)) return "absent" return "failed" }, async imageDigest(image) { - const repoDigest = await exec( - executable, + const repoDigest = await invoke( ["image", "inspect", "--format", "{{index .RepoDigests 0}}", image], IMAGE_INSPECT_TIMEOUT_MS, ) @@ -134,11 +164,7 @@ export function createDefaultContainerCli(executable: ContainerExecutable): Cont } // A locally built image has no repo digest. The engine image ID is // still an immutable local identity. - const imageId = await exec( - executable, - ["image", "inspect", "--format", "{{.Id}}", image], - IMAGE_INSPECT_TIMEOUT_MS, - ) + const imageId = await invoke(["image", "inspect", "--format", "{{.Id}}", image], IMAGE_INSPECT_TIMEOUT_MS) return imageId.code === 0 ? imageId.stdout.trim() || undefined : undefined }, } diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index f69c6106b2..d1c3bb92d4 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -1,3 +1,4 @@ +import { buildWorktreeEnvironment } from "./coding-trust.ts" import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" @@ -31,7 +32,11 @@ export interface HostRuntimeOptions { const readCommit = async (root: string): Promise => { try { - const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { + cwd: root, + timeout: 5_000, + env: buildWorktreeEnvironment(process.env), + }) const sha = stdout.trim() return /^[0-9a-f]{40}$/i.test(sha) ? { available: true, value: sha } From 3ce847dadc98850191d51dbc764855ae4a6ad071 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 5 Nov 2026 00:53:03 +0000 Subject: [PATCH 17/22] fix(evals): probe container engines and preserve Docker endpoint settings --- scripts/agent-evals/coding-trust.test.ts | 27 ++++++++++ scripts/agent-evals/coding-trust.ts | 21 +++++++- scripts/agent-evals/container-runtime.test.ts | 51 +++++++++++++++++++ scripts/agent-evals/container-runtime.ts | 16 +++--- 4 files changed, 107 insertions(+), 8 deletions(-) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index bc1e22fc73..3bb7a2cbbb 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -10,6 +10,8 @@ import { assertCredentialEnvAllowed, assertEvaluatedTreeClean, buildSideEnvironment, + buildContainerCliEnvironment, + CONTAINER_ENDPOINT_ALLOWLIST, buildWorktreeEnvironment, DirtyEvaluatedTreeError, loadTrustPolicy, @@ -451,3 +453,28 @@ for (const kind of ["symlink", "filter"] as const) { } }) } + +describe("container endpoint environment", () => { + const endpoints = Object.fromEntries(CONTAINER_ENDPOINT_ALLOWLIST.map((name) => [name, `value-${name}`])) + it("preserves endpoints through the side and CLI while stripping provider and unrelated secrets", () => { + const side = buildSideEnvironment( + { + ...endpoints, + HOME: "/home/operator", + PATH: "/usr/bin", + SOLHEIM_API_KEY: "provider-secret", + OTHER_SECRET: "unrelated-secret", + }, + "SOLHEIM_API_KEY", + "baseline", + ) + assert.equal(side["SOLHEIM_API_KEY"], "provider-secret") + const expected = { ...endpoints, HOME: "/home/operator", PATH: "/usr/bin" } + assert.deepEqual(buildContainerCliEnvironment(side), expected) + for (const [name, value] of Object.entries(endpoints)) assert.equal(side[name], value) + assert.equal(side["OTHER_SECRET"], undefined) + }) + it("does not invent missing endpoint settings or a home", () => { + assert.deepEqual(buildContainerCliEnvironment({ PATH: "/usr/bin" }), { PATH: "/usr/bin" }) + }) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 111087fe22..3f8d4db93f 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -168,6 +168,25 @@ const PROCESS_ENV_ALLOWLIST = [ "PATHEXT", ] as const +// Container client endpoint and TLS configuration. Provider credentials are +// handled separately and never forwarded to the container CLI. +export const CONTAINER_ENDPOINT_ALLOWLIST = [ + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "DOCKER_CERT_PATH", + "DOCKER_TLS_VERIFY", +] as const + +export function buildContainerCliEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const child = buildWorktreeEnvironment(env) + for (const name of [...CONTAINER_ENDPOINT_ALLOWLIST, "HOME"] as const) { + const value = env[name] + if (value !== undefined) child[name] = value + } + return child +} + // Environment for the side process: the shared allowlist plus the one named // credential variable and the side label. export function buildSideEnvironment( @@ -176,7 +195,7 @@ export function buildSideEnvironment( side: "baseline" | "candidate", ): NodeJS.ProcessEnv { const child: NodeJS.ProcessEnv = {} - for (const name of PROCESS_ENV_ALLOWLIST) { + for (const name of [...PROCESS_ENV_ALLOWLIST, ...CONTAINER_ENDPOINT_ALLOWLIST]) { const value = env[name] if (value !== undefined) child[name] = value } diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index ae3da13ed6..dae0e0dedd 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -7,6 +7,8 @@ import path from "node:path" import { describe, it } from "node:test" import { buildContainerArgs, + defaultDetect, + probePath, createContainerGradingRuntime, EXEC_OUTPUT_MAX_BYTES, exec, @@ -470,3 +472,52 @@ it("refuses to run when inspection remains unavailable after a successful pull", assert.deepEqual(cli.runArgs, []) assert.deepEqual(await runtime.imageDigest!(), { available: false, reason: "digestUnavailable" }) }) + +describe("container engine detection", () => { + for (const available of ["docker", "podman", undefined] as const) { + it(`selects ${available ?? "no runtime"} using engine availability`, async () => { + const checked: string[] = [] + assert.equal( + await defaultDetect(async (candidate) => { + checked.push(candidate) + return candidate === available + }), + available, + ) + assert.deepEqual(checked, available === "docker" ? ["docker"] : ["docker", "podman"]) + }) + } + it("probes info with the same endpoint environment as CLI execution", async () => { + const temp = await fs.mkdtemp(path.join(os.tmpdir(), "container-engine-probe-")) + try { + const file = path.join(temp, "engine") + await fs.writeFile( + file, + `#!/bin/sh +[ "$1" = "info" ] && [ "$DOCKER_HOST" = "unix:///custom.sock" ] && [ "$HOME" = "/home/operator" ] && [ -z "$SOLHEIM_API_KEY" ] && [ -z "$OTHER_SECRET" ] +`, + { mode: 0o700 }, + ) + const moduleUrl = new URL("./container-runtime.ts", import.meta.url).href + const script = ` + const { probePath, exec } = await import(${JSON.stringify(moduleUrl)}); + if (!await probePath(${JSON.stringify(file)})) throw new Error("engine probe failed"); + if ((await exec(${JSON.stringify(file)}, ["info"], 5000)).code !== 0) throw new Error("CLI environment failed"); + ` + await promisify(execFile)(process.execPath, ["--import", "tsx", "--input-type=module", "-e", script], { + env: { + ...process.env, + HOME: "/home/operator", + DOCKER_HOST: "unix:///custom.sock", + SOLHEIM_API_KEY: "secret", + OTHER_SECRET: "secret", + }, + timeout: 15_000, + }) + await fs.writeFile(file, '#!/bin/sh\n[ "$1" = "--version" ]\n', { mode: 0o700 }) + assert.equal(await probePath(file), false) + } finally { + await fs.rm(temp, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 6b228fd484..6ba564e9c1 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process" -import { buildWorktreeEnvironment } from "./coding-trust.ts" +import { buildContainerCliEnvironment } from "./coding-trust.ts" import { randomUUID } from "node:crypto" import fs from "node:fs/promises" import path from "node:path" @@ -35,9 +35,11 @@ export interface ContainerCli { imageDigest(image: string): Promise } -const defaultDetect = async (): Promise => { +export const defaultDetect = async ( + probe: (candidate: ContainerExecutable) => Promise = probeExecutable, +): Promise => { for (const candidate of ["docker", "podman"] as const) { - const usable = await probeExecutable(candidate) + const usable = await probe(candidate) if (usable) return candidate } return undefined @@ -64,12 +66,12 @@ const probeExecutable = async (candidate: ContainerExecutable): Promise return probePath(executable) } -const probePath = (candidate: string): Promise => +export const probePath = (candidate: string): Promise => new Promise((resolve) => { try { - const child = spawn(candidate, ["--version"], { + const child = spawn(candidate, ["info"], { stdio: "ignore", - env: buildWorktreeEnvironment(process.env), + env: buildContainerCliEnvironment(process.env), }) const timer = setTimeout(() => { child.kill("SIGKILL") @@ -106,7 +108,7 @@ export const exec = ( try { const child = spawn(file, [...args], { stdio: ["ignore", "pipe", "pipe"], - env: buildWorktreeEnvironment(process.env), + env: buildContainerCliEnvironment(process.env), timeout: timeoutMs, killSignal: "SIGKILL", }) From a17059bd4f7664b7c084442f2a2080ca2f91f934 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 6 Nov 2026 01:13:04 +0000 Subject: [PATCH 18/22] fix(evals): qualify grader image and test output permissions --- scripts/agent-evals/container-runtime.test.ts | 25 +++++++++++++++++++ scripts/agent-evals/container-runtime.ts | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index dae0e0dedd..ea723eb015 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -106,6 +106,31 @@ const recordingCli = ( } describe("container grading runtime", () => { + it("uses the fully qualified grader image for non-interactive lookup and pull", () => { + assert.equal(GRADER_IMAGE, "docker.io/library/node:22-bookworm") + }) + + it("makes the output directory writable before the container writes its result", async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-output-mode-")) + try { + const outDir = path.join(scratch, "grading-out") + await fs.mkdir(outDir, { mode: 0o700 }) + const cli = recordingCli({ + run: async (args) => { + const output = outDirFromArgs(args) + assert.equal((await fs.stat(output)).mode & 0o777, 0o777) + await fs.writeFile(path.join(output, "result.json"), driverCompletedJson()) + return 0 + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run({ ...makeJob(), gradingDir: path.join(scratch, "grading-copy") }) + assert.equal(verdict.kind, "passed") + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + it("builds a locked-down, network-less invocation", () => { const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out") const flagValue = (flag: string): string | undefined => { diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 6ba564e9c1..23d2da48e5 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -8,7 +8,7 @@ import type { Measured } from "./trace.ts" // Grader image tag used only for lookup and pull. Containers launch by the // resolved immutable identity, which is also recorded in the comparison pins. -export const GRADER_IMAGE = "node:22-bookworm" +export const GRADER_IMAGE = "docker.io/library/node:22-bookworm" export type ContainerExecutable = "docker" | "podman" From 42788a0dc98c4c08317ac299a5cf875759c6e6c4 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 6 Nov 2026 23:21:30 +0000 Subject: [PATCH 19/22] fix(evals): address review feedback and failing CI assertions --- .../agent-evals/coding-compare-driver.test.ts | 102 ++++++++++++++ scripts/agent-evals/coding-compare.ts | 54 ++++---- scripts/agent-evals/coding-fixtures.test.ts | 2 +- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/download-vscode.mjs | 6 + scripts/agent-evals/grader-driver.mjs | 3 +- scripts/agent-evals/grader-driver.test.ts | 128 +++++++++++++++++- scripts/agent-evals/launcher.test.ts | 54 +++++++- scripts/agent-evals/launcher.ts | 21 ++- 9 files changed, 338 insertions(+), 34 deletions(-) create mode 100644 scripts/agent-evals/download-vscode.mjs diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts index cc9700a384..6d8bbee908 100644 --- a/scripts/agent-evals/coding-compare-driver.test.ts +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -368,3 +368,105 @@ it("removes a published generation when interrupted during final cleanup", async await fs.rm(dir, { recursive: true, force: true }) } }) + +it("invalidates previous reports before a side starts and removes reports on failure", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-stale-")) + const out = path.join(dir, "out") + const names = ["comparison-report.complete", "comparison-report.json", "comparison-report.md"] + try { + await fs.mkdir(out) + await Promise.all(names.map((name) => fs.writeFile(path.join(out, name), "stale"))) + await assert.rejects( + executeComparisonFlow( + flowWiring({ + flag: createInterruptFlag(), + out, + scratchDir: path.join(dir, "scratch"), + runSide: async () => { + assert.deepEqual(await fs.readdir(out), []) + await fs.writeFile(path.join(out, names[0]!), "partial") + throw new Error("side failed") + }, + }), + ), + /side failed/, + ) + assert.deepEqual(await fs.readdir(out), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +it( + "waits for timeout termination without sending a delayed kill after side cleanup", + { timeout: 10_000 }, + async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-timeout-")) + const originalKill = process.kill + const kills: NodeJS.Signals[] = [] + t.mock.method(process, "kill", (pid: number, signal?: NodeJS.Signals | number) => { + if (signal === "SIGKILL") kills.push(signal) + return originalKill(pid, signal) + }) + try { + const fixture = path.join(dir, "side.cjs") + await fs.writeFile(fixture, "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)") + const tracked: (number | undefined)[] = [] + const wiring = sideWiring({ + flag: createInterruptFlag(), + tracked, + sideScript: fixture, + createSideOutDir: async (out) => { + await fs.mkdir(out, { recursive: true }) + }, + }) + await assert.rejects( + runCompareSide("baseline", COMMIT, dir, { ...wiring, sideTimeoutMs: 500, killGraceMs: 200 }), + /side failed/, + ) + assert.equal(tracked.at(-1), undefined) + await new Promise((resolve) => setTimeout(resolve, 300)) + assert.deepEqual(kills, []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, +) + +it("attempts every report removal and refuses to start sides when invalidation fails", async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-cleanup-")) + const out = path.join(dir, "out") + const remove = fs.rm + const removed: string[] = [] + let started = false + t.mock.method(fs, "rm", async (target: Parameters[0], options: Parameters[1]) => { + if (String(target).startsWith(out + path.sep)) { + removed.push(path.basename(String(target))) + if (String(target).endsWith("comparison-report.complete")) throw new Error("marker removal refused") + } + await remove(target, options) + }) + try { + await fs.mkdir(out) + await fs.writeFile(path.join(out, "comparison-report.complete"), "stale") + await assert.rejects( + executeComparisonFlow( + flowWiring({ + flag: createInterruptFlag(), + out, + scratchDir: path.join(dir, "scratch"), + runSide: async (side) => { + started = true + return record(side, COMMIT) + }, + }), + ), + AggregateError, + ) + assert.equal(started, false) + for (const name of ["comparison-report.complete", "comparison-report.json", "comparison-report.md"]) + assert.ok(removed.includes(name)) + } finally { + await remove(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index c42cdd9e99..77c47448a6 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -8,7 +8,7 @@ import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, typ import { DEFAULT_REPETITIONS } from "./coding-stats.ts" import type { CodingRunRecord } from "./coding-run.ts" import { buildSideEnvironment, buildWorktreeEnvironment } from "./coding-trust.ts" -import type { SideProcessGuard } from "./side-process-guard.ts" +import { createSideProcessGuard, type SideProcessGuard } from "./side-process-guard.ts" import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" export { buildComparisonReport, renderMarkdownReport } @@ -203,31 +203,22 @@ export const runCompareSide = async ( ) wiring.guard.track(child.pid) let timedOut = false + const timeoutGuard = createSideProcessGuard(wiring.killGraceMs) + timeoutGuard.track(child.pid) + let termination: Promise = Promise.resolve() const timer = setTimeout(() => { timedOut = true - try { - if (child.pid) process.kill(-child.pid, "SIGTERM") - } catch { - child.kill("SIGTERM") - } - setTimeout(() => { - try { - if (child.pid) process.kill(-child.pid, "SIGKILL") - } catch { - child.kill("SIGKILL") - } - }, wiring.killGraceMs).unref() + termination = timeoutGuard.interrupt() }, wiring.sideTimeoutMs) - child.once("error", () => { - clearTimeout(timer) - wiring.guard.track(undefined) - resolve(1) - }) - child.once("exit", (code) => { + const finish = (code: number): void => { clearTimeout(timer) - wiring.guard.track(undefined) - resolve(timedOut ? 1 : (code ?? 1)) - }) + void termination.then(() => { + wiring.guard.track(undefined) + resolve(timedOut ? 1 : code) + }) + } + child.once("error", () => finish(1)) + child.once("exit", (code) => finish(code ?? 1)) }) if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) @@ -265,6 +256,7 @@ export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promi let completed: ComparisonReport | undefined let failure: unknown try { + await removeComparisonReport(wiring.out) const report = buildComparisonReport({ baseline: toSideResult(await wiring.runSide("baseline")), candidate: toSideResult(await wiring.runSide("candidate")), @@ -303,7 +295,14 @@ export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promi await removeComparisonReport(wiring.out) return { status: "interrupted", signal } } - if (failure !== undefined) throw failure + if (failure !== undefined) { + try { + await removeComparisonReport(wiring.out) + } catch (cleanupError) { + throw new AggregateError([failure, cleanupError], "comparison failed and report cleanup failed") + } + throw failure + } if (completed === undefined) throw new Error("comparison ended without a report or a failure") return { status: "completed", report: completed } } @@ -396,8 +395,13 @@ const COMPARISON_REPORT_COMPLETE = "comparison-report.complete" // comparison leaves no report files behind. Consumers must require the // completion marker and match its generation against the report files. export async function removeComparisonReport(outDir: string): Promise { - for (const name of [COMPARISON_REPORT_COMPLETE, COMPARISON_REPORT_JSON, COMPARISON_REPORT_MD]) - await fs.rm(path.join(outDir, name), { force: true }) + const results = await Promise.allSettled( + [COMPARISON_REPORT_COMPLETE, COMPARISON_REPORT_JSON, COMPARISON_REPORT_MD].map((name) => + fs.rm(path.join(outDir, name), { force: true }), + ), + ) + const errors = results.filter((result) => result.status === "rejected").map((result) => result.reason) + if (errors.length) throw new AggregateError(errors, "comparison report cleanup failed") } export async function publishComparisonReport( diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index cf9b207b33..d420728a15 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -65,7 +65,7 @@ describe("coding fixture suite", () => { assert.ok(bundle.grading.resources.memoryMb > 0) assert.ok(bundle.grading.resources.cpus > 0) assert.ok(bundle.grading.resources.pidsLimit > 0) - assert.match(bundle.grading.resources.image, /^node:/) + assert.equal(bundle.grading.resources.image, "docker.io/library/node:22-bookworm") } }) diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index 54d8e6799a..be474acd2a 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -67,6 +67,6 @@ describe("coding graders against reference solutions", () => { it("pins one grader version across the suite", () => { assert.equal(GRADER_VERSION, "coding-grader-v5") - assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) + assert.equal(CODING_GRADER_RESOURCES.image, "docker.io/library/node:22-bookworm") }) }) diff --git a/scripts/agent-evals/download-vscode.mjs b/scripts/agent-evals/download-vscode.mjs new file mode 100644 index 0000000000..3ffeb0ef6d --- /dev/null +++ b/scripts/agent-evals/download-vscode.mjs @@ -0,0 +1,6 @@ +// Download/extraction helpers must never inherit the provider credential. +import { downloadAndUnzipVSCode, SilentReporter } from "@vscode/test-electron" + +const [version, cachePath] = process.argv.slice(2) +const executable = await downloadAndUnzipVSCode({ version, cachePath, reporter: new SilentReporter() }) +process.stdout.write(JSON.stringify(executable)) diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs index f4709f861a..85ca5d82fc 100644 --- a/scripts/agent-evals/grader-driver.mjs +++ b/scripts/agent-evals/grader-driver.mjs @@ -136,7 +136,8 @@ try { failedCount += counts.failed cancelledCount += counts.cancelled ?? 0 if (counts.failed > 0 && failures.length < 50) - for (const name of readFailureNames(outcome.tap)) failures.push({ name, file }) + for (const name of readFailureNames(outcome.tap).slice(0, 50 - failures.length)) + failures.push({ name, file }) } if (runnerBroke) { writeResult({ kind: "driver-error", message: "testRunnerUnavailable" }) diff --git a/scripts/agent-evals/grader-driver.test.ts b/scripts/agent-evals/grader-driver.test.ts index 835d5bd81f..d7d374e99c 100644 --- a/scripts/agent-evals/grader-driver.test.ts +++ b/scripts/agent-evals/grader-driver.test.ts @@ -9,14 +9,18 @@ import { driverFilePath } from "./coding-grader.ts" // Runs the real driver against a real workspace with the real filesystem. // The driver is the grading publication boundary, so these checks exercise // the actual temp-write-and-rename behavior, not a stub. -const runDriver = async (args: readonly string[], cwd: string): Promise => { +const runDriver = async (args: readonly string[], cwd: string, nodeArgs: readonly string[] = []): Promise => { // This test file itself runs under the node:test runner, whose // NODE_TEST_CONTEXT would leak into the driver and then into its inner // `node --test` children, switching them into runner-child mode with no // TAP summary. The driver gets an environment without that variable. const { NODE_TEST_CONTEXT: _runnerContext, ...driverEnv } = process.env return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [driverFilePath(), ...args], { cwd, env: driverEnv, stdio: "ignore" }) + const child = spawn(process.execPath, [...nodeArgs, driverFilePath(), ...args], { + cwd, + env: driverEnv, + stdio: "ignore", + }) child.once("error", reject) child.once("close", (code) => resolve(code ?? -1)) }) @@ -78,3 +82,123 @@ describe("grader driver result publication", () => { } }) }) + +for (const scenario of [ + "missing", + "empty", + "unsupported", + "passing", + "failing", + "cancelled", + "discovery-error", + "failure-cap", +] as const) { + it(`reports every grading field for ${scenario}`, { timeout: 60_000 }, async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "grader-fields-")) + try { + const tests = path.join(scratch, "test") + const out = path.join(scratch, "result.json") + if (scenario !== "missing") await fs.mkdir(tests) + if (scenario === "unsupported") await fs.writeFile(path.join(tests, "ignored.test.txt"), "ignored") + if (scenario === "discovery-error") { + await fs.rm(tests, { recursive: true }) + await fs.writeFile(tests, "not a directory") + } + const source = + scenario === "passing" + ? passingTest + : scenario === "failing" + ? "import { test } from 'node:test'; test('wrong', () => { throw new Error('wrong') })" + : scenario === "cancelled" + ? "import { test } from 'node:test'; test('pending', () => new Promise(() => {}))" + : scenario === "failure-cap" + ? "import { test } from 'node:test'; for (let i = 0; i < 30; i++) test('wrong-' + i, () => { throw new Error('wrong') })" + : undefined + if (source) await fs.writeFile(path.join(tests, "first.test.mjs"), source) + if (scenario === "failure-cap") await fs.writeFile(path.join(tests, "second.test.mjs"), source!) + assert.equal(await runDriver([out, "test"], scratch), 0) + const result = JSON.parse(await fs.readFile(out, "utf8")) as Record + if (scenario === "discovery-error") { + assert.deepEqual(result, { + kind: "driver-error", + message: "Error", + }) + return + } + const empty = ["missing", "empty", "unsupported"].includes(scenario) + const files = empty + ? [] + : [ + path.join("test", "first.test.mjs"), + ...(scenario === "failure-cap" ? [path.join("test", "second.test.mjs")] : []), + ] + const failures = + scenario === "failing" + ? [{ name: "wrong", file: files[0] }] + : scenario === "failure-cap" + ? Array.from({ length: 50 }, (_, i) => ({ + name: `wrong-${i % 30}`, + file: files[i < 30 ? 0 : 1], + })) + : [] + assert.equal(typeof result.durationMs, "number") + assert.ok(Number(result.durationMs) >= 0) + assert.deepEqual( + { ...result, durationMs: 0 }, + { + kind: "completed", + passed: scenario === "passing", + noTestsDiscovered: empty, + testFiles: files, + passedCount: scenario === "passing" ? 1 : 0, + failedCount: scenario === "failing" ? 1 : scenario === "failure-cap" ? 60 : 0, + cancelledCount: scenario === "cancelled" ? 1 : 0, + failures, + durationMs: 0, + }, + ) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +} + +it("rejects a missing result path", async () => { + assert.equal(await runDriver([], os.tmpdir()), 2) +}) + +for (const scenario of ["child-error", "missing-summary"] as const) { + it(`reports a driver failure for ${scenario}`, async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "grader-runner-error-")) + try { + const workspace = await writeWorkspace(scratch) + const preload = path.join(scratch, "runner.cjs") + await fs.writeFile( + preload, + ` +const cp = require('node:child_process'); +const { EventEmitter } = require('node:events'); +cp.spawn = () => { + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + setImmediate(() => { + ${scenario === "child-error" ? "child.emit('error', new Error('spawn refused'));" : "child.stdout.emit('data', Buffer.from('TAP version 13\\n'));"} + child.emit('close', 0); + }); + return child; +}; +require('node:module').syncBuiltinESMExports(); +`, + ) + const out = path.join(scratch, "result.json") + assert.equal(await runDriver([out, "test"], workspace, ["--require", preload]), 0) + assert.deepEqual(JSON.parse(await fs.readFile(out, "utf8")), { + kind: "driver-error", + message: "testRunnerUnavailable", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts index 8218646bd3..73f16af153 100644 --- a/scripts/agent-evals/launcher.test.ts +++ b/scripts/agent-evals/launcher.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { describe, it } from "node:test" -import { fileURLToPath } from "node:url" +import { fileURLToPath, pathToFileURL } from "node:url" import { createHostRuntime, readWorkspaceFiles } from "./launcher.ts" const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -188,3 +188,55 @@ it("ignores poisoned caller VS Code caches and removes its fresh private cache", await fs.rm(dir, { recursive: true, force: true }) } }) + +it("keeps provider credentials out of downloader extraction helpers", async () => { + const dir = await tempDir("download-env-") + const savedPath = process.env.PATH + const savedKey = process.env.SOLHEIM_API_KEY + try { + const helper = path.join(dir, "tar") + await fs.writeFile(helper, '#!/bin/sh\nif [ -n "$SOLHEIM_API_KEY" ]; then exit 9; fi\nprintf clean\n', { + mode: 0o700, + }) + const downloader = path.join(dir, "download.mjs") + await fs.writeFile( + downloader, + `import { execFileSync } from 'node:child_process'; +const result = execFileSync('tar', [], { encoding: 'utf8' }); +if (result !== 'clean') throw new Error('extraction environment contaminated'); +process.stdout.write(JSON.stringify(process.execPath));`, + ) + process.env.PATH = `${dir}${path.delimiter}${savedPath ?? ""}` + process.env.SOLHEIM_API_KEY = "test-provider-secret" + const { downloadVSCodeIsolated } = await import("./launcher.ts") + assert.equal( + await downloadVSCodeIsolated({ version: "1.100.0", cachePath: dir }, pathToFileURL(downloader)), + process.execPath, + ) + } finally { + if (savedPath === undefined) delete process.env.PATH + else process.env.PATH = savedPath + if (savedKey === undefined) delete process.env.SOLHEIM_API_KEY + else process.env.SOLHEIM_API_KEY = savedKey + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +it("captures workspace files through the host runtime after launch", async () => { + const dir = await tempDir("host-workspace-") + const runtime = createHostRuntime({ + root: repoRoot, + artifactsDir: dir, + downloadVSCode: async () => process.execPath, + }) + let host: Awaited> | undefined + try { + const files = [{ path: "nested/answer.js", content: "export const answer = 42\n" }] + host = await runtime.launch(files) + assert.deepEqual(await runtime.readFinalWorkspace?.(), { files, truncated: false }) + } finally { + host?.signal("SIGKILL") + await runtime.cleanup() + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index d1c3bb92d4..fc855b1045 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -4,7 +4,7 @@ import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { promisify } from "node:util" -import { downloadAndUnzipVSCode } from "@vscode/test-electron" +import { fileURLToPath } from "node:url" import { IpcClient } from "@roo-code/ipc" import { IpcMessageType } from "@roo-code/types" import { buildChildEnvironment, buildLaunchArgs } from "./isolation.ts" @@ -14,13 +14,28 @@ import { identityUnavailable, measured, type MeasuredString, type TraceRunIdenti const execFileAsync = promisify(execFile) +export const downloadVSCodeIsolated = async ( + options: { version?: string; cachePath?: string }, + downloader = new URL("./download-vscode.mjs", import.meta.url), +): Promise => { + if (typeof options !== "object" || options === null) throw new Error("download options required") + const { stdout } = await execFileAsync( + process.execPath, + [fileURLToPath(downloader), options.version ?? "stable", options.cachePath ?? ""], + { env: buildWorktreeEnvironment(process.env), timeout: 300_000 }, + ) + const executable: unknown = JSON.parse(stdout) + if (typeof executable !== "string" || !path.isAbsolute(executable)) throw new Error("invalid VS Code executable") + return executable +} + export interface HostRuntimeOptions { // Repository root of the evaluated extension. Reads of package metadata, // commit identity, and the extension source all stay inside it. root: string artifactsDir: string // Test seam for the download boundary; caches always live in private scratch. - downloadVSCode?: typeof downloadAndUnzipVSCode + downloadVSCode?: (options: { version: string; cachePath: string }) => Promise now?: () => number pause?: (ms: number) => Promise // Optional --extensionTestsPath module. The module's run() must never @@ -140,7 +155,7 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { devDependencies: Record } - const executable = await (options.downloadVSCode ?? downloadAndUnzipVSCode)({ + const executable = await (options.downloadVSCode ?? downloadVSCodeIsolated)({ version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", cachePath: path.join(temp, "vscode-cache"), }) From 3c264014be585ca40aa0f292891c4ad6ea79308d Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 7 Nov 2026 00:21:06 +0000 Subject: [PATCH 20/22] fix(evals): reuse verified private VS Code cache and preserve proxy settings --- scripts/agent-evals/README.md | 2 +- scripts/agent-evals/coding-side.mts | 3 ++ scripts/agent-evals/coding-trust.test.ts | 19 ++++++++ scripts/agent-evals/coding-trust.ts | 14 +++++- scripts/agent-evals/download-vscode.mjs | 7 ++- scripts/agent-evals/launcher.test.ts | 20 +++++++++ scripts/agent-evals/launcher.ts | 10 +++-- scripts/agent-evals/vscode-cache.test.ts | 31 +++++++++++++ scripts/agent-evals/vscode-cache.ts | 57 ++++++++++++++++++++++++ 9 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 scripts/agent-evals/vscode-cache.test.ts create mode 100644 scripts/agent-evals/vscode-cache.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index b4ece269cc..eb1f9cc3cb 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -158,7 +158,7 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. The driver creates each side's fresh detached worktree of the approved commit. Each side then constructs its own fresh detached worktree of that same commit, installs the frozen lockfile dependencies and builds inside it, and launches every host against it, so each side runs only its own compiled extension of the approved commit. The install and the build consume the side's own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The launcher downloads VS Code into fresh private per-run scratch space and removes the cache during cleanup. It never executes a cached VS Code binary from the caller checkout. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children receive an allowlisted environment and a temporary HOME with isolated XDG and npm configuration. Parent package-manager configuration and registry URLs are not forwarded. The prepared tree is checked against the raw bytes in the approved commit, and tracked symlinks are rejected. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The side downloads VS Code once into fresh private scratch space and records a SHA-256 digest of the cache contents. Before each repetition it verifies the cache, copies it into private per-run scratch, and verifies the copy before launch. Evaluated hosts never receive the reusable cache path. Side and run cleanup remove their caches. It never executes a cached VS Code binary from the caller checkout. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. Downloader children receive only the process allowlist plus `npm_config_proxy`, `npm_config_https_proxy`, and `NODE_EXTRA_CA_CERTS`. Proxy credentials stay out of evaluated build, host, and grader environments and reports. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children receive an allowlisted environment and a temporary HOME with isolated XDG and npm configuration. Parent package-manager configuration and registry URLs are not forwarded. The prepared tree is checked against the raw bytes in the approved commit, and tracked symlinks are rejected. Every driver child, the side included, runs detached in its own process group and is tracked by one guard. The driver's guard tracks the side; the side's guard tracks its own worktree, install, and build. A SIGINT or SIGTERM that reaches only the driver terminates the side group through a bounded SIGTERM-then-SIGKILL escalation. The side starts the same escalation for its preparation children and waits for it before it removes its worktree or exits. The driver's worktree cleanup also waits for the driver's escalation before it removes any tree. diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index ecc059c78e..d9b32eaa9c 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -19,6 +19,7 @@ import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" import { createHostRuntime } from "./launcher.ts" +import { createPrivateVSCodeDownloader } from "./vscode-cache.ts" import { createContainerGradingRuntime } from "./container-runtime.ts" import { runCodingFixtureOnce, type CodingRunRecord } from "./coding-run.ts" import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" @@ -166,6 +167,7 @@ await invalidateSideArtifacts(out) // executed. const sideScratch = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-side-")) const evaluatedWorktree = path.join(sideScratch, "worktree") +const downloadVSCode = createPrivateVSCodeDownloader(path.join(sideScratch, "vscode-cache")) // Removal is best effort and bounded, like the comparison flow cleanup. The // escalation of an in-flight preparation child settles before any tree is @@ -273,6 +275,7 @@ try { runtime: createHostRuntime({ root: evaluatedWorktree, artifactsDir, + downloadVSCode, }), gradingRuntime, scratchDir: artifactsDir, diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index 3bb7a2cbbb..367a99f1aa 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -478,3 +478,22 @@ describe("container endpoint environment", () => { assert.deepEqual(buildContainerCliEnvironment({ PATH: "/usr/bin" }), { PATH: "/usr/bin" }) }) }) + +it("preserves downloader network configuration across both boundaries without forwarding provider secrets", async () => { + const { buildDownloaderEnvironment } = await import("./coding-trust.ts") + const network = { + npm_config_proxy: "http://user:proxy-password@localhost:8080", + npm_config_https_proxy: "http://user:proxy-password@localhost:8443", + NODE_EXTRA_CA_CERTS: "/private/ca.pem", + } + const env = { ...network, SOLHEIM_API_KEY: "provider-secret", UNRELATED_SECRET: "unrelated" } + const side = buildSideEnvironment(env, "SOLHEIM_API_KEY", "candidate") + const child = buildDownloaderEnvironment(side) + for (const name of Object.keys(network)) { + assert.equal(buildWorktreeEnvironment(side)[name], undefined) + assert.equal(buildContainerCliEnvironment(side)[name], undefined) + } + assert.equal(child.SOLHEIM_API_KEY, undefined) + assert.equal(child.UNRELATED_SECRET, undefined) + assert.deepEqual(child, network) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 3f8d4db93f..02f128adae 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -168,6 +168,18 @@ const PROCESS_ENV_ALLOWLIST = [ "PATHEXT", ] as const +// Proxy URLs can contain credentials. Forward them only to the side and +// downloader, never to evaluated builds, hosts, reports, or graders. +const DOWNLOAD_NETWORK_ALLOWLIST = ["npm_config_proxy", "npm_config_https_proxy", "NODE_EXTRA_CA_CERTS"] as const + +export function buildDownloaderEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const child = buildWorktreeEnvironment(env) + for (const name of DOWNLOAD_NETWORK_ALLOWLIST) { + if (env[name] !== undefined) child[name] = env[name] + } + return child +} + // Container client endpoint and TLS configuration. Provider credentials are // handled separately and never forwarded to the container CLI. export const CONTAINER_ENDPOINT_ALLOWLIST = [ @@ -195,7 +207,7 @@ export function buildSideEnvironment( side: "baseline" | "candidate", ): NodeJS.ProcessEnv { const child: NodeJS.ProcessEnv = {} - for (const name of [...PROCESS_ENV_ALLOWLIST, ...CONTAINER_ENDPOINT_ALLOWLIST]) { + for (const name of [...PROCESS_ENV_ALLOWLIST, ...CONTAINER_ENDPOINT_ALLOWLIST, ...DOWNLOAD_NETWORK_ALLOWLIST]) { const value = env[name] if (value !== undefined) child[name] = value } diff --git a/scripts/agent-evals/download-vscode.mjs b/scripts/agent-evals/download-vscode.mjs index 3ffeb0ef6d..24d42c425a 100644 --- a/scripts/agent-evals/download-vscode.mjs +++ b/scripts/agent-evals/download-vscode.mjs @@ -1,6 +1,11 @@ // Download/extraction helpers must never inherit the provider credential. +import process from "node:process" import { downloadAndUnzipVSCode, SilentReporter } from "@vscode/test-electron" const [version, cachePath] = process.argv.slice(2) -const executable = await downloadAndUnzipVSCode({ version, cachePath, reporter: new SilentReporter() }) +const executable = await downloadAndUnzipVSCode({ + version, + cachePath: cachePath || undefined, + reporter: new SilentReporter(), +}) process.stdout.write(JSON.stringify(executable)) diff --git a/scripts/agent-evals/launcher.test.ts b/scripts/agent-evals/launcher.test.ts index 73f16af153..a41a746246 100644 --- a/scripts/agent-evals/launcher.test.ts +++ b/scripts/agent-evals/launcher.test.ts @@ -240,3 +240,23 @@ it("captures workspace files through the host runtime after launch", async () => await fs.rm(dir, { recursive: true, force: true }) } }) + +it("does not expose authenticated proxy URLs from downloader failures", async () => { + const dir = await tempDir("download-failure-") + try { + const downloader = path.join(dir, "download.mjs") + await fs.writeFile(downloader, "throw new Error('http://user:proxy-secret@localhost:8080')") + const { downloadVSCodeIsolated } = await import("./launcher.ts") + await assert.rejects( + downloadVSCodeIsolated({ cachePath: dir }, pathToFileURL(downloader)), + (error: unknown) => { + assert.ok(error instanceof Error) + assert.equal(error.message, "VS Code download failed") + assert.equal(error.cause, undefined) + return true + }, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index fc855b1045..7d5a4bfee0 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -1,4 +1,4 @@ -import { buildWorktreeEnvironment } from "./coding-trust.ts" +import { buildDownloaderEnvironment, buildWorktreeEnvironment } from "./coding-trust.ts" import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" @@ -22,8 +22,12 @@ export const downloadVSCodeIsolated = async ( const { stdout } = await execFileAsync( process.execPath, [fileURLToPath(downloader), options.version ?? "stable", options.cachePath ?? ""], - { env: buildWorktreeEnvironment(process.env), timeout: 300_000 }, - ) + { env: buildDownloaderEnvironment(process.env), timeout: 300_000 }, + ).catch(() => { + // Download errors can include authenticated proxy URLs in stderr. + // Do not propagate subprocess output into repetition diagnostics. + throw new Error("VS Code download failed") + }) const executable: unknown = JSON.parse(stdout) if (typeof executable !== "string" || !path.isAbsolute(executable)) throw new Error("invalid VS Code executable") return executable diff --git a/scripts/agent-evals/vscode-cache.test.ts b/scripts/agent-evals/vscode-cache.test.ts new file mode 100644 index 0000000000..d331fd0fef --- /dev/null +++ b/scripts/agent-evals/vscode-cache.test.ts @@ -0,0 +1,31 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { it } from "node:test" +import { createPrivateVSCodeDownloader } from "./vscode-cache.ts" + +it("downloads once, isolates run mutations, and rejects a poisoned reusable cache", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-cache-test-")) + let calls = 0 + let source = "" + const download = createPrivateVSCodeDownloader(path.join(dir, "private"), async ({ cachePath }) => { + calls++ + source = cachePath + const executable = path.join(cachePath, "code") + await fs.writeFile(executable, "trusted", { mode: 0o700 }) + return executable + }) + try { + const first = await download({ version: "test", cachePath: path.join(dir, "run1") }) + await fs.writeFile(first, "poisoned by evaluated run") + const second = await download({ version: "test", cachePath: path.join(dir, "run2") }) + assert.equal(await fs.readFile(second, "utf8"), "trusted") + assert.equal(calls, 1) + await fs.writeFile(path.join(source, "code"), "poisoned source") + await assert.rejects(download({ version: "test", cachePath: path.join(dir, "run3") }), /integrity mismatch/) + await assert.rejects(fs.access(path.join(dir, "run3"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/vscode-cache.ts b/scripts/agent-evals/vscode-cache.ts new file mode 100644 index 0000000000..9d2e35fddd --- /dev/null +++ b/scripts/agent-evals/vscode-cache.ts @@ -0,0 +1,57 @@ +// The side owns this cache in fresh private scratch. Evaluated hosts receive +// independent copies, never the reusable download. Verify both source and +// copy against the first download before returning an executable. +import { createHash } from "node:crypto" +import fs from "node:fs/promises" +import path from "node:path" +import { downloadVSCodeIsolated } from "./launcher.ts" + +async function digestTree(root: string): Promise { + const hash = createHash("sha256") + const visit = async (relative: string): Promise => { + const full = path.join(root, relative) + const stat = await fs.lstat(full) + hash.update(JSON.stringify([relative, stat.mode])) + if (stat.isSymbolicLink()) { + const target = await fs.readlink(full) + const resolved = path.resolve(path.dirname(full), target) + if (path.isAbsolute(target) || !resolved.startsWith(root + path.sep)) + throw new Error("VS Code cache symlink escapes cache") + hash.update(JSON.stringify(target)) + } else if (stat.isDirectory()) { + for (const name of (await fs.readdir(full)).sort()) await visit(path.join(relative, name)) + } else if (stat.isFile()) { + hash.update(JSON.stringify(stat.size)) + hash.update(await fs.readFile(full)) + } else throw new Error("unsupported VS Code cache entry") + } + await visit("") + return hash.digest("hex") +} + +export function createPrivateVSCodeDownloader( + cacheRoot: string, + downloader: (options: { version: string; cachePath: string }) => Promise = downloadVSCodeIsolated, +): (options: { version: string; cachePath: string }) => Promise { + const downloads = new Map>() + return async ({ version, cachePath }) => { + let download = downloads.get(version) + if (!download) { + download = (async () => { + await fs.mkdir(cacheRoot, { recursive: true, mode: 0o700 }) + const source = await fs.mkdtemp(path.join(cacheRoot, "download-")) + const executable = await downloader({ version, cachePath: source }) + const relative = path.relative(source, executable) + if (relative.startsWith("..") || path.isAbsolute(relative)) + throw new Error("VS Code executable outside cache") + return { source, relative, digest: await digestTree(source) } + })() + downloads.set(version, download) + } + const cached = await download + if ((await digestTree(cached.source)) !== cached.digest) throw new Error("VS Code cache integrity mismatch") + await fs.cp(cached.source, cachePath, { recursive: true, verbatimSymlinks: true }) + if ((await digestTree(cachePath)) !== cached.digest) throw new Error("VS Code cache copy integrity mismatch") + return path.join(cachePath, cached.relative) + } +} From e0427c6c792562857411b96ac01af66786fa937c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 7 Nov 2026 01:03:57 +0000 Subject: [PATCH 21/22] fix(evals): disable host commands and lock comparison output --- scripts/agent-evals/README.md | 4 + .../agent-evals/coding-compare-driver.test.ts | 63 +++++++++++ scripts/agent-evals/coding-compare.ts | 23 ++++ scripts/agent-evals/coding-fixtures.ts | 25 +++-- scripts/agent-evals/vscode-cache.test.ts | 100 +++++++++++++++++- scripts/agent-evals/vscode-cache.ts | 5 +- .../__tests__/coding-eval-boundary.spec.ts | 26 +++++ 7 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 src/core/tools/__tests__/coding-eval-boundary.spec.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index eb1f9cc3cb..a38b72c463 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -170,6 +170,8 @@ A side run invalidates its output directory first. Before the first repetition e The side process arms one flag when it receives SIGINT or SIGTERM. The flag starts no further repetition and blocks the completion marker, so an interrupted side exits with the conventional signal code and publishes nothing. +Comparisons acquire an exclusive lock for the canonical output directory before invalidation and hold it through publication and all failure or interrupt cleanup. An overlapping comparison (including a symlink alias of `--out`) fails before touching reports. Locks live in the system temporary directory; after a hard crash, remove a stale lock only after verifying no comparison owns that output. + The comparison reports are published as one generation. `comparison-report.json` and `comparison-report.md` carry the report's `generatedAt` stamp, and `comparison-report.complete` holds that stamp; it is written last. Consumers must require the marker and match its stamp against the report files. `readPublishedComparisonReport` enforces this: it returns the report only when the marker exists and its stamp matches both files. The marker is removed before publication starts, so a crash between the two file publications leaves a detectable incomplete generation instead of a mixed one. A thrown failure rolls both files back; a failed comparison leaves no report files. ### Commands @@ -192,3 +194,5 @@ pnpm coding-evals:compare --baseline --candidate --trust-file trust- 1. Release integration: trigger runs, store baselines, and promote them. See issue #2085. 2. Later suites: repository navigation, debugging, and cost stress, toward the 21-task set. 3. Optimization experiments: per-request context instrumentation, non-progress loop detection, tool-output filtering, compaction strategies, and dynamic tool-schema loading. + +Coding fixtures run in a host VS Code process without a filesystem sandbox. Command tools are disabled in both the mode and tool policy, and command auto-approval is off. Agents can read and edit workspace files; test execution happens only in the isolated grader. Fixture prompts do not request host commands. diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts index 6d8bbee908..857da0798f 100644 --- a/scripts/agent-evals/coding-compare-driver.test.ts +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -7,6 +7,7 @@ import { describe, it } from "node:test" import { executeComparisonFlow, publishedComparisonGeneration, + readPublishedComparisonReport, runCompareSide, type CompareSideWiring, type ComparisonFlowWiring, @@ -470,3 +471,65 @@ it("attempts every report removal and refuses to start sides when invalidation f await remove(dir, { recursive: true, force: true }) } }) + +for (const fails of [false, true]) { + it(`excludes overlapping comparisons through ${fails ? "failure" : "successful"} cleanup`, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-overlap-")) + let release: () => void = () => {} + let entered: () => void = () => {} + const gate = new Promise((resolve) => { + release = resolve + }) + const started = new Promise((resolve) => { + entered = resolve + }) + const out = path.join(dir, "out") + const first = flowWiring({ + flag: createInterruptFlag(), + out, + scratchDir: path.join(dir, "first"), + runSide: async (side) => record(side, COMMIT), + }) + first.interruptKill = async () => { + entered() + await gate + } + if (fails) + first.runSide = async () => { + throw new Error("first failed") + } + const running = executeComparisonFlow(first) + const settled = running.then( + () => undefined, + (error: unknown) => error, + ) + try { + await started + const before = await publishedComparisonGeneration(out) + let ran = false + const second = flowWiring({ + flag: createInterruptFlag(), + out, + scratchDir: path.join(dir, "second"), + runSide: async (side) => { + ran = true + return record(side, COMMIT) + }, + }) + await fs.symlink(out, path.join(dir, "alias")) + await assert.rejects(executeComparisonFlow({ ...second, out: path.join(dir, "alias") }), /output lock/) + assert.equal(ran, false) + assert.equal(await publishedComparisonGeneration(out), before) + if (!fails) assert.equal((await readPublishedComparisonReport(out))?.pins.baseline.commit, COMMIT) + release() + const error = await settled + assert.equal(error instanceof Error, fails) + assert.equal((await executeComparisonFlow(second)).status, "completed") + assert.equal((await readPublishedComparisonReport(out))?.pins.baseline.commit, COMMIT) + } finally { + release() + await settled + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 77c47448a6..6887e367e0 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto" import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import path from "node:path" @@ -253,6 +254,28 @@ export type ComparisonFlowOutcome = // caller exits with the conventional code on the normal and the throwing // path alike; any other failure stays a thrown failure. export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promise => { + await fs.mkdir(wiring.out, { recursive: true }) + const canonicalOut = await fs.realpath(wiring.out) + // A canonical path also excludes aliases through symlinks. Keep the lock + // outside scratch/output so cleanup cannot remove it before ownership ends. + const key = createHash("sha256").update(canonicalOut).digest("hex") + const lock = path.join(os.tmpdir(), `coding-comparison-${key}.lock`) + try { + await fs.mkdir(lock, { mode: 0o700 }) + } catch (error) { + throw new Error( + `cannot acquire comparison output lock for ${canonicalOut}; another comparison may be running`, + { cause: error }, + ) + } + try { + return await executeLockedComparisonFlow(wiring) + } finally { + await fs.rmdir(lock) + } +} + +const executeLockedComparisonFlow = async (wiring: ComparisonFlowWiring): Promise => { let completed: ComparisonReport | undefined let failure: unknown try { diff --git a/scripts/agent-evals/coding-fixtures.ts b/scripts/agent-evals/coding-fixtures.ts index f5990e9816..09ec3870e5 100644 --- a/scripts/agent-evals/coding-fixtures.ts +++ b/scripts/agent-evals/coding-fixtures.ts @@ -9,7 +9,10 @@ export const CODING_BASE_URL = "https://api.solheim.ai/v1" export const CODING_MODEL_ID = "qwen3.8-27b" const CODING_ROLE_DEFINITION = "Complete one bounded coding task in this workspace. Work only inside the workspace." +// Hosts have no filesystem sandbox. Keep command execution unavailable until +// an equivalent boundary isolates commands from host secrets. const CODING_DISABLED_TOOLS = [ + "execute_command", "ask_followup_question", "new_task", "switch_mode", @@ -42,7 +45,7 @@ function buildCodingConfiguration(apiKey: string): RooCodeSettings { slug: "coding-eval", name: "Coding eval", roleDefinition: CODING_ROLE_DEFINITION, - groups: ["read", "edit", "command"], + groups: ["read", "edit"], }, ], disabledTools: [...CODING_DISABLED_TOOLS], @@ -52,7 +55,7 @@ function buildCodingConfiguration(apiKey: string): RooCodeSettings { alwaysAllowWrite: true, alwaysAllowWriteOutsideWorkspace: false, alwaysAllowWriteProtected: false, - alwaysAllowExecute: true, + alwaysAllowExecute: false, alwaysAllowMcp: false, alwaysAllowModeSwitch: false, alwaysAllowSubtasks: false, @@ -77,7 +80,7 @@ const codingSettingsProjection: Record = { slug: "coding-eval", name: "Coding eval", roleDefinition: CODING_ROLE_DEFINITION, - groups: ["read", "edit", "command"], + groups: ["read", "edit"], }, ], disabledTools: [...CODING_DISABLED_TOOLS], @@ -87,7 +90,7 @@ const codingSettingsProjection: Record = { alwaysAllowWrite: true, alwaysAllowWriteOutsideWorkspace: false, alwaysAllowWriteProtected: false, - alwaysAllowExecute: true, + alwaysAllowExecute: false, alwaysAllowMcp: false, alwaysAllowModeSwitch: false, alwaysAllowSubtasks: false, @@ -148,7 +151,7 @@ const bundle = ( toolPolicy: { mode: "coding-eval", roleDefinition: CODING_ROLE_DEFINITION, - groups: ["read", "edit", "command"], + groups: ["read", "edit"], disabledTools: [...CODING_DISABLED_TOOLS], allow: { readOnly: true, @@ -156,7 +159,7 @@ const bundle = ( write: true, writeOutsideWorkspace: false, writeProtected: false, - execute: true, + execute: false, mcp: false, modeSwitch: false, subtasks: false, @@ -600,7 +603,7 @@ export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ "clamp-range", "1.0.0", "Implement clamp(value, min, max) with empty-range semantics; visible tests miss the empty-range case.", - "Implement the clamp function in src/clamp.js so the test suite passes. The header comment in src/clamp.js defines the exact behavior, including the empty range rule. Check your work with npm test. Do not modify the test files or package.json.", + "Implement the clamp function in src/clamp.js so the test suite passes. The header comment in src/clamp.js defines the exact behavior, including the empty range rule. Command execution is disabled; review your changes without running commands. Do not modify the test files or package.json.", [ workspacePackageJson("clamp-range"), { path: "src/clamp.js", content: clampStub }, @@ -614,7 +617,7 @@ export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ "slugify-text", "1.0.0", "Implement slugify(text) with punctuation-as-separator semantics; visible tests miss punctuation runs.", - "Implement the slugify function in src/slugify.js so the test suite passes. The header comment in src/slugify.js defines the exact behavior. Check your work with npm test. Do not modify the test files or package.json.", + "Implement the slugify function in src/slugify.js so the test suite passes. The header comment in src/slugify.js defines the exact behavior. Command execution is disabled; review your changes without running commands. Do not modify the test files or package.json.", [ workspacePackageJson("slugify-text"), { path: "src/slugify.js", content: slugifyStub }, @@ -628,7 +631,7 @@ export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ "parse-csv-line", "1.0.0", "Implement parseCsvLine(line) with doubled-quote unescaping; visible tests miss escaped quotes.", - "Implement the parseCsvLine function in src/parse-csv-line.js so the test suite passes. The header comment in src/parse-csv-line.js defines the quoting rules, including doubled quotes. Check your work with npm test. Do not modify the test files or package.json.", + "Implement the parseCsvLine function in src/parse-csv-line.js so the test suite passes. The header comment in src/parse-csv-line.js defines the quoting rules, including doubled quotes. Command execution is disabled; review your changes without running commands. Do not modify the test files or package.json.", [ workspacePackageJson("parse-csv-line"), { path: "src/parse-csv-line.js", content: csvStub }, @@ -642,7 +645,7 @@ export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ "format-bytes", "1.0.0", "Implement formatBytes(bytes) with SI decimal units; visible tests miss zero and the MB and GB boundaries.", - "Implement the formatBytes function in src/format-bytes.js so the test suite passes. The header comment in src/format-bytes.js defines the exact units and rounding. Check your work with npm test. Do not modify the test files or package.json.", + "Implement the formatBytes function in src/format-bytes.js so the test suite passes. The header comment in src/format-bytes.js defines the exact units and rounding. Command execution is disabled; review your changes without running commands. Do not modify the test files or package.json.", [ workspacePackageJson("format-bytes"), { path: "src/format-bytes.js", content: bytesStub }, @@ -656,7 +659,7 @@ export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ "deep-get", "1.0.0", "Implement get(object, path) that never throws on missing data; visible tests miss the missing-data paths.", - "Implement the get function in src/deep-get.js so the test suite passes. The header comment in src/deep-get.js defines the missing-data rules. Check your work with npm test. Do not modify the test files or package.json.", + "Implement the get function in src/deep-get.js so the test suite passes. The header comment in src/deep-get.js defines the missing-data rules. Command execution is disabled; review your changes without running commands. Do not modify the test files or package.json.", [ workspacePackageJson("deep-get"), { path: "src/deep-get.js", content: deepGetStub }, diff --git a/scripts/agent-evals/vscode-cache.test.ts b/scripts/agent-evals/vscode-cache.test.ts index d331fd0fef..af9e9a77c0 100644 --- a/scripts/agent-evals/vscode-cache.test.ts +++ b/scripts/agent-evals/vscode-cache.test.ts @@ -23,9 +23,107 @@ it("downloads once, isolates run mutations, and rejects a poisoned reusable cach assert.equal(await fs.readFile(second, "utf8"), "trusted") assert.equal(calls, 1) await fs.writeFile(path.join(source, "code"), "poisoned source") - await assert.rejects(download({ version: "test", cachePath: path.join(dir, "run3") }), /integrity mismatch/) + await assert.rejects( + download({ version: "test", cachePath: path.join(dir, "run3") }), + /^Error: VS Code cache integrity mismatch$/, + ) await assert.rejects(fs.access(path.join(dir, "run3"))) } finally { await fs.rm(dir, { recursive: true, force: true }) } }) + +it("retries a failed download on the next call", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-retry-")) + let calls = 0 + try { + const download = createPrivateVSCodeDownloader(path.join(dir, "private"), async ({ cachePath }) => { + if (++calls === 1) throw new Error("transient download failure") + await fs.writeFile(path.join(cachePath, "code"), "trusted") + return path.join(cachePath, "code") + }) + await assert.rejects(download({ version: "test", cachePath: path.join(dir, "first") }), /transient/) + const executable = await download({ version: "test", cachePath: path.join(dir, "second") }) + assert.equal(await fs.readFile(executable, "utf8"), "trusted") + assert.equal(calls, 2) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +for (const target of ["code", "/outside/code", "../outside/code"]) { + it(`validates cache symlink target ${target}`, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-link-")) + try { + const download = createPrivateVSCodeDownloader(path.join(dir, "private"), async ({ cachePath }) => { + await fs.writeFile(path.join(cachePath, "code"), "trusted") + await fs.symlink(target, path.join(cachePath, "link")) + return path.join(cachePath, "code") + }) + const result = download({ version: "test", cachePath: path.join(dir, "run") }) + if (target === "code") + assert.equal(await fs.readFile(path.join(path.dirname(await result), "link"), "utf8"), "trusted") + else await assert.rejects(result, /VS Code cache symlink escapes cache/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +} + +it("rejects an executable outside the downloaded tree", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-path-")) + try { + const download = createPrivateVSCodeDownloader(path.join(dir, "private"), async () => path.join(dir, "outside")) + await assert.rejects( + download({ version: "test", cachePath: path.join(dir, "run") }), + /VS Code executable outside cache/, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +it("rejects a corrupted copy with a distinct integrity error", async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-copy-")) + const copy = fs.cp + t.mock.method(fs, "cp", async (...args: Parameters) => { + await copy(...args) + await fs.writeFile(path.join(String(args[1]), "code"), "corrupted") + }) + try { + const download = createPrivateVSCodeDownloader(path.join(dir, "private"), async ({ cachePath }) => { + await fs.writeFile(path.join(cachePath, "code"), "trusted") + return path.join(cachePath, "code") + }) + await assert.rejects( + download({ version: "test", cachePath: path.join(dir, "run") }), + /VS Code cache copy integrity mismatch/, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +it("rejects unsupported cache entries", async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "vscode-entry-")) + const lstat = fs.lstat + t.mock.method(fs, "lstat", async (...args: Parameters) => { + const stat = await lstat(...args) + if (String(args[0]).endsWith(path.sep + "special")) { + return Object.assign(stat, { isFile: () => false, isDirectory: () => false, isSymbolicLink: () => false }) + } + return stat + }) + try { + const download = createPrivateVSCodeDownloader(path.join(dir, "private"), async ({ cachePath }) => { + await fs.writeFile(path.join(cachePath, "special"), "fixture") + return path.join(cachePath, "code") + }) + await assert.rejects( + download({ version: "test", cachePath: path.join(dir, "run") }), + /unsupported VS Code cache entry/, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/scripts/agent-evals/vscode-cache.ts b/scripts/agent-evals/vscode-cache.ts index 9d2e35fddd..4682e50e51 100644 --- a/scripts/agent-evals/vscode-cache.ts +++ b/scripts/agent-evals/vscode-cache.ts @@ -45,7 +45,10 @@ export function createPrivateVSCodeDownloader( if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("VS Code executable outside cache") return { source, relative, digest: await digestTree(source) } - })() + })().catch((error: unknown) => { + downloads.delete(version) + throw error + }) downloads.set(version, download) } const cached = await download diff --git a/src/core/tools/__tests__/coding-eval-boundary.spec.ts b/src/core/tools/__tests__/coding-eval-boundary.spec.ts new file mode 100644 index 0000000000..fe914042a9 --- /dev/null +++ b/src/core/tools/__tests__/coding-eval-boundary.spec.ts @@ -0,0 +1,26 @@ +import { describe, expect, it, vi } from "vitest" +import type { AgentFixture } from "../../../../scripts/agent-evals/contract" +import { validateToolUse } from "../validateToolUse" + +describe("coding eval host command boundary", () => { + it.each(["clamp-range", "slugify-text", "parse-csv-line", "format-bytes", "deep-get"])( + "rejects an outside-workspace git diff in %s before command execution", + async (name) => { + // Load the script module through Vitest: it uses a separate tsconfig + // that permits .ts import extensions, unlike the extension sources. + const { CODING_FIXTURE_BUNDLES } = await vi.importActual<{ + CODING_FIXTURE_BUNDLES: { fixture: AgentFixture }[] + }>("../../../../scripts/agent-evals/coding-fixtures") + const fixture = CODING_FIXTURE_BUNDLES.find(({ fixture }) => fixture.identity.name === name)!.fixture + const config = fixture.buildConfiguration("test-key") + const params = { command: "git diff --no-index /dev/null /home/user/.aws/credentials" } + expect(config.alwaysAllowExecute).toBe(false) + expect(config.disabledTools).toContain("execute_command") + expect(() => + validateToolUse("execute_command", "coding-eval", config.customModes, undefined, params), + ).toThrow('Tool "execute_command" is not allowed') + expect(() => validateToolUse("read_file", "coding-eval", config.customModes)).not.toThrow() + expect(() => validateToolUse("write_to_file", "coding-eval", config.customModes)).not.toThrow() + }, + ) +}) From 958fdac6aba0002ae0b74923be5403e56a7837d9 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 7 Nov 2026 15:17:15 +0000 Subject: [PATCH 22/22] test(evals): verify disabled commands are blocked at dispatch --- .../__tests__/coding-eval-boundary.spec.ts | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/core/tools/__tests__/coding-eval-boundary.spec.ts b/src/core/tools/__tests__/coding-eval-boundary.spec.ts index fe914042a9..cca75f8e9f 100644 --- a/src/core/tools/__tests__/coding-eval-boundary.spec.ts +++ b/src/core/tools/__tests__/coding-eval-boundary.spec.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from "vitest" import type { AgentFixture } from "../../../../scripts/agent-evals/contract" +import type { Task } from "../../task/Task" +import { DispatchState } from "../../task/dispatchState" +import { presentAssistantMessage } from "../../assistant-message/presentAssistantMessage" +import { executeCommandTool } from "../ExecuteCommandTool" import { validateToolUse } from "../validateToolUse" describe("coding eval host command boundary", () => { @@ -24,3 +28,69 @@ describe("coding eval host command boundary", () => { }, ) }) + +vi.mock("../ExecuteCommandTool", () => ({ + executeCommandTool: { handle: vi.fn().mockResolvedValue(undefined) }, +})) +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + instance: { captureToolUsage: vi.fn(), captureEvent: vi.fn() }, + }, +})) + +describe("coding eval command dispatch", () => { + it.each([ + { mode: "coding-eval", disabled: true, allowed: false }, + { mode: "code", disabled: true, allowed: false }, + { mode: "code", disabled: false, allowed: true }, + ])("dispatches in $mode with disabled=$disabled only when allowed", async ({ mode, disabled, allowed }) => { + vi.mocked(executeCommandTool.handle).mockClear() + const { CODING_FIXTURE_BUNDLES } = await vi.importActual<{ + CODING_FIXTURE_BUNDLES: { fixture: AgentFixture }[] + }>("../../../../scripts/agent-evals/coding-fixtures") + const config = CODING_FIXTURE_BUNDLES[0].fixture.buildConfiguration("test-key") + const pushToolResultToUserContent = vi.fn().mockReturnValue(true) + const task = { + taskId: "boundary-test", + instanceId: "boundary-instance", + abort: false, + dispatchState: DispatchState.IDLE, + pendingWork: null, + currentStreamingContentIndex: 0, + assistantMessageContent: [ + { + type: "tool_use", + id: "outside-read", + name: "execute_command", + nativeArgs: { command: "git diff --no-index /dev/null /home/user/.aws/credentials" }, + params: { command: "git diff --no-index /dev/null /home/user/.aws/credentials" }, + partial: false, + }, + ], + userMessageContent: [], + activeToolContexts: new Map(), + consecutiveMistakeCount: 0, + getTaskMode: vi.fn().mockResolvedValue(mode), + recordToolUsage: vi.fn(), + recordToolError: vi.fn(), + toolRepetitionDetector: { check: vi.fn().mockReturnValue({ action: "allow" }) }, + pushToolResultToUserContent, + say: vi.fn().mockResolvedValue(undefined), + } + // The dispatcher needs only the stream state, policy, and recording methods supplied above. + await presentAssistantMessage(task as unknown as Task, { + customModes: config.customModes, + disabledTools: disabled ? config.disabledTools : [], + experiments: {}, + }) + expect(executeCommandTool.handle).toHaveBeenCalledTimes(allowed ? 1 : 0) + if (!allowed) { + expect(pushToolResultToUserContent).toHaveBeenCalledWith( + expect.objectContaining({ + is_error: true, + content: expect.stringContaining("is not allowed"), + }), + ) + } + }) +})