diff --git a/AGENTS.md b/AGENTS.md index 86faadb2d0..e48418f993 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,8 @@ Defined in `src/types.ts`. Both extractors and resolvers must use these exact st **Required** for every new language/framework: validate on small/medium/large real repos with >=3 flow prompts; deterministic `scripts/agent-eval/probe-explore.mjs` probes, then agent A/B (`scripts/agent-eval/run-all.sh` / `ab-new-vs-baseline.sh`). Pass bar: ~0 Read/Grep within the explore-call budget, faster than without-codegraph, no control-repo regression. +**Every other retrieval change** (explore ranking, budgets, rendering, search) is gated by replaying saved agent queries, not by an agent A/B: `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (about 10 s on 46 queries). Read every changed or regressed answer and say in the PR why each is acceptable. Run one agent A/B per release batch, before a build is promoted or released, and add its new query shapes to the corpus with `harvest-queries.mjs`. Details: `docs/AGENTS.md` § Query replay. + Full methodology (feedback metrics, CLI contamination guard, Sonnet/`--effort high` model policy, daemon pre-warm), the Excalidraw worked example, and coverage matrix live in: - `docs/AGENTS.md` (nested; also loaded when cwd is under `docs/`) - `docs/design/dynamic-dispatch-coverage-playbook.md` diff --git a/__tests__/query-replay.test.ts b/__tests__/query-replay.test.ts new file mode 100644 index 0000000000..97ba75fc26 --- /dev/null +++ b/__tests__/query-replay.test.ts @@ -0,0 +1,85 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +// @ts-expect-error untyped ESM script +import { callsInLine, repoSlug } from '../scripts/agent-eval/harvest-queries.mjs'; +// @ts-expect-error untyped ESM script +import { compareRow } from '../scripts/agent-eval/replay-queries.mjs'; + +const row = (files: string[], over: Record = {}) => ({ + id: 'q', chars: 100, sourceAt: 10, files, mustMatchAt: [], ...over, +}); + +describe('replay comparison', () => { + it('passes the same files in another order', () => { + const r = compareRow(row(['a.ts', 'b.ts']), row(['b.ts', 'a.ts'])); + expect(r.regressed).toBe(false); + expect(r.changed).toBe(false); + }); + + it('flags a lost file, and names it when the query asked for it', () => { + const r = compareRow(row(['a.ts', 'src/b.ts']), row(['a.ts']), 'explain src/b.ts'); + expect(r.regressed).toBe(true); + expect(r.lostFiles).toEqual(['src/b.ts']); + expect(r.lostNamed).toEqual(['src/b.ts']); + }); + + it('flags a lost source section and a pattern the base found', () => { + expect(compareRow(row(['a.ts']), row([], { sourceAt: -1 })).lostSource).toBe(true); + expect(compareRow(row([], { mustMatchAt: [5] }), row([], { mustMatchAt: [-1] })).lostMatch).toBe(1); + }); + + it('counts different text with the same files and length as a change', () => { + const r = compareRow(row(['a.ts']), row(['a.ts']), '', false); + expect(r).toMatchObject({ regressed: false, changed: true }); + }); + + it('counts a gained file as a change without loss', () => { + const r = compareRow(row(['a.ts']), row(['a.ts', 'c.ts'])); + expect(r).toMatchObject({ regressed: false, changed: true, gainedFiles: ['c.ts'] }); + }); +}); + +describe('query harvest', () => { + it('reads Claude tool_use, Codex function calls and Codex code-mode calls', () => { + expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'mcp__codegraph__codegraph_explore', input: { query: 'A b', projectPath: '/x' } }] } })) + .toEqual([{ query: 'A b', projectPath: '/x' }]); + expect(callsInLine({ payload: { type: 'function_call', name: 'mcp__codegraph__codegraph_explore', arguments: '{"query":"C"}' } })) + .toEqual([{ query: 'C', projectPath: undefined }]); + const input = 'text(await tools.mcp__codegraph__codegraph_explore({projectPath:"/y",query:"D \\"e\\"",maxFiles:8}));'; + expect(callsInLine({ payload: { type: 'custom_tool_call', input } })).toEqual([{ query: 'D "e"', projectPath: '/y', maxFiles: 8 }]); + expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'Read', input: { query: 'no' } }] } })).toEqual([]); + }); + + it('reads code-mode calls with braces, single quotes and file limits', () => { + const call = (input: string) => callsInLine({ payload: { type: 'custom_tool_call', input } }); + expect(call('codegraph_explore({query:"make({a: 1}) } brace"})')).toEqual([{ query: 'make({a: 1}) } brace', projectPath: undefined }]); + expect(call("codegraph_explore({ query: 'auth flow', maxFiles: 12 })")).toEqual([{ query: 'auth flow', projectPath: undefined, maxFiles: 12 }]); + expect(call('codegraph_explore({query:`a ${x}`})')).toEqual([]); + expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'mcp__codegraph__codegraph_explore', input: { query: 'Q', maxFiles: 3 } }] } })) + .toEqual([{ query: 'Q', projectPath: undefined, maxFiles: 3 }]); + }); + + it('maps GitHub remotes to owner/repo', () => { + expect(repoSlug('git@github.com:Owner/Repo.git\n')).toBe('owner/repo'); + expect(repoSlug('https://github.com/owner/repo')).toBe('owner/repo'); + expect(repoSlug('ssh://git@github.com/owner/repo.git')).toBe('owner/repo'); + expect(repoSlug('https://gitlab.com/owner/repo.git')).toBeNull(); + expect(repoSlug('https://notgithub.com/owner/repo.git')).toBeNull(); + expect(repoSlug('git@privategithub.com:owner/repo.git')).toBeNull(); + }); +}); + +describe('committed replay corpora', () => { + const dir = resolve(import.meta.dirname, '../scripts/agent-eval/replay'); + for (const name of readdirSync(dir).filter((f) => f.endsWith('.json'))) { + it(`${name} is pinned, unique and has no local paths`, () => { + const text = readFileSync(join(dir, name), 'utf8'); + const corpus = JSON.parse(text); + expect(corpus.commit).toMatch(/^[0-9a-f]{7,40}$/); + const ids = corpus.tasks.map((t: { id: string }) => t.id); + expect(new Set(ids).size).toBe(ids.length); + expect(text).not.toMatch(/\/home\/|\/Users\/|[A-Z]:\\\\|~\//); + }); + } +}); diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 9a2a6d0769..f6a93cdfd4 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -10,7 +10,7 @@ For each **language × framework**, validate on **small, medium, and large** rea 1. **Pick the canonical flow** for the framework ("how does X reach Y": state→render, request→handler→view, query→SQL, action→reducer→store…). 2. **Deterministic probes** (`scripts/agent-eval/probe-{node,explore}.mjs` against the built `dist/`): `codegraph_explore` with the flow's symbol names connects from→to end-to-end with no break (its Flow section shows the path); **no node explosion** (`select count(*) from nodes` stable before/after re-index); synthesized-edge **precision** spot-check (`select … where provenance='heuristic'`). -3. **Agent A/B** (`scripts/agent-eval/run-all.sh ""`): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`); export `CODEGRAPH_REPO_ROOT` as this checkout's absolute path before launching Claude. +3. **Agent A/B** (`scripts/agent-eval/run-all.sh ""`), for a new language or framework and once per release batch (a change to existing retrieval uses the query replay below instead): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`); export `CODEGRAPH_REPO_ROOT` as this checkout's absolute path before launching Claude. - **Every run also reports three feedback metrics** — residual context occupancy, explore sufficiency (what the agent did NEXT after each explore), and allocation efficiency (share of returned bytes the answer cited) — under each run, plus a side-by-side arm table (`compare-arms.mjs`). Entry point: `docs/benchmarks/agent-eval-feedback-metrics.md`. Reading them: `Read a file we returned` is an allocation miss, `Read a file we did NOT return`/`Grep` is recall; allocation efficiency is **relative** (attribution is by citation) so it is only valid between builds on the same question; occupancy *shares* are Claude Code / 200k and don't transfer to another host — the arm ratio does. - **The `codegraph` CLI is blocked in every arm** (`no-cli-shim.sh`: sanitized PATH + a PreToolUse hook, shared by both harnesses). Without it 14 of 15 without-arm runs in one 7-repo pass reached codegraph through Bash. Check the contamination row before believing any number: `CLI calls that RETURNED output` > 0 invalidates the run (in a new-vs-baseline A/B it silently drops calls from all three metrics, since a CLI explore is not a tool call). - **Model selection and availability.** The Claude harness defaults to `--model sonnet --effort high` for floor-model validation. If that model is unavailable through quota, an outage or an unsupported route, use the next available eligible model and effort from the operator's current benchmark-backed bug-finding order. Primary, secondary and tertiary choices change with benchmark results; read the current selection rather than fixing fallback names here. Record the selected model, effort, ranking source and fallback reason. @@ -21,6 +21,17 @@ For each **language × framework**, validate on **small, medium, and large** rea Full playbook + per-mechanism design: `docs/design/dynamic-dispatch-coverage-playbook.md` and `docs/design/callback-edge-synthesis.md`. +### Query replay (gate for changes to existing retrieval) + +A change to explore ranking, budgets, rendering or search replays real agent queries against the base and candidate builds instead of running an agent A/B. The past change-gating A/Bs mostly agreed with the deterministic probes; the defects only agents caught came from query shapes the probes lacked, and replaying those agents' queries reproduced each one. The corpus keeps those shapes. + +1. Check out the corpus repository at the corpus's `commit` and index it with either build (`codegraph init -y`). The replay refuses a checkout at another commit. +2. `node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/.json --base --cand --out ` (each build is a checkout with `dist/` built). It prints a table of answers whose text differs and exits 1 when an answer lost its source section, a file the base rendered (marked when the query names the file) or a `mustMatch` pattern. Answers are saved under `/base` and `/cand`. +3. Read every changed and regressed answer. A lost file can be a fix (a peripheral file swapped for a better one); say so per query in the PR. Same-build replay is identical, so any change comes from the candidate. +4. Once per release batch, before a build is promoted or released, run one agent A/B (`ab-new-vs-baseline.sh`) on a flow question the corpus covers least, then harvest its queries: `node scripts/agent-eval/harvest-queries.mjs --public ...`. It keeps queries only on repositories named with `--public` and writes no paths; still read every prompt before committing, since a public query can quote a private name. `__tests__/query-replay.test.ts` fails a committed corpus that contains a local path. + +Measured on `replay/codegraph.json` (46 queries, this repository at `2a4af7c3`; 29 replay the agent's own `maxFiles`): same build 46 of 46 identical in 9 s; a build whose default limit is two files flagged 16 of the 17 queries that use the default, 4 of them for a file the query named; the root `__tests__/` test-path fix (#500) changed 6 answers and flagged none, one of them with the same length and different text. + ### Worked example — Excalidraw (TS/React, medium, 643 files) This table records the historical trace-era evaluation; current validation uses diff --git a/scripts/agent-eval/harvest-queries.mjs b/scripts/agent-eval/harvest-queries.mjs new file mode 100644 index 0000000000..0bc854375b --- /dev/null +++ b/scripts/agent-eval/harvest-queries.mjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// Collect the codegraph_explore queries real agent sessions asked, for replay +// with replay-queries.mjs. +// +// node scripts/agent-eval/harvest-queries.mjs --public [--public ...] [--since 2026-09-01] ... +// +// Reads Claude Code transcripts (assistant tool_use blocks) and Codex rollouts +// (MCP tool calls and code-mode `tools.mcp__codegraph__codegraph_explore({...})` +// calls). Each call's project is its projectPath, else the session's working +// directory, mapped to a repository through `git remote get-url origin`. Only +// queries on a repository named by --public are kept; everything else is +// dropped and counted, so a private project never reaches the output. Output +// on stdout: `{ repos: { "": { tasks: [{ id, prompt, maxFiles? }] } } }`, +// deduplicated on query and file limit, with no paths. Review every prompt before committing it: a +// query on a public repository can still quote a private name. +import { execFileSync } from 'node:child_process'; +import { createReadStream, readdirSync, statSync } from 'node:fs'; +import { createInterface } from 'node:readline'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; + +/** Explore calls in one transcript line: [{ query, projectPath?, maxFiles? }]. */ +export function callsInLine(entry) { + const calls = []; + const blocks = entry?.message?.content; + if (Array.isArray(blocks)) { + for (const b of blocks) { + if (b?.type === 'tool_use' && /codegraph_explore$/.test(b.name ?? '') && typeof b.input?.query === 'string') { + calls.push({ query: b.input.query, projectPath: b.input.projectPath, maxFiles: b.input.maxFiles }); + } + } + } + const p = entry?.payload; + if (p?.type === 'function_call' && /codegraph_explore$/.test(p.name ?? '')) { + try { + const args = JSON.parse(p.arguments); + if (typeof args.query === 'string') calls.push({ query: args.query, projectPath: args.projectPath, maxFiles: args.maxFiles }); + } catch { /* not JSON arguments */ } + } + if (p?.type === 'custom_tool_call' && typeof p.input === 'string') { + for (const m of p.input.matchAll(/codegraph_explore\(\s*\{/g)) { + const fields = literalFields(p.input, m.index + m[0].length - 1); + if (typeof fields.query === 'string') calls.push({ query: fields.query, projectPath: fields.projectPath, maxFiles: fields.maxFiles }); + } + } + return calls.map((c) => (typeof c.maxFiles === 'number' ? c : { query: c.query, projectPath: c.projectPath })); +} + +const ESCAPES = { n: '\n', t: '\t', r: '\r', b: '\b', f: '\f', v: '\v', 0: '\0' }; + +/** + * The top-level string and number fields of the JavaScript object literal that + * opens at `src[open]` ('{'). Code-mode calls are JavaScript, not JSON: keys may + * be bare, strings may use any quote and may contain braces. Values that are + * not plain literals (expressions, template strings with `${`) are skipped. + */ +export function literalFields(src, open) { + const fields = {}; + let depth = 0, key, expectKey = false, expectValue = false; + for (let i = open; i < src.length; i++) { + const ch = src[i]; + if (ch === '"' || ch === "'" || ch === '`') { + let s = '', literal = true, j = i + 1; + for (; j < src.length && src[j] !== ch; j++) { + if (src[j] === '\\') { j++; s += ESCAPES[src[j]] ?? src[j]; } + else if (ch === '`' && src[j] === '$' && src[j + 1] === '{') { literal = false; s += src[j]; } + else s += src[j]; + } + i = j; + if (depth === 1 && expectKey) { key = s; expectKey = false; } + else if (depth === 1 && expectValue && key !== undefined && literal) fields[key] = s; + expectValue = false; + } else if (ch === '{' || ch === '[' || ch === '(') { + depth++; + if (depth === 1) expectKey = true; + expectValue = false; + } else if (ch === '}' || ch === ']' || ch === ')') { + if (--depth === 0) break; + } else if (depth === 1 && ch === ',') { + expectKey = true; expectValue = false; key = undefined; + } else if (depth === 1 && ch === ':') { + expectValue = true; + } else if (depth === 1 && /[A-Za-z_$0-9]/.test(ch)) { + const word = /^[A-Za-z_$0-9.]+/.exec(src.slice(i))[0]; + i += word.length - 1; + if (expectKey) { key = word; expectKey = false; } + else if (expectValue && key !== undefined && /^\d+$/.test(word)) fields[key] = Number(word); + expectValue = false; + } + } + return fields; +} + +/** `owner/repo` from a remote on github.com itself (HTTPS, SSH URL or scp form), or null. */ +export function repoSlug(url) { + const text = url.trim(); + const scp = /^git@github\.com:([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/.exec(text); + if (scp) return `${scp[1]}/${scp[2]}`.toLowerCase(); + let parsed; + try { parsed = new URL(text); } catch { return null; } + if (parsed.hostname.toLowerCase() !== 'github.com') return null; + const m = /^\/([^/]+)\/([^/]+?)(?:\.git)?\/?$/.exec(parsed.pathname); + return m ? `${m[1]}/${m[2]}`.toLowerCase() : null; +} + +function* jsonlFiles(path) { + const st = statSync(path, { throwIfNoEntry: false }); + if (!st) return; + if (st.isFile()) { if (path.endsWith('.jsonl')) yield path; return; } + for (const name of readdirSync(path)) yield* jsonlFiles(join(path, name)); +} + +async function main(argv) { + const pub = new Set(); + const inputs = []; + let since = ''; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--public') pub.add(argv[++i].toLowerCase()); + else if (argv[i] === '--since') since = argv[++i]; + else inputs.push(argv[i]); + } + if (!pub.size || !inputs.length) { + console.error('usage: harvest-queries.mjs --public [--public ...] [--since YYYY-MM-DD] ...'); + return 2; + } + const slugOf = new Map(); + const slugFor = (dir) => { + if (!slugOf.has(dir)) { + let slug = null; + try { + slug = repoSlug(execFileSync('git', ['-C', dir, 'remote', 'get-url', 'origin'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] })); + } catch { /* gone or not a repository */ } + slugOf.set(dir, slug); + } + return slugOf.get(dir); + }; + const repos = {}; + const seen = new Set(); + let kept = 0, dropped = 0, duplicate = 0; + for (const input of inputs) { + for (const file of jsonlFiles(input)) { + let cwd; + // Line by line: a long transcript can exceed the largest string a runtime allows. + for await (const line of createInterface({ input: createReadStream(file, 'utf8'), crlfDelay: Infinity })) { + if (!line.includes('codegraph_explore') && !line.includes('"cwd"')) continue; + let entry; + try { entry = JSON.parse(line); } catch { continue; } + cwd = entry.cwd ?? entry.payload?.cwd ?? cwd; + if (since && (entry.timestamp ?? '') < since) continue; + for (const call of callsInLine(entry)) { + const dir = call.projectPath ?? cwd; + const slug = dir ? slugFor(dir) : null; + if (!slug || !pub.has(slug)) { dropped++; continue; } + const prompt = call.query.trim().replace(/\s+/g, ' '); + const limit = call.maxFiles === undefined ? '' : `\0${call.maxFiles}`; + const key = `${slug}\0${prompt.toLowerCase()}${limit}`; + if (seen.has(key)) { duplicate++; continue; } + seen.add(key); + const id = createHash('sha256').update(prompt + limit).digest('hex').slice(0, 10); + const task = { id: `q-${id}`, prompt }; + if (call.maxFiles !== undefined) task.maxFiles = call.maxFiles; + (repos[slug] ??= { tasks: [] }).tasks.push(task); + kept++; + } + } + } + } + console.log(JSON.stringify({ repos }, null, 2)); + console.error(`kept ${kept}, duplicates ${duplicate}, dropped ${dropped} (not on a --public repository)`); + return 0; +} + +if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2)).then((code) => process.exit(code)); diff --git a/scripts/agent-eval/probe-explore.mjs b/scripts/agent-eval/probe-explore.mjs index 11e6b28faf..c757b22636 100644 --- a/scripts/agent-eval/probe-explore.mjs +++ b/scripts/agent-eval/probe-explore.mjs @@ -14,7 +14,8 @@ // files the answer renders) and mustMatchAt (first offset of each task's // mustMatch pattern, -1 when missing). A final row with id "memory" reports // resident memory idle and after the run. tasks.json is `{ "tasks": [...] }` -// or a bare array of `{ id, prompt, mustMatch? }`. +// or a bare array of `{ id, prompt, mustMatch?, maxFiles? }`; maxFiles is +// passed to codegraph_explore as the caller's file limit. // // Options: // --build codegraph checkout whose dist/ answers (default: this one) @@ -103,8 +104,8 @@ async function inProcess(build, repo) { const handler = new ToolHandler(cg); return { pid: process.pid, - async explore(query) { - const res = await handler.execute('codegraph_explore', { query }); + async explore(query, maxFiles) { + const res = await handler.execute('codegraph_explore', maxFiles === undefined ? { query } : { query, maxFiles }); if (res.isError) throw new Error(`codegraph_explore: ${JSON.stringify(res.content)}`); return (res.content ?? []).map((c) => c.text ?? '').join('\n'); }, @@ -171,8 +172,9 @@ async function overMcp(build, repo, runtime, timeoutMs) { proc.stdin.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized', params: {} }) + '\n'); return { pid: proc.pid, - async explore(query) { - const res = await rpc('tools/call', { name: 'codegraph_explore', arguments: { query, projectPath: repo } }); + async explore(query, maxFiles) { + const args = maxFiles === undefined ? { query, projectPath: repo } : { query, projectPath: repo, maxFiles }; + const res = await rpc('tools/call', { name: 'codegraph_explore', arguments: args }); return (res.result?.content ?? []).map((c) => c.text ?? '').join('\n'); }, close, @@ -206,7 +208,7 @@ async function main() { const idle = memMB(session.pid); for (const task of tasks) { const t0 = Date.now(); - const text = await session.explore(task.prompt); + const text = await session.explore(task.prompt, task.maxFiles); const row = probeRow(task.id, text, Date.now() - t0, task.mustMatch); missing += row.mustMatchAt.filter((at) => at < 0).length; if (opts.out) writeFileSync(join(opts.out, `${task.id}.md`), text); diff --git a/scripts/agent-eval/replay-queries.mjs b/scripts/agent-eval/replay-queries.mjs new file mode 100644 index 0000000000..ad7944c20c --- /dev/null +++ b/scripts/agent-eval/replay-queries.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node +// Replay saved agent queries against two builds and diff the answers. +// +// node scripts/agent-eval/replay-queries.mjs --tasks --base --cand --out [--ids globs] +// +// is an indexed checkout at the corpus's pinned commit (the corpus file +// names it). Each build answers every query through probe-explore.mjs in its +// own process. A query regresses when the candidate's answer loses its Source +// Code section, drops a source file the base rendered (worst when the query +// names that file), or misses a mustMatch pattern the base found. Prints a Markdown table of changed queries and a +// summary line; exits 1 on any regression, 0 otherwise. Answers are saved +// under /base and /cand for reading the diff; keep off /tmp. +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +/** How one query's answer moved from base to candidate. */ +export function compareRow(base, cand, prompt = '', sameText = base.chars === cand.chars) { + const lostFiles = base.files.filter((f) => !cand.files.includes(f)); + const lostNamed = lostFiles.filter((f) => prompt.includes(f)); + const gainedFiles = cand.files.filter((f) => !base.files.includes(f)); + const lostSource = base.sourceAt >= 0 && cand.sourceAt < 0; + const lostMatch = base.mustMatchAt.filter((at, i) => at >= 0 && (cand.mustMatchAt[i] ?? -1) < 0).length; + const regressed = lostSource || lostFiles.length > 0 || lostMatch > 0; + const changed = regressed || gainedFiles.length > 0 || !sameText; + return { id: base.id, regressed, changed, lostSource, lostFiles, lostNamed, gainedFiles, lostMatch, baseChars: base.chars, candChars: cand.chars }; +} + +function probe(build, repo, tasks, ids, out) { + const script = fileURLToPath(new URL('./probe-explore.mjs', import.meta.url)); + const args = [script, repo, '--tasks', tasks, '--build', build, '--out', out]; + if (ids) args.push('--ids', ids); + const res = spawnSync(process.execPath, args, { encoding: 'utf8', maxBuffer: 64 << 20, stdio: ['ignore', 'pipe', 'inherit'] }); + if (res.status !== 0) throw new Error(`probe-explore with ${build} exited ${res.status}`); + const rows = new Map(); + for (const line of res.stdout.split('\n')) { + if (!line.startsWith('{')) continue; + const row = JSON.parse(line); + if (row.id !== 'memory') rows.set(row.id, row); + } + return rows; +} + +function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { tasks: { type: 'string' }, base: { type: 'string' }, cand: { type: 'string' }, out: { type: 'string' }, ids: { type: 'string' } }, + }); + const [repoArg] = positionals; + if (!repoArg || !values.tasks || !values.base || !values.cand || !values.out) { + console.error('usage: replay-queries.mjs --tasks --base --cand --out [--ids globs]'); + return 2; + } + const repo = resolve(repoArg); + const out = resolve(values.out); + const corpus = JSON.parse(readFileSync(values.tasks, 'utf8')); + if (corpus.commit) { + const head = spawnSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { encoding: 'utf8' }).stdout.trim(); + if (!head.startsWith(corpus.commit)) { + console.error(`${repo} is at ${head.slice(0, 12)}, the corpus is pinned to ${corpus.commit}; answers would not be comparable`); + return 2; + } + } + const base = probe(resolve(values.base), repo, values.tasks, values.ids, join(out, 'base')); + const cand = probe(resolve(values.cand), repo, values.tasks, values.ids, join(out, 'cand')); + if (base.size === 0) { + console.error('no queries selected; check --ids and the corpus'); + return 2; + } + const prompts = new Map(corpus.tasks.map((t) => [t.id, t.prompt])); + const answer = (side, id) => readFileSync(join(out, side, `${id}.md`), 'utf8'); + const rows = [...base.values()].map((b) => { + const c = cand.get(b.id); + if (!c) throw new Error(`candidate has no answer for ${b.id}`); + return compareRow(b, c, prompts.get(b.id), answer('base', b.id) === answer('cand', b.id)); + }); + const changed = rows.filter((r) => r.changed); + const regressed = rows.filter((r) => r.regressed); + if (changed.length) { + console.log('| Query | Result | Chars (base → cand) | Files lost | Files gained |'); + console.log('|---|---|---:|---|---|'); + for (const r of changed) { + const result = !r.regressed ? 'changed' + : r.lostSource ? 'REGRESSED: no source' + : r.lostNamed.length ? 'REGRESSED: named file lost' + : r.lostMatch ? 'REGRESSED: pattern missing' + : 'REGRESSED: file lost'; + console.log(`| ${r.id} | ${result} | ${r.baseChars} → ${r.candChars} | ${r.lostFiles.join(', ') || '-'} | ${r.gainedFiles.join(', ') || '-'} |`); + } + console.log(''); + } + console.log(`${rows.length} queries: ${rows.length - changed.length} identical, ${changed.length - regressed.length} changed without loss, ${regressed.length} regressed. Answers: ${out}`); + return regressed.length ? 1 : 0; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + process.exit(main()); + } catch (error) { + console.error(`[replay-queries] ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} diff --git a/scripts/agent-eval/replay/codegraph.json b/scripts/agent-eval/replay/codegraph.json new file mode 100644 index 0000000000..b7f7d951d7 --- /dev/null +++ b/scripts/agent-eval/replay/codegraph.json @@ -0,0 +1,223 @@ +{ + "$comment": [ + "Real codegraph_explore queries agents asked about this repository, harvested by harvest-queries.mjs and reviewed by hand.", + "Replay them against bompus/codegraph checked out at commit, indexed with either build: node scripts/agent-eval/replay-queries.mjs --tasks scripts/agent-eval/replay/codegraph.json --base --cand --out ." + ], + "repository": "bompus/codegraph", + "commit": "2a4af7c3", + "tasks": [ + { + "id": "q-89489f4611", + "prompt": "is_lexically_reachable is_bare_js_call js_store_binding js_unit_to_byte match_by_exact_name" + }, + { + "id": "q-5a5324c072", + "prompt": "call_shape_target kotlin_call_receiver_type call_site kotlin_unknown_receiver_allowed kotlin_infix_site retain_lang_scope_tracked" + }, + { + "id": "q-58b626100e", + "prompt": "KernelResolver read_file source cache sorted_files kotlin_companion_receiver_memo" + }, + { + "id": "q-0bf6fd8205", + "prompt": "getExploreBudget getExploreOutputBudget src/mcp/tools.ts" + }, + { + "id": "q-4ce3a49f75", + "prompt": "writer lock CODEGRAPH_NO_DAEMON readonly no watch serve mcp auto sync environment" + }, + { + "id": "q-5329a0730f", + "prompt": "server-instructions.ts SERVER_INSTRUCTIONS" + }, + { + "id": "q-2c1bab2282", + "prompt": "instructions-template.ts INSTRUCTIONS_BODY upsertInstructionsEntry CODEGRAPH_START" + }, + { + "id": "q-bb5c2f3458", + "prompt": "CHANGELOG.md Unreleased New Features Fixes" + }, + { + "id": "q-ce29b1b841", + "prompt": "writer.pid fallback mode daemon start writer lock acquire SIGTERM shutdown clients idle timeout refresh launcher fallback proxy existing MCP session", + "maxFiles": 10 + }, + { + "id": "q-83904696a8", + "prompt": "handleExplore getExploreOutputBudget pinnedFiles source excerpts exact path literal selection Vue test assertions" + }, + { + "id": "q-0a123254bf", + "prompt": "src/extraction/index.ts extractFileWithParser retryOnCorruption: trace WASM heap recreation retry guards to parseNativeTree", + "maxFiles": 2 + }, + { + "id": "q-08aa47e98e", + "prompt": "TreeSitterExtractor extract isTreeSitterTimeoutError parseAst src/extraction/parse-worker.ts: how are native parser errors and memory messages propagated to retries?", + "maxFiles": 3 + }, + { + "id": "q-75f1ef5562", + "prompt": "src/extraction/parse-pool.ts ParseWorkerPool recycle recycleAll handleWorkerError onMessage timeout: which live native worker errors trigger restart and retry?", + "maxFiles": 1 + }, + { + "id": "q-79895a605a", + "prompt": "src/mcp explicit project idle writer lock LRU release structuredContent text-only stale explore status; codegraph-kernel/src/resolve/bound.rs lexical siblings closure declaration method_call shadow field" + }, + { + "id": "q-4b9d4fe1e9", + "prompt": "src/mcp/project-lifecycle.ts src/mcp/tools.ts trimProjects setProjectLifecycle src/mcp/handlers/base.ts structuredContent textResult __tests__ idle explicit project release stale text-only structuredContent" + }, + { + "id": "q-82c2bc4abb", + "prompt": "__tests__/mcp-projectpath-lifecycle.test.ts __tests__/mcp-idle-release-order.test.ts __tests__/mcp-stale-refusal.test.ts __tests__/mcp-status-freshness.test.ts" + }, + { + "id": "q-56ad613635", + "prompt": "codegraph-kernel/src/resolve/receivers.rs infer_local_receiver_type codegraph-kernel/src/resolve/names.rs innermost_binding codegraph-kernel/src/resolve/method_call.rs receiver_field_type __tests__/bindings-receiver.test.ts" + }, + { + "id": "q-5bee81ff55", + "prompt": "codegraph-kernel/src/resolve/prefilter.rs innermost_binding codegraph-kernel/src/resolve/method_call.rs match_method_call java_field_receiver __tests__/bindings-receiver.test.ts __tests__/receiver-owners.test.ts" + }, + { + "id": "q-84a7939e6c", + "prompt": "__tests__/bindings-receiver.test.ts __tests__/bound-owner-walk.test.ts codegraph-kernel/src/tsjs/bindings.rs local_scope codegraph-kernel/src/go/bindings.rs" + }, + { + "id": "q-e3cbe24999", + "prompt": "infer_java_field_receiver_type infer_declared_member_receiver_type jvm_field_receiver" + }, + { + "id": "q-1d3452b493", + "prompt": "SQLite database driver node:sqlite read-only worker pool query-worker database connection", + "maxFiles": 4 + }, + { + "id": "q-512a0429cd", + "prompt": "src/mcp/tools.ts trimProjects DEFAULT_PROJECT_IDLE getProject defaultProject src/mcp/project-lifecycle.ts writer lock fallback idle release", + "maxFiles": 5 + }, + { + "id": "q-41b6397dee", + "prompt": "__tests__/mcp-project-idle.test.ts idleRelease engine default project CODEGRAPH_PROJECT_IDLE_TIMEOUT_MS", + "maxFiles": 3 + }, + { + "id": "q-48aa64ac59", + "prompt": "__tests__/mcp-projectpath-lifecycle.test.ts __tests__/mcp-idle-release-order.test.ts __tests__/mcp-writer-lock.test.ts", + "maxFiles": 3 + }, + { + "id": "q-4184a8ada8", + "prompt": "src/mcp/proxy.ts ensureEngine retireEngine scheduleDaemonRetry CODEGRAPH_DAEMON_RETRY_MS src/mcp/engine.ts ensureInitialized stop defaultLease", + "maxFiles": 2 + }, + { + "id": "q-01c72f6fb7", + "prompt": "src/mcp/proxy.ts LocalHandshakeDeps runLocalHandshakeProxy stdin onData getDaemonSocket makeEngine __tests__/proxy-shutdown.test.ts __tests__/mcp-daemon.test.ts fallback retries hand writer", + "maxFiles": 3 + }, + { + "id": "q-982311adc5", + "prompt": "src/codegraph.ts initSync src/kernel/loader.ts require codegraph-kernel src/mcp/engine.ts catchUpSync startWatching", + "maxFiles": 3 + }, + { + "id": "q-17fd054490", + "prompt": "__tests__/proxy-shutdown.test.ts startProxy env src/mcp/liveness-watchdog.ts installMainThreadWatchdog env", + "maxFiles": 2 + }, + { + "id": "q-6132e3251f", + "prompt": "kernel build stamp untracked source invalidation build-kernel scripts build stamp hash git ls-files", + "maxFiles": 4 + }, + { + "id": "q-91ba466d1c", + "prompt": "scripts/kernel-stamp.mjs scripts/ensure-kernel.mjs __tests__/ensure-kernel-stamp.test.ts", + "maxFiles": 3 + }, + { + "id": "q-65ca5f2c30", + "prompt": "src/mcp/tools.ts callersTool calleesTool codegraph_callers codegraph_callees", + "maxFiles": 1 + }, + { + "id": "q-7759bc1b75", + "prompt": "handleCallers handleCallees collectWiringNotes nodeHeader", + "maxFiles": 2 + }, + { + "id": "q-46f2f7edeb", + "prompt": "formatNodeList edgeLabel GraphTraverser getCallers getCallees", + "maxFiles": 2 + }, + { + "id": "q-104036dbc2", + "prompt": "scripts/build-kernel.mjs scripts/ensure-kernel.mjs runtime-control.ts checkRuntimeReady protocol version runtime identity build stamp" + }, + { + "id": "q-170426f071", + "prompt": "src/mcp/daemon-registry.ts getRegistryDir src/directory.ts getCodeGraphStateDir registry dir global config", + "maxFiles": 3 + }, + { + "id": "q-7abd239dc0", + "prompt": "src/mcp/version.ts resolveManagedBuildVersion managed source stamp file", + "maxFiles": 1 + }, + { + "id": "q-01ba61171b", + "prompt": "src/codegraph.ts codegraph-kernel/src/lib.rs src/db/sqlite-adapter.ts src/mcp/server.ts runtime architecture Rust kernel Node transport public library API ui browser", + "maxFiles": 5 + }, + { + "id": "q-5d83a6b73c", + "prompt": "codegraph-kernel/src/lib.rs codegraph-kernel/src/resolve/name_matcher_upstream.rs native Rust #[cfg(test)] tests: identify a pure helper with direct Rust unit tests usable by cargo test; contrast __tests__/kernel-scaffold.test.ts and scripts/ensure-kernel.mjs NAPI prebuild test wiring", + "maxFiles": 4 + }, + { + "id": "q-0338db736f", + "prompt": "codegraph-kernel/src/ids.rs codegraph-kernel/src/docstring.rs codegraph-kernel/src/lib.rs cfg(test) NAPI test stubs", + "maxFiles": 3 + }, + { + "id": "q-d19a67e53a", + "prompt": "codegraph-kernel/src/util.rs col16 utf16 tests", + "maxFiles": 3 + }, + { + "id": "q-5ed5e923e9", + "prompt": "Cols col utf16_prefix codegraph-kernel/src/textutil.rs callers parse_tree_inner", + "maxFiles": 2 + }, + { + "id": "q-9a8cf9697a", + "prompt": "__tests__/kernel-parse-tree.test.ts UTF-16 unicode astral", + "maxFiles": 1 + }, + { + "id": "q-c3e864cffb", + "prompt": "src/extraction/kernel/tree.ts NativeNode startIndex endIndex startPosition descendantForIndex descendantForPosition", + "maxFiles": 1 + }, + { + "id": "q-c91e32a1b3", + "prompt": "codegraph-kernel/src/tree.rs parse_tree_inner Row start_byte start_idx utf16_prefix src/extraction/parse-tree.ts TreeNode", + "maxFiles": 2 + }, + { + "id": "q-daf8ef7fb3", + "prompt": "src/graph/branch-guards.ts innermostAt __tests__/kernel-parse-tree.test.ts firstViolation", + "maxFiles": 2 + }, + { + "id": "q-368e0f6b10", + "prompt": "createDatabase SqliteDatabase src/db/schema.sql FTS5", + "maxFiles": 2 + } + ] +}