eval: gate retrieval changes by replaying saved agent queries

AGENTS.md

@@ -94,6 +94,8 @@ Defined in `src/types.ts`. Both extractors and resolvers must use these exact st
 
 **Required** for every new language/framework: validate on small/medium/large real repos with >=3 flow prompts; deterministic `scripts/agent-eval/probe-explore.mjs` probes, then agent A/B (`scripts/agent-eval/run-all.sh` / `ab-new-vs-baseline.sh`). Pass bar: ~0 Read/Grep within the explore-call budget, faster than without-codegraph, no control-repo regression.
 
+**Every other retrieval change** (explore ranking, budgets, rendering, search) is gated by replaying saved agent queries, not by an agent A/B: `node scripts/agent-eval/replay-queries.mjs <checkout at the corpus commit> --tasks scripts/agent-eval/replay/<corpus>.json --base <build> --cand <build> --out <dir>` (about 10 s on 46 queries). Read every changed or regressed answer and say in the PR why each is acceptable. Run one agent A/B per release batch, before a build is promoted or released, and add its new query shapes to the corpus with `harvest-queries.mjs`. Details: `docs/AGENTS.md` § Query replay.
+
 Full methodology (feedback metrics, CLI contamination guard, Sonnet/`--effort high` model policy, daemon pre-warm), the Excalidraw worked example, and coverage matrix live in:
 - `docs/AGENTS.md` (nested; also loaded when cwd is under `docs/`)
 - `docs/design/dynamic-dispatch-coverage-playbook.md`

__tests__/query-replay.test.ts

@@ -0,0 +1,85 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { describe, expect, it } from 'vitest';
+// @ts-expect-error untyped ESM script
+import { callsInLine, repoSlug } from '../scripts/agent-eval/harvest-queries.mjs';
+// @ts-expect-error untyped ESM script
+import { compareRow } from '../scripts/agent-eval/replay-queries.mjs';
+
+const row = (files: string[], over: Record<string, unknown> = {}) => ({
+  id: 'q', chars: 100, sourceAt: 10, files, mustMatchAt: [], ...over,
+});
+
+describe('replay comparison', () => {
+  it('passes the same files in another order', () => {
+    const r = compareRow(row(['a.ts', 'b.ts']), row(['b.ts', 'a.ts']));
+    expect(r.regressed).toBe(false);
+    expect(r.changed).toBe(false);
+  });
+
+  it('flags a lost file, and names it when the query asked for it', () => {
+    const r = compareRow(row(['a.ts', 'src/b.ts']), row(['a.ts']), 'explain src/b.ts');
+    expect(r.regressed).toBe(true);
+    expect(r.lostFiles).toEqual(['src/b.ts']);
+    expect(r.lostNamed).toEqual(['src/b.ts']);
+  });
+
+  it('flags a lost source section and a pattern the base found', () => {
+    expect(compareRow(row(['a.ts']), row([], { sourceAt: -1 })).lostSource).toBe(true);
+    expect(compareRow(row([], { mustMatchAt: [5] }), row([], { mustMatchAt: [-1] })).lostMatch).toBe(1);
+  });
+
+  it('counts different text with the same files and length as a change', () => {
+    const r = compareRow(row(['a.ts']), row(['a.ts']), '', false);
+    expect(r).toMatchObject({ regressed: false, changed: true });
+  });
+
+  it('counts a gained file as a change without loss', () => {
+    const r = compareRow(row(['a.ts']), row(['a.ts', 'c.ts']));
+    expect(r).toMatchObject({ regressed: false, changed: true, gainedFiles: ['c.ts'] });
+  });
+});
+
+describe('query harvest', () => {
+  it('reads Claude tool_use, Codex function calls and Codex code-mode calls', () => {
+    expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'mcp__codegraph__codegraph_explore', input: { query: 'A b', projectPath: '/x' } }] } }))
+      .toEqual([{ query: 'A b', projectPath: '/x' }]);
+    expect(callsInLine({ payload: { type: 'function_call', name: 'mcp__codegraph__codegraph_explore', arguments: '{"query":"C"}' } }))
+      .toEqual([{ query: 'C', projectPath: undefined }]);
+    const input = 'text(await tools.mcp__codegraph__codegraph_explore({projectPath:"/y",query:"D \\"e\\"",maxFiles:8}));';
+    expect(callsInLine({ payload: { type: 'custom_tool_call', input } })).toEqual([{ query: 'D "e"', projectPath: '/y', maxFiles: 8 }]);
+    expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'Read', input: { query: 'no' } }] } })).toEqual([]);
+  });
+
+  it('reads code-mode calls with braces, single quotes and file limits', () => {
+    const call = (input: string) => callsInLine({ payload: { type: 'custom_tool_call', input } });
+    expect(call('codegraph_explore({query:"make({a: 1}) } brace"})')).toEqual([{ query: 'make({a: 1}) } brace', projectPath: undefined }]);
+    expect(call("codegraph_explore({ query: 'auth flow', maxFiles: 12 })")).toEqual([{ query: 'auth flow', projectPath: undefined, maxFiles: 12 }]);
+    expect(call('codegraph_explore({query:`a ${x}`})')).toEqual([]);
+    expect(callsInLine({ message: { content: [{ type: 'tool_use', name: 'mcp__codegraph__codegraph_explore', input: { query: 'Q', maxFiles: 3 } }] } }))
+      .toEqual([{ query: 'Q', projectPath: undefined, maxFiles: 3 }]);
+  });
+
+  it('maps GitHub remotes to owner/repo', () => {
+    expect(repoSlug('git@github.com:Owner/Repo.git\n')).toBe('owner/repo');
+    expect(repoSlug('https://github.com/owner/repo')).toBe('owner/repo');
+    expect(repoSlug('ssh://git@github.com/owner/repo.git')).toBe('owner/repo');
+    expect(repoSlug('https://gitlab.com/owner/repo.git')).toBeNull();
+    expect(repoSlug('https://notgithub.com/owner/repo.git')).toBeNull();
+    expect(repoSlug('git@privategithub.com:owner/repo.git')).toBeNull();
+  });
+});
+
+describe('committed replay corpora', () => {
+  const dir = resolve(import.meta.dirname, '../scripts/agent-eval/replay');
+  for (const name of readdirSync(dir).filter((f) => f.endsWith('.json'))) {
+    it(`${name} is pinned, unique and has no local paths`, () => {
+      const text = readFileSync(join(dir, name), 'utf8');
+      const corpus = JSON.parse(text);
+      expect(corpus.commit).toMatch(/^[0-9a-f]{7,40}$/);
+      const ids = corpus.tasks.map((t: { id: string }) => t.id);
+      expect(new Set(ids).size).toBe(ids.length);
+      expect(text).not.toMatch(/\/home\/|\/Users\/|[A-Z]:\\\\|~\//);
+    });
+  }
+});

docs/AGENTS.md

@@ -10,7 +10,7 @@ For each **language × framework**, validate on **small, medium, and large** rea
 
 1. **Pick the canonical flow** for the framework ("how does X reach Y": state→render, request→handler→view, query→SQL, action→reducer→store…).
 2. **Deterministic probes** (`scripts/agent-eval/probe-{node,explore}.mjs` against the built `dist/`): `codegraph_explore` with the flow's symbol names connects from→to end-to-end with no break (its Flow section shows the path); **no node explosion** (`select count(*) from nodes` stable before/after re-index); synthesized-edge **precision** spot-check (`select … where provenance='heuristic'`).
-3. **Agent A/B** (`scripts/agent-eval/run-all.sh <repo> "<Q>"`): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`); export `CODEGRAPH_REPO_ROOT` as this checkout's absolute path before launching Claude.
+3. **Agent A/B** (`scripts/agent-eval/run-all.sh <repo> "<Q>"`), for a new language or framework and once per release batch (a change to existing retrieval uses the query replay below instead): with vs without codegraph, **≥2 runs/arm** (run-to-run variance is large — never conclude from n=1). Record **duration, total tool calls, Read, Grep**. Optional forced-Read-0 sufficiency proof via the block-read hook (`scripts/agent-eval/hook-settings.json`); export `CODEGRAPH_REPO_ROOT` as this checkout's absolute path before launching Claude.
    - **Every run also reports three feedback metrics** — residual context occupancy, explore sufficiency (what the agent did NEXT after each explore), and allocation efficiency (share of returned bytes the answer cited) — under each run, plus a side-by-side arm table (`compare-arms.mjs`). Entry point: `docs/benchmarks/agent-eval-feedback-metrics.md`. Reading them: `Read a file we returned` is an allocation miss, `Read a file we did NOT return`/`Grep` is recall; allocation efficiency is **relative** (attribution is by citation) so it is only valid between builds on the same question; occupancy *shares* are Claude Code / 200k and don't transfer to another host — the arm ratio does.
    - **The `codegraph` CLI is blocked in every arm** (`no-cli-shim.sh`: sanitized PATH + a PreToolUse hook, shared by both harnesses). Without it 14 of 15 without-arm runs in one 7-repo pass reached codegraph through Bash. Check the contamination row before believing any number: `CLI calls that RETURNED output` > 0 invalidates the run (in a new-vs-baseline A/B it silently drops calls from all three metrics, since a CLI explore is not a tool call).
    - **Model selection and availability.** The Claude harness defaults to `--model sonnet --effort high` for floor-model validation. If that model is unavailable through quota, an outage or an unsupported route, use the next available eligible model and effort from the operator's current benchmark-backed bug-finding order. Primary, secondary and tertiary choices change with benchmark results; read the current selection rather than fixing fallback names here. Record the selected model, effort, ranking source and fallback reason.
@@ -21,6 +21,17 @@ For each **language × framework**, validate on **small, medium, and large** rea
 
 Full playbook + per-mechanism design: `docs/design/dynamic-dispatch-coverage-playbook.md` and `docs/design/callback-edge-synthesis.md`.
 
+### Query replay (gate for changes to existing retrieval)
+
+A change to explore ranking, budgets, rendering or search replays real agent queries against the base and candidate builds instead of running an agent A/B. The past change-gating A/Bs mostly agreed with the deterministic probes; the defects only agents caught came from query shapes the probes lacked, and replaying those agents' queries reproduced each one. The corpus keeps those shapes.
+
+1. Check out the corpus repository at the corpus's `commit` and index it with either build (`codegraph init -y`). The replay refuses a checkout at another commit.
+2. `node scripts/agent-eval/replay-queries.mjs <checkout> --tasks scripts/agent-eval/replay/<corpus>.json --base <base build> --cand <candidate build> --out <dir>` (each build is a checkout with `dist/` built). It prints a table of answers whose text differs and exits 1 when an answer lost its source section, a file the base rendered (marked when the query names the file) or a `mustMatch` pattern. Answers are saved under `<dir>/base` and `<dir>/cand`.
+3. Read every changed and regressed answer. A lost file can be a fix (a peripheral file swapped for a better one); say so per query in the PR. Same-build replay is identical, so any change comes from the candidate.
+4. Once per release batch, before a build is promoted or released, run one agent A/B (`ab-new-vs-baseline.sh`) on a flow question the corpus covers least, then harvest its queries: `node scripts/agent-eval/harvest-queries.mjs --public <owner/repo> <transcript dir>...`. It keeps queries only on repositories named with `--public` and writes no paths; still read every prompt before committing, since a public query can quote a private name. `__tests__/query-replay.test.ts` fails a committed corpus that contains a local path.
+
+Measured on `replay/codegraph.json` (46 queries, this repository at `2a4af7c3`; 29 replay the agent's own `maxFiles`): same build 46 of 46 identical in 9 s; a build whose default limit is two files flagged 16 of the 17 queries that use the default, 4 of them for a file the query named; the root `__tests__/` test-path fix (#500) changed 6 answers and flagged none, one of them with the same length and different text.
+
 ### Worked example — Excalidraw (TS/React, medium, 643 files)
 
 This table records the historical trace-era evaluation; current validation uses

scripts/agent-eval/harvest-queries.mjs

@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+// Collect the codegraph_explore queries real agent sessions asked, for replay
+// with replay-queries.mjs.
+//
+//   node scripts/agent-eval/harvest-queries.mjs --public <owner/repo> [--public ...] [--since 2026-09-01] <dir|file.jsonl>...
+//
+// Reads Claude Code transcripts (assistant tool_use blocks) and Codex rollouts
+// (MCP tool calls and code-mode `tools.mcp__codegraph__codegraph_explore({...})`
+// calls). Each call's project is its projectPath, else the session's working
+// directory, mapped to a repository through `git remote get-url origin`. Only
+// queries on a repository named by --public are kept; everything else is
+// dropped and counted, so a private project never reaches the output. Output
+// on stdout: `{ repos: { "<owner/repo>": { tasks: [{ id, prompt, maxFiles? }] } } }`,
+// deduplicated on query and file limit, with no paths. Review every prompt before committing it: a
+// query on a public repository can still quote a private name.
+import { execFileSync } from 'node:child_process';
+import { createReadStream, readdirSync, statSync } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+
+/** Explore calls in one transcript line: [{ query, projectPath?, maxFiles? }]. */
+export function callsInLine(entry) {
+  const calls = [];
+  const blocks = entry?.message?.content;
+  if (Array.isArray(blocks)) {
+    for (const b of blocks) {
+      if (b?.type === 'tool_use' && /codegraph_explore$/.test(b.name ?? '') && typeof b.input?.query === 'string') {
+        calls.push({ query: b.input.query, projectPath: b.input.projectPath, maxFiles: b.input.maxFiles });
+      }
+    }
+  }
+  const p = entry?.payload;
+  if (p?.type === 'function_call' && /codegraph_explore$/.test(p.name ?? '')) {
+    try {
+      const args = JSON.parse(p.arguments);
+      if (typeof args.query === 'string') calls.push({ query: args.query, projectPath: args.projectPath, maxFiles: args.maxFiles });
+    } catch { /* not JSON arguments */ }
+  }
+  if (p?.type === 'custom_tool_call' && typeof p.input === 'string') {
+    for (const m of p.input.matchAll(/codegraph_explore\(\s*\{/g)) {
+      const fields = literalFields(p.input, m.index + m[0].length - 1);
+      if (typeof fields.query === 'string') calls.push({ query: fields.query, projectPath: fields.projectPath, maxFiles: fields.maxFiles });
+    }
+  }
+  return calls.map((c) => (typeof c.maxFiles === 'number' ? c : { query: c.query, projectPath: c.projectPath }));
+}
+
+const ESCAPES = { n: '\n', t: '\t', r: '\r', b: '\b', f: '\f', v: '\v', 0: '\0' };
+
+/**
+ * The top-level string and number fields of the JavaScript object literal that
+ * opens at `src[open]` ('{'). Code-mode calls are JavaScript, not JSON: keys may
+ * be bare, strings may use any quote and may contain braces. Values that are
+ * not plain literals (expressions, template strings with `${`) are skipped.
+ */
+export function literalFields(src, open) {
+  const fields = {};
+  let depth = 0, key, expectKey = false, expectValue = false;
+  for (let i = open; i < src.length; i++) {
+    const ch = src[i];
+    if (ch === '"' || ch === "'" || ch === '`') {
+      let s = '', literal = true, j = i + 1;
+      for (; j < src.length && src[j] !== ch; j++) {
+        if (src[j] === '\\') { j++; s += ESCAPES[src[j]] ?? src[j]; }
+        else if (ch === '`' && src[j] === '$' && src[j + 1] === '{') { literal = false; s += src[j]; }
+        else s += src[j];
+      }
+      i = j;
+      if (depth === 1 && expectKey) { key = s; expectKey = false; }
+      else if (depth === 1 && expectValue && key !== undefined && literal) fields[key] = s;
+      expectValue = false;
+    } else if (ch === '{' || ch === '[' || ch === '(') {
+      depth++;
+      if (depth === 1) expectKey = true;
+      expectValue = false;
+    } else if (ch === '}' || ch === ']' || ch === ')') {
+      if (--depth === 0) break;
+    } else if (depth === 1 && ch === ',') {
+      expectKey = true; expectValue = false; key = undefined;
+    } else if (depth === 1 && ch === ':') {
+      expectValue = true;
+    } else if (depth === 1 && /[A-Za-z_$0-9]/.test(ch)) {
+      const word = /^[A-Za-z_$0-9.]+/.exec(src.slice(i))[0];
+      i += word.length - 1;
+      if (expectKey) { key = word; expectKey = false; }
+      else if (expectValue && key !== undefined && /^\d+$/.test(word)) fields[key] = Number(word);
+      expectValue = false;
+    }
+  }
+  return fields;
+}
+
+/** `owner/repo` from a remote on github.com itself (HTTPS, SSH URL or scp form), or null. */
+export function repoSlug(url) {
+  const text = url.trim();
+  const scp = /^git@github\.com:([^/\s]+)\/([^/\s]+?)(?:\.git)?\/?$/.exec(text);
+  if (scp) return `${scp[1]}/${scp[2]}`.toLowerCase();
+  let parsed;
+  try { parsed = new URL(text); } catch { return null; }
+  if (parsed.hostname.toLowerCase() !== 'github.com') return null;
+  const m = /^\/([^/]+)\/([^/]+?)(?:\.git)?\/?$/.exec(parsed.pathname);
+  return m ? `${m[1]}/${m[2]}`.toLowerCase() : null;
+}
+
+function* jsonlFiles(path) {
+  const st = statSync(path, { throwIfNoEntry: false });
+  if (!st) return;
+  if (st.isFile()) { if (path.endsWith('.jsonl')) yield path; return; }
+  for (const name of readdirSync(path)) yield* jsonlFiles(join(path, name));
+}
+
+async function main(argv) {
+  const pub = new Set();
+  const inputs = [];
+  let since = '';
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--public') pub.add(argv[++i].toLowerCase());
+    else if (argv[i] === '--since') since = argv[++i];
+    else inputs.push(argv[i]);
+  }
+  if (!pub.size || !inputs.length) {
+    console.error('usage: harvest-queries.mjs --public <owner/repo> [--public ...] [--since YYYY-MM-DD] <dir|file.jsonl>...');
+    return 2;
+  }
+  const slugOf = new Map();
+  const slugFor = (dir) => {
+    if (!slugOf.has(dir)) {
+      let slug = null;
+      try {
+        slug = repoSlug(execFileSync('git', ['-C', dir, 'remote', 'get-url', 'origin'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }));
+      } catch { /* gone or not a repository */ }
+      slugOf.set(dir, slug);
+    }
+    return slugOf.get(dir);
+  };
+  const repos = {};
+  const seen = new Set();
+  let kept = 0, dropped = 0, duplicate = 0;
+  for (const input of inputs) {
+    for (const file of jsonlFiles(input)) {
+      let cwd;
+      // Line by line: a long transcript can exceed the largest string a runtime allows.
+      for await (const line of createInterface({ input: createReadStream(file, 'utf8'), crlfDelay: Infinity })) {
+        if (!line.includes('codegraph_explore') && !line.includes('"cwd"')) continue;
+        let entry;
+        try { entry = JSON.parse(line); } catch { continue; }
+        cwd = entry.cwd ?? entry.payload?.cwd ?? cwd;
+        if (since && (entry.timestamp ?? '') < since) continue;
+        for (const call of callsInLine(entry)) {
+          const dir = call.projectPath ?? cwd;
+          const slug = dir ? slugFor(dir) : null;
+          if (!slug || !pub.has(slug)) { dropped++; continue; }
+          const prompt = call.query.trim().replace(/\s+/g, ' ');
+          const limit = call.maxFiles === undefined ? '' : `\0${call.maxFiles}`;
+          const key = `${slug}\0${prompt.toLowerCase()}${limit}`;
+          if (seen.has(key)) { duplicate++; continue; }
+          seen.add(key);
+          const id = createHash('sha256').update(prompt + limit).digest('hex').slice(0, 10);
+          const task = { id: `q-${id}`, prompt };
+          if (call.maxFiles !== undefined) task.maxFiles = call.maxFiles;
+          (repos[slug] ??= { tasks: [] }).tasks.push(task);
+          kept++;
+        }
+      }
+    }
+  }
+  console.log(JSON.stringify({ repos }, null, 2));
+  console.error(`kept ${kept}, duplicates ${duplicate}, dropped ${dropped} (not on a --public repository)`);
+  return 0;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) main(process.argv.slice(2)).then((code) => process.exit(code));