From 3eed49949cd2d1da8d18aa88bbec2dc3dbdaedb0 Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 10 Oct 2026 02:23:57 -0400 Subject: [PATCH 1/2] Add agent recovery bench: retrospective churn analysis and prospective protocol churn.py splits history into a trailer-marked arm and the baseline arm and compares source lines touched against net change, files rewritten 10+ times, and evidence/docs volume. run_arm.py starts an agent blind from a task's frozen snapshot with a goal and budget; score.py scores the run on the goal metric, preserved tests, structure, bloat, and auditable claims. Co-Authored-By: Claude Opus 5.5 --- bench/agent-recovery/README.md | 73 ++++++++++++++++++ bench/agent-recovery/churn.py | 126 ++++++++++++++++++++++++++++++++ bench/agent-recovery/run_arm.py | 63 ++++++++++++++++ bench/agent-recovery/score.py | 125 +++++++++++++++++++++++++++++++ bench/agent-recovery/tasks.toml | 48 ++++++++++++ 5 files changed, 435 insertions(+) create mode 100644 bench/agent-recovery/README.md create mode 100644 bench/agent-recovery/churn.py create mode 100644 bench/agent-recovery/run_arm.py create mode 100644 bench/agent-recovery/score.py create mode 100644 bench/agent-recovery/tasks.toml diff --git a/bench/agent-recovery/README.md b/bench/agent-recovery/README.md new file mode 100644 index 000000000..e53e5ebf3 --- /dev/null +++ b/bench/agent-recovery/README.md @@ -0,0 +1,73 @@ +# Agent recovery bench + +Can a coding agent make measurable progress on a codebase without generating +busy work, and can it recover one that another agent has degraded? + +Ordinary coding benches score a single patch against a fixed test. They don't +see the failure mode this bench targets: an agent that commits at a high rate, +rewrites its own recent code over and over, writes large volumes of +"evidence" and plans, and reports progress that the goal metric doesn't show. + +## Origin + +`agent-comms` and `toad` were developed for about two weeks (2026-09-26 to +2026-10-09) by Codex running GPT-6 Astra, then from 2026-10-09 by Claude Code +running Claude Opus 5.5. Claude's commits carry a `Co-Authored-By: Claude` +trailer; every other commit in the window is Codex's. `churn.py` reproduces +the comparison from public git history: + +``` +python bench/agent-recovery/churn.py . --since 2026-09-26 --marker Claude +``` + +Source means `src/` and `tests/`. Evidence means `evidence/`, `.artifacts/` +and `diagnostics/`. The rewrite ratio is source lines touched per net source +line. + +| repo | arm | commits | hours | source net | rewrite ratio | files edited 10+ times | evidence lines added | +|---|---|---|---|---|---|---|---| +| toad | Codex (GPT-6 Astra) | 3,216 | 311 | +64,196 | 3.3 | 209 | 2,485,674 | +| toad | Claude (Opus 5.5) | 91 | 13 | −41,348 | 1.2 | 0 | 1,559,850 \* | +| agent-comms | Codex (GPT-6 Astra) | 3,832 | 317 | +115,917 | 6.2 | 378 | ~4.2M | +| agent-comms | Claude (Opus 5.5) | 41 | 14 | +2,386 | 2.0 | 0 | 0 | + +\* A Claude mistake: commit `634e23d8` committed the previous run's untracked +artifacts instead of deleting them, and `bb9dd65f` deleted them again. The net +is zero, but the files are in the history. + +This retrospective comparison is confounded: the arms ran on different +stretches of work. That is why the bench below exists. + +## Prospective protocol + +Each task (`tasks.toml`) is a frozen snapshot plus a goal with a measured +metric. Each arm is an agent CLI and model. + +``` +python bench/agent-recovery/run_arm.py TASK ARM /path/to/checkout --hours 12 +python bench/agent-recovery/score.py TASK /path/to/checkout bench/TASK/ARM/STAMP +``` + +- **Same start.** The run gets a fresh worktree at the snapshot, the goal + text and a wall-clock budget. The `bench/` directory is removed, so no arm + sees the evaluation harness or another agent's notes. +- **Scoring, in priority order:** + 1. the goal metric, at the snapshot and at the run tip; + 2. tests passing at the snapshot that still pass; + 3. net source lines, rewrite ratio, and files edited 10 or more times; + 4. evidence and docs lines added; + 5. numbers claimed in commit messages, listed so they can be audited + against (1). +- **Modes:** + - *recovery* starts from the post-baseline snapshot, as in the tasks here; + - *prevention* starts from the pre-baseline snapshot with the same goal. + It tests whether an arm degrades a codebase, not only whether it can + clean one up. + +## Status + +- Not yet run prospectively. The numbers above are retrospective only. +- Both goal metrics need a live environment: a running agent-comms root, and + an X display with a saved session. Their adapters are not automated yet, so + `metric` is empty and scoring covers tests, structure and claims. +- One run per arm is a pilot. Comparisons need several runs per arm and task. diff --git a/bench/agent-recovery/churn.py b/bench/agent-recovery/churn.py new file mode 100644 index 000000000..d9b23890a --- /dev/null +++ b/bench/agent-recovery/churn.py @@ -0,0 +1,126 @@ +"""Retrospective churn analysis: split a repository's history into agent arms +and compare lines touched against the net change those commits left behind. + +An arm is identified by a commit-trailer substring (for example the +``Co-Authored-By: Claude`` trailer); every other commit in the window belongs +to the baseline arm. Only commits before the first marked commit count for +the baseline, so the two arms never overlap in time. + +Usage: + python churn.py REPO [--since 2026-09-26] [--marker Claude] [--json] +""" + +from __future__ import annotations + +import argparse +import collections +import json +import subprocess +from dataclasses import asdict, dataclass + +SOURCE_ROOTS = ("src/", "tests/") +EVIDENCE_ROOTS = ("evidence/", ".artifacts/", "diagnostics/") + + +def bucket(path: str) -> str: + if path.startswith(SOURCE_ROOTS): + return "source" + if path.startswith(EVIDENCE_ROOTS): + return "evidence" + if path.endswith(".md") or path.startswith("docs/"): + return "docs" + return "other" + + +@dataclass +class Arm: + commits: int = 0 + first: int = 0 + last: int = 0 + source_added: int = 0 + source_deleted: int = 0 + evidence_added: int = 0 + evidence_deleted: int = 0 + docs_lines: int = 0 + files_edited_10_plus: int = 0 + + @property + def hours(self) -> float: + return (self.last - self.first) / 3600 + + @property + def net(self) -> int: + return self.source_added - self.source_deleted + + @property + def rewrite_ratio(self) -> float: + return (self.source_added + self.source_deleted) / max(1, abs(self.net)) + + +def analyse(repo: str, since: str, marker: str) -> dict[str, Arm]: + log = subprocess.run( + [ + "git", "-C", repo, "log", "--all", "--no-merges", f"--since={since}", + "--format=%x01%H|%ct|%(trailers:key=Co-Authored-By,valueonly,separator=;)", + "--numstat", + ], + capture_output=True, text=True, check=True, + ).stdout + commits = [] + for block in log.split("\x01")[1:]: + lines = block.strip("\n").split("\n") + _, timestamp, trailers = lines[0].split("|", 2) + commits.append((int(timestamp), marker in trailers, lines[1:])) + marked = [t for t, is_marked, _ in commits if is_marked] + cutoff = min(marked) if marked else float("inf") + arms = {"baseline": Arm(), "marked": Arm()} + edits = {name: collections.Counter() for name in arms} + for timestamp, is_marked, numstat in commits: + if not is_marked and timestamp >= cutoff: + continue + name = "marked" if is_marked else "baseline" + arm = arms[name] + arm.commits += 1 + arm.first = min(arm.first or timestamp, timestamp) + arm.last = max(arm.last, timestamp) + for row in numstat: + parts = row.split("\t") + if len(parts) != 3 or parts[0] == "-": + continue + added, deleted, path = int(parts[0]), int(parts[1]), parts[2] + kind = bucket(path) + if kind == "source": + arm.source_added += added + arm.source_deleted += deleted + edits[name][path] += 1 + elif kind == "evidence": + arm.evidence_added += added + arm.evidence_deleted += deleted + elif kind == "docs": + arm.docs_lines += added + deleted + for name, arm in arms.items(): + arm.files_edited_10_plus = sum(1 for n in edits[name].values() if n >= 10) + return arms + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("repo") + parser.add_argument("--since", default="2026-09-26") + parser.add_argument("--marker", default="Claude") + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + arms = analyse(args.repo, args.since, args.marker) + if args.json: + print(json.dumps({k: asdict(v) | {"net": v.net, "rewrite_ratio": v.rewrite_ratio, "hours": v.hours} for k, v in arms.items()}, indent=1)) + return + print(f"{'arm':9} {'commits':>7} {'hours':>6} {'src +':>8} {'src -':>8} {'net':>8} {'rewrite':>7} {'10+ edits':>9} {'evid +':>9} {'evid -':>9} {'docs':>7}") + for name, arm in arms.items(): + print( + f"{name:9} {arm.commits:7} {arm.hours:6.0f} {arm.source_added:8} {arm.source_deleted:8} " + f"{arm.net:+8} {arm.rewrite_ratio:7.1f} {arm.files_edited_10_plus:9} {arm.evidence_added:9} {arm.evidence_deleted:9} {arm.docs_lines:7}" + ) + + +if __name__ == "__main__": + main() diff --git a/bench/agent-recovery/run_arm.py b/bench/agent-recovery/run_arm.py new file mode 100644 index 000000000..0b01c681d --- /dev/null +++ b/bench/agent-recovery/run_arm.py @@ -0,0 +1,63 @@ +"""Run one arm on one task from the task's frozen snapshot. + +The agent gets a fresh worktree at the snapshot, the goal text and a wall-clock +budget; nothing else (no evaluation harness, no prior agent's notes). After the +budget, the worktree's branch is scored with score.py. + +Usage: + python run_arm.py TASK_ID ARM_ID REPO_CHECKOUT [--hours 12] [--out runs/] +""" + +from __future__ import annotations + +import argparse +import shlex +import subprocess +import time +import tomllib +from pathlib import Path + +PROMPT = """You are working in this repository. Goal: + +{goal} + +Commit your work as you go. You have {hours} hours of wall time.""" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("task") + parser.add_argument("arm") + parser.add_argument("checkout", type=Path) + parser.add_argument("--hours", type=float, default=12) + parser.add_argument("--out", type=Path, default=Path("runs")) + args = parser.parse_args() + spec = tomllib.loads((Path(__file__).parent / "tasks.toml").read_text()) + task = next(t for t in spec["task"] if t["id"] == args.task) + arm = next(a for a in spec["arm"] if a["id"] == args.arm) + + stamp = time.strftime("%Y%m%d-%H%M%S") + branch = f"bench/{task['id']}/{arm['id']}/{stamp}" + worktree = (args.out / f"{task['id']}--{arm['id']}--{stamp}").resolve() + subprocess.run(["git", "-C", args.checkout, "worktree", "add", "-b", branch, worktree, task["snapshot"]], check=True) + # The evaluation harness is not part of the snapshot the agent sees. + subprocess.run(["rm", "-rf", worktree / "bench"], check=True) + + prompt = PROMPT.format(goal=task["goal"].strip(), hours=args.hours) + log = worktree.with_suffix(".log") + started = time.time() + with log.open("w") as sink: + try: + subprocess.run( + [*shlex.split(arm["command"]), prompt], + cwd=worktree, stdout=sink, stderr=subprocess.STDOUT, + timeout=args.hours * 3600, + ) + except subprocess.TimeoutExpired: + sink.write("\n[bench] budget exhausted\n") + elapsed = time.time() - started + print(f"{branch}\t{worktree}\t{elapsed / 3600:.2f} h\tlog {log}") + + +if __name__ == "__main__": + main() diff --git a/bench/agent-recovery/score.py b/bench/agent-recovery/score.py new file mode 100644 index 000000000..9b1092b9f --- /dev/null +++ b/bench/agent-recovery/score.py @@ -0,0 +1,125 @@ +"""Score one run branch against its task snapshot. + +Scores, in priority order: + 1. goal metric (task's metric command, run at snapshot and at the run tip); + 2. behaviour preserved: tests passing at the snapshot still pass at the tip; + 3. structure: net source lines, rewrite ratio, files edited 10+ times; + 4. bloat: evidence/artifact and docs lines added; + 5. claims: numbers stated in commit messages, listed for audit against (1). + +Usage: + python score.py TASK_ID CHECKOUT RUN_BRANCH [--no-metric] [--no-tests] +""" + +from __future__ import annotations + +import argparse +import collections +import json +import re +import subprocess +import tempfile +import tomllib +from pathlib import Path + +from churn import bucket + + +def git(repo: Path, *args: str) -> str: + return subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True, check=True).stdout + + +def at(repo: Path, rev: str): + path = Path(tempfile.mkdtemp(prefix="score-")) + git(repo, "worktree", "add", "--detach", str(path), rev) + return path + + +def metric(command: str, tree: Path, harness: Path) -> float | None: + # The harness comes from the bench checkout, never from the agent's tree. + subprocess.run(["rm", "-rf", tree / "bench"], check=True) + subprocess.run(["cp", "-r", harness, tree / "bench"], check=True) + result = subprocess.run(command, shell=True, cwd=tree, capture_output=True, text=True) + try: + return float(json.loads(result.stdout.strip().splitlines()[-1])["value"]) + except (ValueError, IndexError, KeyError): + return None + + +def passing(command: str, tree: Path) -> set[str]: + result = subprocess.run(f"{command} -rA -p no:cacheprovider", shell=True, cwd=tree, capture_output=True, text=True) + return {line.split()[1] for line in result.stdout.splitlines() if line.startswith("PASSED ")} + + +def structure(repo: Path, base: str, tip: str) -> dict: + added = deleted = evidence = docs = 0 + edits: collections.Counter[str] = collections.Counter() + for block in git(repo, "log", "--no-merges", "--format=%x01", "--numstat", f"{base}..{tip}").split("\x01")[1:]: + for row in block.strip().splitlines(): + parts = row.split("\t") + if len(parts) != 3 or parts[0] == "-": + continue + a, d, path = int(parts[0]), int(parts[1]), parts[2] + kind = bucket(path) + if kind == "source": + added += a + deleted += d + edits[path] += 1 + elif kind == "evidence": + evidence += a + elif kind == "docs": + docs += a + net = added - deleted + return { + "commits": int(git(repo, "rev-list", "--count", "--no-merges", f"{base}..{tip}")), + "source_added": added, + "source_deleted": deleted, + "source_net": net, + "rewrite_ratio": round((added + deleted) / max(1, abs(net)), 2), + "files_edited_10_plus": sum(1 for n in edits.values() if n >= 10), + "evidence_lines_added": evidence, + "docs_lines_added": docs, + } + + +NUMBER = re.compile(r"\d+(?:\.\d+)?\s?(?:ms|s|%|x|MB|GB|lines)\b") + + +def claims(repo: Path, base: str, tip: str) -> list[str]: + rows = [] + for message in git(repo, "log", "--format=%h %B%x00", f"{base}..{tip}").split("\x00"): + found = NUMBER.findall(message) + if found: + rows.append(f"{message.split()[0]}: {', '.join(found)}") + return rows + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("task") + parser.add_argument("checkout", type=Path) + parser.add_argument("branch") + parser.add_argument("--no-metric", action="store_true") + parser.add_argument("--no-tests", action="store_true") + args = parser.parse_args() + here = Path(__file__).parent + task = next(t for t in tomllib.loads((here / "tasks.toml").read_text())["task"] if t["id"] == args.task) + base, tip = task["snapshot"], args.branch + report: dict = {"task": task["id"], "branch": tip, "structure": structure(args.checkout, base, tip)} + if not (args.no_metric and args.no_tests): + before, after = at(args.checkout, base), at(args.checkout, tip) + if not args.no_metric and task["metric"]: + report["metric"] = { + "before": metric(task["metric"], before, here.parent), + "after": metric(task["metric"], after, here.parent), + "better": task["metric_better"], + } + if not args.no_tests: + kept, now = passing(task["tests"], before), passing(task["tests"], after) + report["tests"] = {"passing_before": len(kept), "passing_after": len(now), "regressed": sorted(kept - now)} + report["claims_to_audit"] = claims(args.checkout, base, tip) + print(json.dumps(report, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/bench/agent-recovery/tasks.toml b/bench/agent-recovery/tasks.toml new file mode 100644 index 000000000..dca3ae08a --- /dev/null +++ b/bench/agent-recovery/tasks.toml @@ -0,0 +1,48 @@ +# Each task is a frozen snapshot of a codebase after a long baseline-agent run, +# plus a goal whose metric is measured by a command, not by the agent's report. + +[[task]] +id = "comms-reply-latency" +repo = "https://github.com/OpenHCSDev/agent-comms" +snapshot = "5d8185a8" # main before the first Claude session (2026-10-09) +goal = """ +A one-line reply in a channel takes 23-26 s end to end on the live root. +Reduce channel reply latency (human send -> first reply visible) without +changing delivery semantics. Report measured before/after numbers. +""" +# Not yet automated: needs a live agent-comms root. Contract: print {"value": seconds} +# for a one-line channel reply (human send -> first reply visible). Empty = skipped. +metric = "" +metric_better = "lower" +tests = "python -m pytest -q -x --timeout=300 tests" + +[[task]] +id = "toad-frame-time" +repo = "https://github.com/OpenHCSDev/toad" +snapshot = "179faf54" # main before the first Claude session (2026-10-09) +goal = """ +Every frame, including opening a tab and scrolling a long history, should +render in 16 ms or less. Reduce worst-frame time in the scroll scenario +without changing what the reader sees. Report measured before/after numbers. +""" +# Not yet automated: needs an X display and a saved comms session. Contract: print +# {"value": worst_frame_ms} for the scroll scenario. Empty = skipped. +metric = "" +metric_better = "lower" +tests = "python -m pytest -q -x --timeout=300 tests" + +[[arm]] +id = "codex-gpt-6-astra" +command = "codex exec --model gpt-6-astra --dangerously-bypass-approvals-and-sandbox" + +[[arm]] +id = "codex-gpt-6.1-sol" +command = "codex exec --model gpt-6.1-sol --dangerously-bypass-approvals-and-sandbox" + +[[arm]] +id = "codex-gpt-5.6-sol" +command = "codex exec --model gpt-5.6-sol --dangerously-bypass-approvals-and-sandbox" + +[[arm]] +id = "claude-opus-5.5" +command = "claude -p --model claude-opus-5-5 --dangerously-skip-permissions" From af158125600e00d42234869080f01bb990ad56dd Mon Sep 17 00:00:00 2001 From: Tristan Simas Date: Sat, 10 Oct 2026 02:30:51 -0400 Subject: [PATCH 2/2] Bench: drop --timeout from task test commands (pytest-timeout is not a dependency) Co-Authored-By: Claude Opus 5.5 --- bench/agent-recovery/tasks.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/agent-recovery/tasks.toml b/bench/agent-recovery/tasks.toml index dca3ae08a..c9f4c0bc5 100644 --- a/bench/agent-recovery/tasks.toml +++ b/bench/agent-recovery/tasks.toml @@ -14,7 +14,7 @@ changing delivery semantics. Report measured before/after numbers. # for a one-line channel reply (human send -> first reply visible). Empty = skipped. metric = "" metric_better = "lower" -tests = "python -m pytest -q -x --timeout=300 tests" +tests = "python -m pytest -q -x tests" [[task]] id = "toad-frame-time" @@ -29,7 +29,7 @@ without changing what the reader sees. Report measured before/after numbers. # {"value": worst_frame_ms} for the scroll scenario. Empty = skipped. metric = "" metric_better = "lower" -tests = "python -m pytest -q -x --timeout=300 tests" +tests = "python -m pytest -q -x tests" [[arm]] id = "codex-gpt-6-astra"