From b0a152a9f56c8a4d08059a5569825c31787e2276 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 18:29:45 +0100 Subject: [PATCH 1/2] Antigravity CLI: an adapter for Gemini legs on Google's agy A new cli, antigravity-cli, runs Google's Antigravity CLI (`agy`) headless: `agy -p` with --model, --output-format stream-json and --dangerously-skip-permissions, plus --effort when the leg sets one; resume is `agy -c -p` with the same flags. The run's HOME gets ~/.gemini/antigravity-cli/settings.json with modelProvider "gemini", so agy reads the virtual key from GEMINI_API_KEY and sends every request to GOOGLE_GEMINI_BASE_URL, the proxy's /gemini passthrough. AGY_CLI_DISABLE_AUTO_UPDATE=true keeps the pinned binary. EFFORT_LEVELS gains agy's levels (low, medium, high, xhigh, max), and the cli joins PASSTHROUGH_CLIS, so its key is minted uncapped like gemini-cli's. The gemini-cli adapter is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/benchmark/README.md | 6 +++--- synth_lib/benchmark/campaign.py | 4 +++- synth_lib/benchmark/cli_adapters.py | 32 ++++++++++++++++++++++++++++ synth_lib/benchmark/run_campaign.py | 2 +- tests/benchmark/test_campaign.py | 14 ++++++++++++ tests/benchmark/test_cli_adapters.py | 25 ++++++++++++++++++++++ tests/benchmark/test_run_campaign.py | 10 +++++---- 7 files changed, 84 insertions(+), 9 deletions(-) diff --git a/synth_lib/benchmark/README.md b/synth_lib/benchmark/README.md index e3dd796..2bfdc12 100644 --- a/synth_lib/benchmark/README.md +++ b/synth_lib/benchmark/README.md @@ -1,8 +1,8 @@ # synth_lib.benchmark — an autonomous-agent benchmark harness -This subpackage runs a coding CLI (Claude Code, Codex, Gemini CLI, Kimi Code, …) as an **autonomous -research agent** whose job is to improve a Synth SN50 miner, and then scores what it produced -against the real field of miners on the live subnet. +This subpackage runs a coding CLI (Claude Code, Codex, Gemini CLI, Kimi Code, Antigravity CLI, …) as +an **autonomous research agent** whose job is to improve a Synth SN50 miner, and then scores what it +produced against the real field of miners on the live subnet. One *campaign* runs one or more *legs*, one leg per model, sequentially on the same hardware. A leg is: an agent alone in a Docker sandbox with a frozen price snapshot, a fixed credit budget and a diff --git a/synth_lib/benchmark/campaign.py b/synth_lib/benchmark/campaign.py index e3b7246..ab31a4f 100644 --- a/synth_lib/benchmark/campaign.py +++ b/synth_lib/benchmark/campaign.py @@ -11,12 +11,14 @@ from synth_lib.benchmark.kinds import DEFAULT_KIND, load_kind -KNOWN_CLIS = ("claude-code", "codex", "gemini-cli", "kimi-code", "fake") +KNOWN_CLIS = ("claude-code", "codex", "gemini-cli", "kimi-code", "antigravity-cli", "fake") # The reasoning-effort levels a leg may set, per CLI and by its own names; a CLI absent here has no effort wired. EFFORT_LEVELS = { "claude-code": ("low", "medium", "high", "xhigh", "max"), "codex": ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"), "kimi-code": ("low", "high", "max"), # the levels Kimi's K3 models take + # agy rejects a level the model lacks at launch: Gemini 4 Argon tops out at high. + "antigravity-cli": ("low", "medium", "high", "xhigh", "max"), } # claude-code's workflowSizeGuideline values: the agent count it tells the model to aim for per workflow. WORKFLOW_SIZES = ("unrestricted", "small", "medium", "large") diff --git a/synth_lib/benchmark/cli_adapters.py b/synth_lib/benchmark/cli_adapters.py index 4c47c0a..7277005 100644 --- a/synth_lib/benchmark/cli_adapters.py +++ b/synth_lib/benchmark/cli_adapters.py @@ -176,6 +176,37 @@ def provision_files(self) -> dict[str, str]: } +class AntigravityAdapter(CLIAdapter): + """Google's Antigravity CLI (`agy`) on the Gemini API: the proxy's /gemini passthrough, the virtual key as + GEMINI_API_KEY.""" + + def launch_cmd(self, prompt: str) -> list[str]: + return ["agy", "-p", prompt, *self._flags()] + + def resume_cmd(self, prompt: str) -> list[str] | None: + # -c: continues the most recent conversation in the current directory (the workspace) + return ["agy", "-c", "-p", prompt, *self._flags()] + + def _flags(self) -> list[str]: + # In print mode an unknown --model exits non-zero instead of falling back to another model. + flags = ["--model", self.spec.model, "--output-format", "stream-json", "--dangerously-skip-permissions"] + if self.spec.effort: + flags += ["--effort", self.spec.effort] + return flags + + def env(self) -> dict[str, str]: + # No GOOGLE_API_KEY: the Gemini SDK prefers it over GEMINI_API_KEY. + return { + "GOOGLE_GEMINI_BASE_URL": f"{self.proxy_url}/gemini", + "GEMINI_API_KEY": self.virtual_key, + "AGY_CLI_DISABLE_AUTO_UPDATE": "true", + } + + def provision_files(self) -> dict[str, str]: + # GEMINI_API_KEY is read only with modelProvider "gemini". + return {"~/.gemini/antigravity-cli/settings.json": json.dumps({"modelProvider": "gemini"}) + "\n"} + + class KimiCodeAdapter(CLIAdapter): """Moonshot's Kimi Code CLI (npm `@moonshot-ai/kimi-code`).""" @@ -226,6 +257,7 @@ def launch_cmd(self, prompt: str) -> list[str]: "codex": CodexAdapter, "gemini-cli": GeminiAdapter, "kimi-code": KimiCodeAdapter, + "antigravity-cli": AntigravityAdapter, "fake": FakeAdapter, } diff --git a/synth_lib/benchmark/run_campaign.py b/synth_lib/benchmark/run_campaign.py index 7b390bb..b22efd5 100644 --- a/synth_lib/benchmark/run_campaign.py +++ b/synth_lib/benchmark/run_campaign.py @@ -32,7 +32,7 @@ SANDBOX_NETWORK = "bridge" # CLIs that reach the proxy via a passthrough route (URL forwarded verbatim, not /v1): their keys # are minted uncapped. -PASSTHROUGH_CLIS = {"gemini-cli"} +PASSTHROUGH_CLIS = {"gemini-cli", "antigravity-cli"} def containerize_url(url: str) -> str: diff --git a/tests/benchmark/test_campaign.py b/tests/benchmark/test_campaign.py index c915458..6454c4b 100644 --- a/tests/benchmark/test_campaign.py +++ b/tests/benchmark/test_campaign.py @@ -83,6 +83,7 @@ def test_effort_ultracode_and_goal_load_per_model(tmp_path): "{id: claude, cli: gemini-cli, model: gemini-2.5-pro, effort: high}", # no effort wired for gemini "{id: claude, cli: kimi-code, model: kimi-k3.1-pro, effort: ultra}", # a codex level "{id: claude, cli: kimi-code, model: kimi-k3.1-pro, effort: medium}", # not a K3 level + "{id: claude, cli: antigravity-cli, model: gemini-4-argon, effort: ultra}", # a codex level ], ) def test_effort_outside_the_cli_levels_rejected(tmp_path, model): @@ -99,6 +100,14 @@ def test_kimi_code_effort_loads(tmp_path): assert load_campaign(_write(tmp_path, text)).models[0].effort == "max" +def test_antigravity_cli_effort_loads(tmp_path): + text = VALID_YAML.replace( + "{id: claude, cli: claude-code, model: claude-model}", + "{id: argon, cli: antigravity-cli, model: gemini-4-argon, effort: high}", + ) + assert load_campaign(_write(tmp_path, text)).models[0].effort == "high" + + def test_workflow_size_and_agents_load_with_ultracode(tmp_path): text = VALID_YAML.replace( "{id: claude, cli: claude-code, model: claude-model}", @@ -129,6 +138,11 @@ def test_claude_code_settings_rejected_on_other_clis(tmp_path, flag): bad = VALID_YAML.replace("wire_api: responses}", f"wire_api: responses, {flag}: true}}") with pytest.raises(ValueError, match=flag): load_campaign(_write(tmp_path, bad)) + bad = VALID_YAML.replace( + "cli: gemini-cli, model: gemini/gemini-2.5-pro}", f"cli: antigravity-cli, model: gemini-4-argon, {flag}: true}}" + ) + with pytest.raises(ValueError, match=flag): + load_campaign(_write(tmp_path, bad)) def test_duplicate_model_ids_rejected(tmp_path): diff --git a/tests/benchmark/test_cli_adapters.py b/tests/benchmark/test_cli_adapters.py index 784efb2..87b4906 100644 --- a/tests/benchmark/test_cli_adapters.py +++ b/tests/benchmark/test_cli_adapters.py @@ -155,6 +155,31 @@ def test_kimi_adapter_writes_the_thinking_effort(): assert tomllib.loads(a.provision_files()["~/.kimi-code/config.toml"]) == {"thinking": {"effort": "max"}} +def test_antigravity_adapter_pins_model_and_effort_and_routes_through_the_passthrough(): + a = build_adapter( + ModelSpec(id="g", cli="antigravity-cli", model="gemini-4-argon", effort="high"), + proxy_url="http://localhost:4000", + virtual_key="sk-v", + ) + cmd = a.launch_cmd("go") + assert cmd[:3] == ["agy", "-p", "go"] and "--dangerously-skip-permissions" in cmd + assert cmd[cmd.index("--model") + 1] == "gemini-4-argon" and cmd[cmd.index("--effort") + 1] == "high" + assert cmd[cmd.index("--output-format") + 1] == "stream-json" + resume = a.resume_cmd("again") + assert resume[:4] == ["agy", "-c", "-p", "again"] and resume[4:] == cmd[3:] # same flags on every launch + env = a.env() + assert env["GOOGLE_GEMINI_BASE_URL"] == "http://localhost:4000/gemini" and env["GEMINI_API_KEY"] == "sk-v" + # the SDK would prefer GOOGLE_API_KEY over the virtual key + assert env["AGY_CLI_DISABLE_AUTO_UPDATE"] == "true" and "GOOGLE_API_KEY" not in env + settings = json.loads(a.provision_files()["~/.gemini/antigravity-cli/settings.json"]) + assert settings == {"modelProvider": "gemini"} # without it GEMINI_API_KEY is ignored + + +def test_antigravity_adapter_without_effort_passes_no_flag(): + a = build_adapter(ModelSpec(id="g", cli="antigravity-cli", model="gemini-4-argon"), "http://x", "sk-v") + assert "--effort" not in a.launch_cmd("go") and "--effort" not in a.resume_cmd("go") + + def test_fake_adapter_runs_fake_cli(tmp_path): a = build_adapter(ModelSpec(id="f", cli="fake", model="none"), proxy_url="http://x", virtual_key="sk-v") cmd = a.launch_cmd("go") diff --git a/tests/benchmark/test_run_campaign.py b/tests/benchmark/test_run_campaign.py index bb7744d..b112378 100644 --- a/tests/benchmark/test_run_campaign.py +++ b/tests/benchmark/test_run_campaign.py @@ -140,10 +140,10 @@ def run(self): def test_passthrough_legs_mint_uncapped_keys(tmp_path, monkeypatch): - """gemini-cli reaches the proxy via the passthrough, where LiteLLM's budget reservation leaks - exactly max_budget and any cap self-trips (PROXY_COMPAT "Phantom budget 429"). Its key must be - minted with cap=None; /v1-routed legs keep the real cap. The driver enforces the budget either - way, from cfg.budget_usd_per_model and true spend.""" + """gemini-cli and antigravity-cli reach the proxy via the passthrough, where LiteLLM's budget + reservation leaks exactly max_budget and any cap self-trips (PROXY_COMPAT "Phantom budget 429"). + Their keys must be minted with cap=None; /v1-routed legs keep the real cap. The driver enforces + the budget either way, from cfg.budget_usd_per_model and true spend.""" import synth_lib.benchmark.run_campaign as rc from synth_lib.benchmark.campaign import BaselineSpec, CampaignConfig, ModelSpec @@ -153,6 +153,7 @@ def test_passthrough_legs_mint_uncapped_keys(tmp_path, monkeypatch): models=( ModelSpec(id="c", cli="claude-code", model="claude-model"), ModelSpec(id="g", cli="gemini-cli", model="gemini-3.1-pro-preview"), + ModelSpec(id="a", cli="antigravity-cli", model="gemini-4-argon"), ), budget_usd_per_model=10.0, deadline_hours_per_model=1.0, @@ -186,6 +187,7 @@ def run(self): assert minted["camp-c"] == 10.0 # /v1 leg: real cap assert minted["camp-g"] is None # passthrough leg: uncapped, driver-only enforcement + assert minted["camp-a"] is None def _two_leg_state(root: Path) -> CampaignState: From a76118d93048b2e48b93725e431abcc4f5d17882 Mon Sep 17 00:00:00 2001 From: Thykof Date: Fri, 6 Nov 2026 19:40:56 +0100 Subject: [PATCH 2/2] Antigravity CLI: accept only the effort levels agy's Gemini models take EFFORT_LEVELS['antigravity-cli'] narrows to low, medium and high: agy refuses at launch a level the model lacks, and no Gemini model on this route takes xhigh or max. The comment above the effort check now says the fall-back to a default effort is claude-code's. The fallback-model scan is noted as covering claude-code transcripts only, and the README says the passthrough CLIs (gemini-cli, antigravity-cli) get uncapped keys, whose budget the driver enforces from the key's spend. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kmUsyYSEMPGiwbSuvR9hh --- synth_lib/benchmark/README.md | 12 +++++++----- synth_lib/benchmark/campaign.py | 5 ++--- synth_lib/benchmark/driver.py | 1 + tests/benchmark/test_campaign.py | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/synth_lib/benchmark/README.md b/synth_lib/benchmark/README.md index 2bfdc12..9061d55 100644 --- a/synth_lib/benchmark/README.md +++ b/synth_lib/benchmark/README.md @@ -32,10 +32,12 @@ a grace period the leg is killed and relaunched once with the landing order carr already finished cannot argue with "no such file"). At 100% it is killed. Nominating is final: a leg that stops with `agent/CHAMPION` present is done and forfeits the rest of its envelope. -**Metering.** All model traffic goes through a LiteLLM proxy (`metering/`), one budget-capped -virtual key per leg, so every model is measured with the same instrument and the ledger is the -single source of truth for spend. Probe any new provider or CLI before it carries a real leg (the -deployment repo has the runbook) and keep the answers: the adapters are written from them. +**Metering.** All model traffic goes through a LiteLLM proxy (`metering/`), one virtual key per +leg, so every model is measured with the same instrument and the ledger is the single source of +truth for spend. Each key is budget-capped, except for the passthrough CLIs (gemini-cli, +antigravity-cli): their keys are uncapped, and the driver enforces their budget from the key's +spend. Probe any new provider or CLI before it carries a real leg (the deployment repo has the +runbook) and keep the answers: the adapters are written from them. **Sandbox.** `sandbox/` builds one image with the pinned CLIs; each leg runs in a container with its workspace, a read-only price snapshot at `/workspace/market_data`, a per-run HOME, capped CPU/RAM, @@ -107,7 +109,7 @@ The verdict above is monte-carlo's: a kind registered elsewhere brings its own. | `nomination.py` | the `CHAMPION` contract and the `simulate()` interface probe | | `cli_adapters.py` | per-CLI argv/env, including a `fake` CLI for zero-cost dry runs | | `sandbox/` | `docker run` construction (mounts, caps, network, GPU) | -| `metering/` | the LiteLLM admin client: one budget-capped virtual key per leg | +| `metering/` | the LiteLLM admin client: one virtual key per leg, budget-capped except for the passthrough CLIs | | `verdict/` | prediction generation and scoring of an archived champion | | `generate_predictions.py` | the generation core; copied into champion clones, so numpy/pandas/stdlib only | diff --git a/synth_lib/benchmark/campaign.py b/synth_lib/benchmark/campaign.py index ab31a4f..4c3c724 100644 --- a/synth_lib/benchmark/campaign.py +++ b/synth_lib/benchmark/campaign.py @@ -17,8 +17,7 @@ "claude-code": ("low", "medium", "high", "xhigh", "max"), "codex": ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"), "kimi-code": ("low", "high", "max"), # the levels Kimi's K3 models take - # agy rejects a level the model lacks at launch: Gemini 4 Argon tops out at high. - "antigravity-cli": ("low", "medium", "high", "xhigh", "max"), + "antigravity-cli": ("low", "medium", "high"), # the levels agy's Gemini models take } # claude-code's workflowSizeGuideline values: the agent count it tells the model to aim for per workflow. WORKFLOW_SIZES = ("unrestricted", "small", "medium", "large") @@ -161,7 +160,7 @@ def _validate(cfg: CampaignConfig) -> None: def _validate_model(m: ModelSpec) -> None: if m.cli not in KNOWN_CLIS: raise ValueError(f"unknown cli: {m.cli!r} (expected: {KNOWN_CLIS})") - # An unknown level does not fail the launch: claude warns and runs at its default effort. + # An unknown level would not fail a claude-code launch: claude warns and runs at its default effort. levels = EFFORT_LEVELS.get(m.cli, ()) if m.effort is not None and m.effort not in levels: raise ValueError(f"{m.id}: effort {m.effort!r} is not a {m.cli} level (expected one of: {levels})") diff --git a/synth_lib/benchmark/driver.py b/synth_lib/benchmark/driver.py index 2e70a8b..58c0a27 100644 --- a/synth_lib/benchmark/driver.py +++ b/synth_lib/benchmark/driver.py @@ -48,6 +48,7 @@ # Routing is then sticky for about an hour, so one declined turn can hand the rest of a leg to a # model the campaign never chose — and the transcript is the only place it shows. A leg whose # champion came from a substitute model does not measure the model on the panel. +# The scan covers claude-code transcripts only. FALLBACK_MARKER = b"model_refusal_fallback" # Under artifacts_dir: each Claude Code session's workflow records and subagent transcripts, copied from # the run's HOME, where they are not part of the stream-json transcript. diff --git a/tests/benchmark/test_campaign.py b/tests/benchmark/test_campaign.py index 6454c4b..1eab053 100644 --- a/tests/benchmark/test_campaign.py +++ b/tests/benchmark/test_campaign.py @@ -84,6 +84,8 @@ def test_effort_ultracode_and_goal_load_per_model(tmp_path): "{id: claude, cli: kimi-code, model: kimi-k3.1-pro, effort: ultra}", # a codex level "{id: claude, cli: kimi-code, model: kimi-k3.1-pro, effort: medium}", # not a K3 level "{id: claude, cli: antigravity-cli, model: gemini-4-argon, effort: ultra}", # a codex level + "{id: claude, cli: antigravity-cli, model: gemini-4-argon, effort: max}", # no Gemini model takes it + "{id: claude, cli: antigravity-cli, model: gemini-4-argon, effort: xhigh}", # no Gemini model takes it ], ) def test_effort_outside_the_cli_levels_rejected(tmp_path, model):