diff --git a/synth_lib/benchmark/agy_gateway.py b/synth_lib/benchmark/agy_gateway.py new file mode 100644 index 0000000..963b2bf --- /dev/null +++ b/synth_lib/benchmark/agy_gateway.py @@ -0,0 +1,89 @@ +"""Localhost relay between the Antigravity CLI's model gateway and the proxy's /gemini passthrough. + +Runs inside a leg's sandbox as a standalone script (standard library only): `agy` with AGY_LLM_GATEWAY_URL +sends Gemini-format requests with no credential, and the relay forwards each one upstream with the leg's +virtual key. Run as a script it binds, detaches, and returns, so `relay && exec agy …` starts agy only once +the port is listening. + + SYNTH_AGY_UPSTREAM=/gemini SYNTH_AGY_KEY= python3 agy_gateway.py +""" + +from __future__ import annotations + +import os +import sys +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# agy accepts a plain-http gateway only on localhost. +PORT = 4100 +UPSTREAM_ENV = "SYNTH_AGY_UPSTREAM" +KEY_ENV = "SYNTH_AGY_KEY" +# Seconds without a byte from upstream before the request fails; a high thinking level can stay silent for minutes. +UPSTREAM_TIMEOUT = 900 +LOG_FILE = os.path.expanduser("~/.agy-gateway.log") + + +def make_server(upstream: str, key: str, port: int = PORT) -> ThreadingHTTPServer: + class Relay(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + self._relay(None) + + def do_POST(self) -> None: + self._relay(self.rfile.read(int(self.headers.get("Content-Length", 0)))) + + def _relay(self, body: bytes | None) -> None: + headers = {"x-goog-api-key": key} + if body is not None: + headers["Content-Type"] = self.headers.get("Content-Type", "application/json") + request = urllib.request.Request(upstream + self.path, data=body, method=self.command, headers=headers) + try: + response = urllib.request.urlopen(request, timeout=UPSTREAM_TIMEOUT) + except urllib.error.HTTPError as exc: + response = exc + except OSError as exc: + self._reply(502, f"relay: upstream unreachable: {exc}".encode()) + return + with response: + self.send_response(response.status) + self.send_header("Content-Type", response.headers.get("Content-Type", "application/json")) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + while chunk := response.read(8192): + self.wfile.write(b"%x\r\n%s\r\n" % (len(chunk), chunk)) + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + + def _reply(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: + # Errors only: every request is already in the proxy's spend log. + if args and str(args[1]) not in ("200", "-"): + sys.stderr.write(f"{self.log_date_time_string()} {format % args}\n") + + return ThreadingHTTPServer(("127.0.0.1", port), Relay) + + +def main() -> None: + server = make_server(os.environ[UPSTREAM_ENV], os.environ.pop(KEY_ENV)) + if os.fork(): + os._exit(0) + os.setsid() + log = os.open(LOG_FILE, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + devnull = os.open(os.devnull, os.O_RDWR) + os.dup2(devnull, 0) + os.dup2(log, 1) + os.dup2(log, 2) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/synth_lib/benchmark/campaign.py b/synth_lib/benchmark/campaign.py index 4c3c724..67c3236 100644 --- a/synth_lib/benchmark/campaign.py +++ b/synth_lib/benchmark/campaign.py @@ -44,6 +44,7 @@ class ModelSpec: goal: bool = False # claude-code only: an evaluator holds the session to cli_adapters.GOAL_CONDITION workflow_size: str | None = None # with ultracode: a WORKFLOW_SIZES name; None = the CLI's default max_workflow_agents: int | None = None # with ultracode: workflow agents at once; None = the CLI's default + gateway: bool = False # antigravity-cli only: agy's model gateway, through cli_adapters' localhost relay PACKAGED_BASELINE = "synth_lib.benchmark.verdict.baseline_default" @@ -168,6 +169,8 @@ def _validate_model(m: ModelSpec) -> None: raise ValueError(f"{m.id}: ultracode is a claude-code setting") if m.goal and m.cli != "claude-code": raise ValueError(f"{m.id}: goal is a claude-code setting") + if m.gateway and m.cli != "antigravity-cli": + raise ValueError(f"{m.id}: gateway is an antigravity-cli setting") if (m.workflow_size is not None or m.max_workflow_agents is not None) and not m.ultracode: raise ValueError(f"{m.id}: workflow_size and max_workflow_agents need ultracode") if m.workflow_size is not None and m.workflow_size not in WORKFLOW_SIZES: diff --git a/synth_lib/benchmark/cli_adapters.py b/synth_lib/benchmark/cli_adapters.py index 9329fe5..2cc9df4 100644 --- a/synth_lib/benchmark/cli_adapters.py +++ b/synth_lib/benchmark/cli_adapters.py @@ -5,9 +5,12 @@ import json from pathlib import Path +from synth_lib.benchmark import agy_gateway from synth_lib.benchmark.campaign import ModelSpec FAKE_CLI_PATH = Path(__file__).parent / "fake_cli.py" +# Where an antigravity-cli gateway leg's relay is written, relative to the run's HOME. +GATEWAY_SCRIPT = ".agy-gateway.py" # The stopping condition a claude-code session is held to, the same rules as the constitution's envelope. # Its evaluator sees only the transcript and the hook arguments, so each case names what proves it. # "LANDING ORDER" is in the heading of every kind's landing_order and in driver.LANDING_PROMPT. @@ -178,14 +181,21 @@ def provision_files(self) -> dict[str, str]: class AntigravityAdapter(CLIAdapter): """Google's Antigravity CLI (`agy`) on the Gemini API: the proxy's /gemini passthrough, the virtual key as - GEMINI_API_KEY.""" + GEMINI_API_KEY. With `gateway`, agy uses its model gateway instead: the agy_gateway relay on localhost + adds the virtual key, and agy takes its model list from the Gemini API rather than its built-in one.""" def launch_cmd(self, prompt: str) -> list[str]: - return ["agy", "-p", prompt, *self._flags()] + return self._with_relay(["agy", "-p", prompt, *self._flags()]) def resume_cmd(self, prompt: str) -> list[str] | None: # -c: continues the most recent conversation in the current directory (the workspace) - return ["agy", "-c", "-p", prompt, *self._flags()] + return self._with_relay(["agy", "-c", "-p", prompt, *self._flags()]) + + def _with_relay(self, cmd: list[str]) -> list[str]: + if not self.spec.gateway: + return cmd + # The relay returns once it listens; "$@" keeps the prompt out of the shell string. + return ["sh", "-c", f'python3 "$HOME/{GATEWAY_SCRIPT}" && exec "$@"', "agy-gateway", *cmd] def _flags(self) -> list[str]: # In print mode an unknown --model exits non-zero instead of falling back to another model. @@ -195,6 +205,13 @@ def _flags(self) -> list[str]: return flags def env(self) -> dict[str, str]: + if self.spec.gateway: + return { + "AGY_LLM_GATEWAY_URL": f"http://127.0.0.1:{agy_gateway.PORT}", + agy_gateway.UPSTREAM_ENV: f"{self.proxy_url}/gemini", + agy_gateway.KEY_ENV: self.virtual_key, + "AGY_CLI_DISABLE_AUTO_UPDATE": "true", + } # No GOOGLE_API_KEY: the Gemini SDK prefers it over GEMINI_API_KEY. return { "GOOGLE_GEMINI_BASE_URL": f"{self.proxy_url}/gemini", @@ -203,6 +220,8 @@ def env(self) -> dict[str, str]: } def provision_files(self) -> dict[str, str]: + if self.spec.gateway: + return {f"~/{GATEWAY_SCRIPT}": Path(agy_gateway.__file__).read_text()} # GEMINI_API_KEY is read only with modelProvider "gemini". return {"~/.gemini/antigravity-cli/settings.json": json.dumps({"modelProvider": "gemini"}) + "\n"} diff --git a/tests/benchmark/test_agy_gateway.py b/tests/benchmark/test_agy_gateway.py new file mode 100644 index 0000000..92d0005 --- /dev/null +++ b/tests/benchmark/test_agy_gateway.py @@ -0,0 +1,78 @@ +import json +import threading +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from synth_lib.benchmark.agy_gateway import make_server + + +class Upstream(BaseHTTPRequestHandler): + """Records what reaches the proxy; /fail answers 400, everything else echoes the request.""" + + seen: list[dict] = [] + + def _answer(self, body: bytes) -> None: + Upstream.seen.append({"method": self.command, "path": self.path, "key": self.headers.get("x-goog-api-key")}) + status = 400 if self.path.startswith("/gemini/fail") else 200 + payload = json.dumps({"path": self.path, "body": body.decode()}).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: + self._answer(b"") + + def do_POST(self) -> None: + self._answer(self.rfile.read(int(self.headers["Content-Length"]))) + + def log_message(self, *args) -> None: + pass + + +def _serve(server: ThreadingHTTPServer) -> str: + threading.Thread(target=server.serve_forever, daemon=True).start() + return f"http://127.0.0.1:{server.server_address[1]}" + + +@pytest.fixture +def relay(): + Upstream.seen.clear() + upstream = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + server = make_server(_serve(upstream) + "/gemini", "sk-v", port=0) + yield _serve(server) + server.shutdown() + upstream.shutdown() + + +def test_relay_adds_the_key_and_keeps_path_and_body(relay): + listing = json.load(urllib.request.urlopen(relay + "/v1beta/models?pageSize=1000")) + assert listing["path"] == "/gemini/v1beta/models?pageSize=1000" + request = urllib.request.Request( + relay + "/v1beta/models/gemini-4-carbon:streamGenerateContent?alt=sse", + data=b'{"contents": []}', + headers={"Content-Type": "application/json"}, + ) + assert json.load(urllib.request.urlopen(request))["body"] == '{"contents": []}' + assert [(s["method"], s["key"]) for s in Upstream.seen] == [("GET", "sk-v"), ("POST", "sk-v")] + + +def test_relay_passes_upstream_errors_through(relay): + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(relay + "/fail") + assert exc.value.code == 400 and json.load(exc.value)["path"] == "/gemini/fail" + + +def test_relay_answers_502_when_upstream_is_down(): + closed = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + dead = f"http://127.0.0.1:{closed.server_address[1]}" + closed.server_close() + server = make_server(dead, "sk-v", port=0) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(_serve(server) + "/v1beta/models") + server.shutdown() + assert exc.value.code == 502 diff --git a/tests/benchmark/test_campaign.py b/tests/benchmark/test_campaign.py index 1eab053..e08f7a7 100644 --- a/tests/benchmark/test_campaign.py +++ b/tests/benchmark/test_campaign.py @@ -110,6 +110,20 @@ def test_antigravity_cli_effort_loads(tmp_path): assert load_campaign(_write(tmp_path, text)).models[0].effort == "high" +def test_antigravity_cli_gateway_loads_and_is_rejected_elsewhere(tmp_path): + text = VALID_YAML.replace( + "{id: claude, cli: claude-code, model: claude-model}", + "{id: carbon, cli: antigravity-cli, model: gemini-4-carbon, effort: high, gateway: true}", + ) + assert load_campaign(_write(tmp_path, text)).models[0].gateway + bad = VALID_YAML.replace( + "{id: claude, cli: claude-code, model: claude-model}", + "{id: c, cli: claude-code, model: claude-model, gateway: true}", + ) + with pytest.raises(ValueError, match="gateway"): + load_campaign(_write(tmp_path, bad)) + + def test_workflow_size_and_agents_load_with_ultracode(tmp_path): text = VALID_YAML.replace( "{id: claude, cli: claude-code, model: claude-model}", diff --git a/tests/benchmark/test_cli_adapters.py b/tests/benchmark/test_cli_adapters.py index 340159d..b3d74ed 100644 --- a/tests/benchmark/test_cli_adapters.py +++ b/tests/benchmark/test_cli_adapters.py @@ -1,8 +1,10 @@ import json import tomllib +from pathlib import Path +from synth_lib.benchmark import agy_gateway from synth_lib.benchmark.campaign import ModelSpec -from synth_lib.benchmark.cli_adapters import GOAL_CONDITION, build_adapter +from synth_lib.benchmark.cli_adapters import GATEWAY_SCRIPT, GOAL_CONDITION, build_adapter def test_claude_adapter_cmd_and_env(): @@ -176,6 +178,27 @@ def test_antigravity_adapter_pins_model_and_effort_and_routes_through_the_passth assert settings == {"modelProvider": "gemini"} # without it GEMINI_API_KEY is ignored +def test_antigravity_adapter_gateway_starts_the_relay_before_agy(): + a = build_adapter( + ModelSpec(id="c", cli="antigravity-cli", model="gemini-4-carbon", effort="high", gateway=True), + proxy_url="http://host.docker.internal:4000", + virtual_key="sk-v", + ) + plain = build_adapter( + ModelSpec(id="c", cli="antigravity-cli", model="gemini-4-carbon", effort="high"), "http://x", "sk-v" + ) + cmd = a.launch_cmd("go") + assert cmd[:2] == ["sh", "-c"] and f'"$HOME/{GATEWAY_SCRIPT}" && exec "$@"' in cmd[2] + assert cmd[4:] == plain.launch_cmd("go") # the prompt is an argument, never part of the shell string + assert a.resume_cmd("again")[4:] == plain.resume_cmd("again") + env = a.env() + assert env["AGY_LLM_GATEWAY_URL"] == f"http://127.0.0.1:{agy_gateway.PORT}" + assert env[agy_gateway.UPSTREAM_ENV] == "http://host.docker.internal:4000/gemini" + assert env[agy_gateway.KEY_ENV] == "sk-v" + assert "GEMINI_API_KEY" not in env and "GOOGLE_GEMINI_BASE_URL" not in env + assert a.provision_files() == {f"~/{GATEWAY_SCRIPT}": Path(agy_gateway.__file__).read_text()} + + def test_antigravity_adapter_without_effort_passes_no_flag(): a = build_adapter(ModelSpec(id="g", cli="antigravity-cli", model="gemini-4-argon"), "http://x", "sk-v") assert "--effort" not in a.launch_cmd("go") and "--effort" not in a.resume_cmd("go")