Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions synth_lib/benchmark/agy_gateway.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
"""Localhost relay between the Antigravity CLI's model gateway and the proxy's /gemini passthrough.

Runs inside a leg's sandbox as a standalone script (standard library only): `agy` with AGY_LLM_GATEWAY_URL
sends Gemini-format requests with no credential, and the relay forwards each one upstream with the leg's
virtual key. Run as a script it binds, detaches, and returns, so `relay && exec agy …` starts agy only once
the port is listening.

SYNTH_AGY_UPSTREAM=<proxy>/gemini SYNTH_AGY_KEY=<virtual key> python3 agy_gateway.py
"""

from __future__ import annotations

import os
import sys
import urllib.error
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

# agy accepts a plain-http gateway only on localhost.
PORT = 4100
UPSTREAM_ENV = "SYNTH_AGY_UPSTREAM"
KEY_ENV = "SYNTH_AGY_KEY"
# Seconds without a byte from upstream before the request fails; a high thinking level can stay silent for minutes.
UPSTREAM_TIMEOUT = 900
LOG_FILE = os.path.expanduser("~/.agy-gateway.log")


def make_server(upstream: str, key: str, port: int = PORT) -> ThreadingHTTPServer:
class Relay(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"

def do_GET(self) -> None:
self._relay(None)

def do_POST(self) -> None:
self._relay(self.rfile.read(int(self.headers.get("Content-Length", 0))))

def _relay(self, body: bytes | None) -> None:
headers = {"x-goog-api-key": key}
if body is not None:
headers["Content-Type"] = self.headers.get("Content-Type", "application/json")
request = urllib.request.Request(upstream + self.path, data=body, method=self.command, headers=headers)
try:
response = urllib.request.urlopen(request, timeout=UPSTREAM_TIMEOUT)
except urllib.error.HTTPError as exc:
response = exc
except OSError as exc:
self._reply(502, f"relay: upstream unreachable: {exc}".encode())
return
with response:
self.send_response(response.status)
self.send_header("Content-Type", response.headers.get("Content-Type", "application/json"))
self.send_header("Transfer-Encoding", "chunked")
self.end_headers()
while chunk := response.read(8192):
self.wfile.write(b"%x\r\n%s\r\n" % (len(chunk), chunk))
self.wfile.flush()
self.wfile.write(b"0\r\n\r\n")

def _reply(self, status: int, body: bytes) -> None:
self.send_response(status)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)

def log_message(self, format: str, *args) -> None:
# Errors only: every request is already in the proxy's spend log.
if args and str(args[1]) not in ("200", "-"):
sys.stderr.write(f"{self.log_date_time_string()} {format % args}\n")

return ThreadingHTTPServer(("127.0.0.1", port), Relay)


def main() -> None:
server = make_server(os.environ[UPSTREAM_ENV], os.environ.pop(KEY_ENV))
if os.fork():
os._exit(0)
os.setsid()
log = os.open(LOG_FILE, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
devnull = os.open(os.devnull, os.O_RDWR)
os.dup2(devnull, 0)
os.dup2(log, 1)
os.dup2(log, 2)
server.serve_forever()


if __name__ == "__main__":
main()
3 changes: 3 additions & 0 deletions synth_lib/benchmark/campaign.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class ModelSpec:
goal: bool = False # claude-code only: an evaluator holds the session to cli_adapters.GOAL_CONDITION
workflow_size: str | None = None # with ultracode: a WORKFLOW_SIZES name; None = the CLI's default
max_workflow_agents: int | None = None # with ultracode: workflow agents at once; None = the CLI's default
gateway: bool = False # antigravity-cli only: agy's model gateway, through cli_adapters' localhost relay


PACKAGED_BASELINE = "synth_lib.benchmark.verdict.baseline_default"
Expand Down Expand Up @@ -168,6 +169,8 @@ def _validate_model(m: ModelSpec) -> None:
raise ValueError(f"{m.id}: ultracode is a claude-code setting")
if m.goal and m.cli != "claude-code":
raise ValueError(f"{m.id}: goal is a claude-code setting")
if m.gateway and m.cli != "antigravity-cli":
raise ValueError(f"{m.id}: gateway is an antigravity-cli setting")
if (m.workflow_size is not None or m.max_workflow_agents is not None) and not m.ultracode:
raise ValueError(f"{m.id}: workflow_size and max_workflow_agents need ultracode")
if m.workflow_size is not None and m.workflow_size not in WORKFLOW_SIZES:
Expand Down
25 changes: 22 additions & 3 deletions synth_lib/benchmark/cli_adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
import json
from pathlib import Path

from synth_lib.benchmark import agy_gateway
from synth_lib.benchmark.campaign import ModelSpec

FAKE_CLI_PATH = Path(__file__).parent / "fake_cli.py"
# Where an antigravity-cli gateway leg's relay is written, relative to the run's HOME.
GATEWAY_SCRIPT = ".agy-gateway.py"
# The stopping condition a claude-code session is held to, the same rules as the constitution's envelope.
# Its evaluator sees only the transcript and the hook arguments, so each case names what proves it.
# "LANDING ORDER" is in the heading of every kind's landing_order and in driver.LANDING_PROMPT.
Expand Down Expand Up @@ -178,14 +181,21 @@ def provision_files(self) -> dict[str, str]:

class AntigravityAdapter(CLIAdapter):
"""Google's Antigravity CLI (`agy`) on the Gemini API: the proxy's /gemini passthrough, the virtual key as
GEMINI_API_KEY."""
GEMINI_API_KEY. With `gateway`, agy uses its model gateway instead: the agy_gateway relay on localhost
adds the virtual key, and agy takes its model list from the Gemini API rather than its built-in one."""

def launch_cmd(self, prompt: str) -> list[str]:
return ["agy", "-p", prompt, *self._flags()]
return self._with_relay(["agy", "-p", prompt, *self._flags()])

def resume_cmd(self, prompt: str) -> list[str] | None:
# -c: continues the most recent conversation in the current directory (the workspace)
return ["agy", "-c", "-p", prompt, *self._flags()]
return self._with_relay(["agy", "-c", "-p", prompt, *self._flags()])

def _with_relay(self, cmd: list[str]) -> list[str]:
if not self.spec.gateway:
return cmd
# The relay returns once it listens; "$@" keeps the prompt out of the shell string.
return ["sh", "-c", f'python3 "$HOME/{GATEWAY_SCRIPT}" && exec "$@"', "agy-gateway", *cmd]

def _flags(self) -> list[str]:
# In print mode an unknown --model exits non-zero instead of falling back to another model.
Expand All @@ -195,6 +205,13 @@ def _flags(self) -> list[str]:
return flags

def env(self) -> dict[str, str]:
if self.spec.gateway:
return {
"AGY_LLM_GATEWAY_URL": f"http://127.0.0.1:{agy_gateway.PORT}",
agy_gateway.UPSTREAM_ENV: f"{self.proxy_url}/gemini",
agy_gateway.KEY_ENV: self.virtual_key,
"AGY_CLI_DISABLE_AUTO_UPDATE": "true",
}
# No GOOGLE_API_KEY: the Gemini SDK prefers it over GEMINI_API_KEY.
return {
"GOOGLE_GEMINI_BASE_URL": f"{self.proxy_url}/gemini",
Expand All @@ -203,6 +220,8 @@ def env(self) -> dict[str, str]:
}

def provision_files(self) -> dict[str, str]:
if self.spec.gateway:
return {f"~/{GATEWAY_SCRIPT}": Path(agy_gateway.__file__).read_text()}
# GEMINI_API_KEY is read only with modelProvider "gemini".
return {"~/.gemini/antigravity-cli/settings.json": json.dumps({"modelProvider": "gemini"}) + "\n"}

Expand Down
78 changes: 78 additions & 0 deletions tests/benchmark/test_agy_gateway.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import json
import threading
import urllib.error
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

import pytest

from synth_lib.benchmark.agy_gateway import make_server


class Upstream(BaseHTTPRequestHandler):
"""Records what reaches the proxy; /fail answers 400, everything else echoes the request."""

seen: list[dict] = []

def _answer(self, body: bytes) -> None:
Upstream.seen.append({"method": self.command, "path": self.path, "key": self.headers.get("x-goog-api-key")})
status = 400 if self.path.startswith("/gemini/fail") else 200
payload = json.dumps({"path": self.path, "body": body.decode()}).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)

def do_GET(self) -> None:
self._answer(b"")

def do_POST(self) -> None:
self._answer(self.rfile.read(int(self.headers["Content-Length"])))

def log_message(self, *args) -> None:
pass


def _serve(server: ThreadingHTTPServer) -> str:
threading.Thread(target=server.serve_forever, daemon=True).start()
return f"http://127.0.0.1:{server.server_address[1]}"


@pytest.fixture
def relay():
Upstream.seen.clear()
upstream = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
server = make_server(_serve(upstream) + "/gemini", "sk-v", port=0)
yield _serve(server)
server.shutdown()
upstream.shutdown()


def test_relay_adds_the_key_and_keeps_path_and_body(relay):
listing = json.load(urllib.request.urlopen(relay + "/v1beta/models?pageSize=1000"))
assert listing["path"] == "/gemini/v1beta/models?pageSize=1000"
request = urllib.request.Request(
relay + "/v1beta/models/gemini-4-carbon:streamGenerateContent?alt=sse",
data=b'{"contents": []}',
headers={"Content-Type": "application/json"},
)
assert json.load(urllib.request.urlopen(request))["body"] == '{"contents": []}'
assert [(s["method"], s["key"]) for s in Upstream.seen] == [("GET", "sk-v"), ("POST", "sk-v")]


def test_relay_passes_upstream_errors_through(relay):
with pytest.raises(urllib.error.HTTPError) as exc:
urllib.request.urlopen(relay + "/fail")
assert exc.value.code == 400 and json.load(exc.value)["path"] == "/gemini/fail"


def test_relay_answers_502_when_upstream_is_down():
closed = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
dead = f"http://127.0.0.1:{closed.server_address[1]}"
closed.server_close()
server = make_server(dead, "sk-v", port=0)
with pytest.raises(urllib.error.HTTPError) as exc:
urllib.request.urlopen(_serve(server) + "/v1beta/models")
server.shutdown()
assert exc.value.code == 502
14 changes: 14 additions & 0 deletions tests/benchmark/test_campaign.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,20 @@ def test_antigravity_cli_effort_loads(tmp_path):
assert load_campaign(_write(tmp_path, text)).models[0].effort == "high"


def test_antigravity_cli_gateway_loads_and_is_rejected_elsewhere(tmp_path):
text = VALID_YAML.replace(
"{id: claude, cli: claude-code, model: claude-model}",
"{id: carbon, cli: antigravity-cli, model: gemini-4-carbon, effort: high, gateway: true}",
)
assert load_campaign(_write(tmp_path, text)).models[0].gateway
bad = VALID_YAML.replace(
"{id: claude, cli: claude-code, model: claude-model}",
"{id: c, cli: claude-code, model: claude-model, gateway: true}",
)
with pytest.raises(ValueError, match="gateway"):
load_campaign(_write(tmp_path, bad))


def test_workflow_size_and_agents_load_with_ultracode(tmp_path):
text = VALID_YAML.replace(
"{id: claude, cli: claude-code, model: claude-model}",
Expand Down
25 changes: 24 additions & 1 deletion tests/benchmark/test_cli_adapters.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import json
import tomllib
from pathlib import Path

from synth_lib.benchmark import agy_gateway
from synth_lib.benchmark.campaign import ModelSpec
from synth_lib.benchmark.cli_adapters import GOAL_CONDITION, build_adapter
from synth_lib.benchmark.cli_adapters import GATEWAY_SCRIPT, GOAL_CONDITION, build_adapter


def test_claude_adapter_cmd_and_env():
Expand Down Expand Up @@ -176,6 +178,27 @@ def test_antigravity_adapter_pins_model_and_effort_and_routes_through_the_passth
assert settings == {"modelProvider": "gemini"} # without it GEMINI_API_KEY is ignored


def test_antigravity_adapter_gateway_starts_the_relay_before_agy():
a = build_adapter(
ModelSpec(id="c", cli="antigravity-cli", model="gemini-4-carbon", effort="high", gateway=True),
proxy_url="http://host.docker.internal:4000",
virtual_key="sk-v",
)
plain = build_adapter(
ModelSpec(id="c", cli="antigravity-cli", model="gemini-4-carbon", effort="high"), "http://x", "sk-v"
)
cmd = a.launch_cmd("go")
assert cmd[:2] == ["sh", "-c"] and f'"$HOME/{GATEWAY_SCRIPT}" && exec "$@"' in cmd[2]
assert cmd[4:] == plain.launch_cmd("go") # the prompt is an argument, never part of the shell string
assert a.resume_cmd("again")[4:] == plain.resume_cmd("again")
env = a.env()
assert env["AGY_LLM_GATEWAY_URL"] == f"http://127.0.0.1:{agy_gateway.PORT}"
assert env[agy_gateway.UPSTREAM_ENV] == "http://host.docker.internal:4000/gemini"
assert env[agy_gateway.KEY_ENV] == "sk-v"
assert "GEMINI_API_KEY" not in env and "GOOGLE_GEMINI_BASE_URL" not in env
assert a.provision_files() == {f"~/{GATEWAY_SCRIPT}": Path(agy_gateway.__file__).read_text()}


def test_antigravity_adapter_without_effort_passes_no_flag():
a = build_adapter(ModelSpec(id="g", cli="antigravity-cli", model="gemini-4-argon"), "http://x", "sk-v")
assert "--effort" not in a.launch_cmd("go") and "--effort" not in a.resume_cmd("go")
Expand Down
Loading