feat: pass captured tool declarations to ADK judges

src/agentevals/adk_bridge.py

@@ -13,6 +13,7 @@
 from collections.abc import Sequence
 from typing import Any
 
+from google.adk.evaluation.app_details import AgentDetails, AppDetails
 from google.adk.evaluation.eval_case import (
     EvalCase,
     IntermediateData,
@@ -101,6 +102,37 @@ def _as_dict(value: Any, key: str) -> dict[str, Any]:
     return {key: value}
 
 
+def _app_details(turn: Turn) -> AppDetails | None:
+    """Include all captured tools, even those the agent never called."""
+    tools_by_agent: dict[str, dict[str, genai_types.FunctionDeclaration]] = {}
+    for call in turn.llm_calls:
+        for definition in call.tool_definitions:
+            if definition.get("type") != "function":
+                continue
+            name = definition.get("name")
+            if not isinstance(name, str) or not name:
+                continue
+            tools = tools_by_agent.setdefault(call.agent_name or turn.agent_name or "agent", {})
+            if name not in tools:
+                description = definition.get("description")
+                tools[name] = genai_types.FunctionDeclaration(
+                    name=name,
+                    description=description if isinstance(description, str) else None,
+                    parameters_json_schema=definition.get("parameters"),
+                )
+    if not tools_by_agent:
+        return None
+    return AppDetails(
+        agent_details={
+            name: AgentDetails(
+                name=name,
+                tool_declarations=[genai_types.Tool(function_declarations=list(tools.values()))],
+            )
+            for name, tools in tools_by_agent.items()
+        }
+    )
+
+
 def to_adk_invocation(turn: Turn) -> Invocation:
     user_text = text_of(turn.user_input)
     final_text = text_of(turn.final_output)
@@ -153,6 +185,7 @@ def to_adk_invocation(turn: Turn) -> Invocation:
         if final_text
         else None,
         intermediate_data=InvocationEvents(invocation_events=events),
+        app_details=_app_details(turn),
         creation_timestamp=turn.start_ns / 1e9,
         **_durations(turn),
     )

tests/test_adk_bridge.py

@@ -0,0 +1,136 @@
+"""Captured tool declarations reach ADK judges without changing exported eval sets."""
+
+from __future__ import annotations
+
+import pytest
+
+from agentevals.adk_bridge import invocation_to_dict, to_adk_invocation
+from agentevals.builtin_metrics import _enrich_app_details
+from agentevals.genai.model import LlmCall, ToolCall, Turn
+from agentevals.otel.model import SpanRef
+
+REF = SpanRef("trace", "turn")
+
+
+def _call(definitions, agent_name=None):
+    return LlmCall(ref=REF, operation="chat", agent_name=agent_name, tool_definitions=definitions)
+
+
+def test_captured_tools_include_uncalled_tools_and_keep_first_definition():
+    schema = {"type": "object", "properties": {"query": {"type": "string"}}}
+    definitions = [
+        {"type": "function", "name": "search", "description": "Search the available evidence.", "parameters": schema},
+        {"type": "function", "name": "lookup", "description": "Look up an item.", "parameters": {}},
+    ]
+    turn = Turn(
+        ref=REF,
+        agent_name="researcher",
+        llm_calls=[
+            _call(definitions),
+            _call(
+                [
+                    {"type": "function", "name": "search", "description": "Changed", "parameters": {}},
+                    {"type": "function", "name": "finish"},
+                ]
+            ),
+        ],
+        tool_calls=[ToolCall(name="search")],
+    )
+
+    invocation = to_adk_invocation(turn)
+
+    declarations = invocation.app_details.agent_details["researcher"].tool_declarations[0].function_declarations
+    assert [declaration.name for declaration in declarations] == ["search", "lookup", "finish"]
+    assert declarations[0].description == definitions[0]["description"]
+    assert declarations[0].parameters_json_schema == schema
+    assert declarations[0].parameters is None
+    assert declarations[1].description == definitions[1]["description"]
+    assert declarations[1].parameters_json_schema == {}
+    assert declarations[2].description is None
+    assert declarations[2].parameters_json_schema is None
+    assert _enrich_app_details([invocation])[0] is invocation
+
+
+@pytest.mark.parametrize(
+    "call_agent,turn_agent,expected", [("worker", "root", "worker"), (None, "root", "root"), (None, None, "agent")]
+)
+def test_tool_agent_name_falls_back_to_turn_then_agent(call_agent, turn_agent, expected):
+    turn = Turn(ref=REF, agent_name=turn_agent, llm_calls=[_call([{"type": "function", "name": "search"}], call_agent)])
+
+    assert list(to_adk_invocation(turn).app_details.agent_details) == [expected]
+
+
+def test_agents_keep_separate_tools_and_do_not_include_system_instructions():
+    first = _call([{"type": "function", "name": "lookup", "description": "Search documents."}], "researcher")
+    first.system_instructions = [{"type": "text", "content": "Private instructions."}]
+    second = _call([{"type": "function", "name": "lookup", "description": "Search billing records."}], "billing")
+
+    invocation = to_adk_invocation(Turn(ref=REF, llm_calls=[first, second]))
+
+    agents = invocation.app_details.agent_details
+    assert list(agents) == ["researcher", "billing"]
+    assert agents["researcher"].tool_declarations[0].function_declarations[0].description == "Search documents."
+    assert agents["billing"].tool_declarations[0].function_declarations[0].description == "Search billing records."
+    assert all(details.instructions == "" for details in agents.values())
+    assert "app_details" not in invocation_to_dict(invocation)
+
+
+@pytest.mark.parametrize(
+    "calls",
+    [
+        [],
+        [_call([])],
+        [
+            _call(
+                [
+                    {},
+                    {"type": "function", "name": ""},
+                    {"type": "function", "name": None},
+                    {"type": "function", "name": 7},
+                ]
+            )
+        ],
+    ],
+)
+def test_missing_or_unusable_definitions_leave_names_only_fallback(calls):
+    turn = Turn(ref=REF, llm_calls=calls, tool_calls=[ToolCall(name="search")])
+    invocation = to_adk_invocation(turn)
+
+    assert invocation.app_details is None
+    (enriched,) = _enrich_app_details([invocation])
+    (declaration,) = enriched.app_details.agent_details["agent"].tool_declarations[0].function_declarations
+    assert declaration.name == "search"
+    assert declaration.description is None
+    assert declaration.parameters_json_schema is None
+    assert invocation.app_details is None
+
+
+def test_malformed_description_does_not_break_trace_conversion():
+    turn = Turn(
+        ref=REF, llm_calls=[_call([{"type": "function", "name": "search", "description": {"unexpected": "object"}}])]
+    )
+
+    (declaration,) = (
+        to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations
+    )
+
+    assert declaration.name == "search"
+    assert declaration.description is None
+
+
+@pytest.mark.parametrize(
+    "definition",
+    [
+        {"type": "google_search", "name": "web"},
+        {"type": "custom", "name": "custom"},
+        {"name": "missing_type"},
+    ],
+)
+def test_only_function_definitions_are_converted(definition):
+    turn = Turn(ref=REF, llm_calls=[_call([definition, {"type": "function", "name": "lookup"}])])
+
+    (declaration,) = (
+        to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations
+    )
+
+    assert declaration.name == "lookup"

tests/test_builtin_metrics.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 import re
 from collections.abc import AsyncGenerator
@@ -298,3 +299,53 @@ async def test_every_prebuilt_metric_receives_invocation_events(monkeypatch, met
     for inv in [*recorder.actual, *recorder.expected]:
         assert isinstance(inv.intermediate_data, InvocationEvents)
         assert inv.app_details is not None
+
+
+class _ToolUseFakeJudge(BaseLlm):
+    prompts: list[str] = []
+
+    async def generate_content_async(
+        self, llm_request: LlmRequest, stream: bool = False
+    ) -> AsyncGenerator[LlmResponse, None]:
+        self.prompts.append(llm_request.contents[0].parts[0].text)
+        text = "ID: rubric_0\nProperty: Uses appropriate tools.\nRationale: fake\nVerdict: yes"
+        yield LlmResponse(content=genai_types.Content(role="model", parts=[genai_types.Part(text=text)]))
+
+
+@pytest.mark.parametrize("metric_name", ["rubric_based_tool_use_quality_v1", "rubric_based_final_response_quality_v1"])
+@pytest.mark.parametrize("builtin", [False, True], ids=["direct-evaluator", "builtin-metric"])
+async def test_replayed_judge_sees_all_available_tools(monkeypatch, builtin, metric_name):
+    (turn,) = extract_conversation(
+        load_traces(os.path.join(os.path.dirname(__file__), "fixtures/otlp/adk_py/v2proj_span_only/S2.json"))
+    ).turns
+    invocations = to_adk_invocations([turn])
+    definitions = turn.llm_calls[0].tool_definitions
+    assert {definition["name"] for definition in definitions} == {"get_weather", "get_time"}
+    assert {call.name for call in turn.tool_calls} == {"get_weather", "get_time"}
+
+    rubrics = ["Uses appropriate tools."]
+    evaluator = builtin_metrics.get_evaluator(
+        builtin_metrics.build_eval_metric(metric_name, None, 0.5, rubrics=rubrics)
+    )
+    judge = _ToolUseFakeJudge(model="fake-judge")
+    evaluator._judge_model = judge
+    if builtin:
+        monkeypatch.setattr(builtin_metrics, "get_evaluator", lambda _metric: evaluator)
+        result = await evaluate_builtin_metric(metric_name, invocations, None, None, 0.5, rubrics=rubrics)
+        assert result.error is None
+        assert result.score == 1.0
+    else:
+        result = await evaluator.evaluate_invocations(actual_invocations=invocations)
+        assert result.overall_score == 1.0
+
+    available_tools = json.loads(judge.prompts[0].rsplit("<available_tools>\n", 1)[1].split("</available_tools>")[0])
+    declarations = {
+        declaration["name"]: declaration
+        for tool in available_tools["tool_declarations"]["weather_agent"]
+        for declaration in tool["function_declarations"]
+    }
+    assert set(declarations) == {definition["name"] for definition in definitions}
+    for definition in definitions:
+        declaration = declarations[definition["name"]]
+        assert declaration["description"] == definition["description"]
+        assert declaration["parameters_json_schema"] == definition["parameters"]