diff --git a/src/agentevals/adk_bridge.py b/src/agentevals/adk_bridge.py index e9bfed6..3a2270c 100644 --- a/src/agentevals/adk_bridge.py +++ b/src/agentevals/adk_bridge.py @@ -13,6 +13,7 @@ from collections.abc import Sequence from typing import Any +from google.adk.evaluation.app_details import AgentDetails, AppDetails from google.adk.evaluation.eval_case import ( EvalCase, IntermediateData, @@ -101,6 +102,37 @@ def _as_dict(value: Any, key: str) -> dict[str, Any]: return {key: value} +def _app_details(turn: Turn) -> AppDetails | None: + """Include all captured tools, even those the agent never called.""" + tools_by_agent: dict[str, dict[str, genai_types.FunctionDeclaration]] = {} + for call in turn.llm_calls: + for definition in call.tool_definitions: + if definition.get("type") != "function": + continue + name = definition.get("name") + if not isinstance(name, str) or not name: + continue + tools = tools_by_agent.setdefault(call.agent_name or turn.agent_name or "agent", {}) + if name not in tools: + description = definition.get("description") + tools[name] = genai_types.FunctionDeclaration( + name=name, + description=description if isinstance(description, str) else None, + parameters_json_schema=definition.get("parameters"), + ) + if not tools_by_agent: + return None + return AppDetails( + agent_details={ + name: AgentDetails( + name=name, + tool_declarations=[genai_types.Tool(function_declarations=list(tools.values()))], + ) + for name, tools in tools_by_agent.items() + } + ) + + def to_adk_invocation(turn: Turn) -> Invocation: user_text = text_of(turn.user_input) final_text = text_of(turn.final_output) @@ -153,6 +185,7 @@ def to_adk_invocation(turn: Turn) -> Invocation: if final_text else None, intermediate_data=InvocationEvents(invocation_events=events), + app_details=_app_details(turn), creation_timestamp=turn.start_ns / 1e9, **_durations(turn), ) diff --git a/tests/test_adk_bridge.py b/tests/test_adk_bridge.py new file mode 100644 index 0000000..d757ae5 --- /dev/null +++ b/tests/test_adk_bridge.py @@ -0,0 +1,136 @@ +"""Captured tool declarations reach ADK judges without changing exported eval sets.""" + +from __future__ import annotations + +import pytest + +from agentevals.adk_bridge import invocation_to_dict, to_adk_invocation +from agentevals.builtin_metrics import _enrich_app_details +from agentevals.genai.model import LlmCall, ToolCall, Turn +from agentevals.otel.model import SpanRef + +REF = SpanRef("trace", "turn") + + +def _call(definitions, agent_name=None): + return LlmCall(ref=REF, operation="chat", agent_name=agent_name, tool_definitions=definitions) + + +def test_captured_tools_include_uncalled_tools_and_keep_first_definition(): + schema = {"type": "object", "properties": {"query": {"type": "string"}}} + definitions = [ + {"type": "function", "name": "search", "description": "Search the available evidence.", "parameters": schema}, + {"type": "function", "name": "lookup", "description": "Look up an item.", "parameters": {}}, + ] + turn = Turn( + ref=REF, + agent_name="researcher", + llm_calls=[ + _call(definitions), + _call( + [ + {"type": "function", "name": "search", "description": "Changed", "parameters": {}}, + {"type": "function", "name": "finish"}, + ] + ), + ], + tool_calls=[ToolCall(name="search")], + ) + + invocation = to_adk_invocation(turn) + + declarations = invocation.app_details.agent_details["researcher"].tool_declarations[0].function_declarations + assert [declaration.name for declaration in declarations] == ["search", "lookup", "finish"] + assert declarations[0].description == definitions[0]["description"] + assert declarations[0].parameters_json_schema == schema + assert declarations[0].parameters is None + assert declarations[1].description == definitions[1]["description"] + assert declarations[1].parameters_json_schema == {} + assert declarations[2].description is None + assert declarations[2].parameters_json_schema is None + assert _enrich_app_details([invocation])[0] is invocation + + +@pytest.mark.parametrize( + "call_agent,turn_agent,expected", [("worker", "root", "worker"), (None, "root", "root"), (None, None, "agent")] +) +def test_tool_agent_name_falls_back_to_turn_then_agent(call_agent, turn_agent, expected): + turn = Turn(ref=REF, agent_name=turn_agent, llm_calls=[_call([{"type": "function", "name": "search"}], call_agent)]) + + assert list(to_adk_invocation(turn).app_details.agent_details) == [expected] + + +def test_agents_keep_separate_tools_and_do_not_include_system_instructions(): + first = _call([{"type": "function", "name": "lookup", "description": "Search documents."}], "researcher") + first.system_instructions = [{"type": "text", "content": "Private instructions."}] + second = _call([{"type": "function", "name": "lookup", "description": "Search billing records."}], "billing") + + invocation = to_adk_invocation(Turn(ref=REF, llm_calls=[first, second])) + + agents = invocation.app_details.agent_details + assert list(agents) == ["researcher", "billing"] + assert agents["researcher"].tool_declarations[0].function_declarations[0].description == "Search documents." + assert agents["billing"].tool_declarations[0].function_declarations[0].description == "Search billing records." + assert all(details.instructions == "" for details in agents.values()) + assert "app_details" not in invocation_to_dict(invocation) + + +@pytest.mark.parametrize( + "calls", + [ + [], + [_call([])], + [ + _call( + [ + {}, + {"type": "function", "name": ""}, + {"type": "function", "name": None}, + {"type": "function", "name": 7}, + ] + ) + ], + ], +) +def test_missing_or_unusable_definitions_leave_names_only_fallback(calls): + turn = Turn(ref=REF, llm_calls=calls, tool_calls=[ToolCall(name="search")]) + invocation = to_adk_invocation(turn) + + assert invocation.app_details is None + (enriched,) = _enrich_app_details([invocation]) + (declaration,) = enriched.app_details.agent_details["agent"].tool_declarations[0].function_declarations + assert declaration.name == "search" + assert declaration.description is None + assert declaration.parameters_json_schema is None + assert invocation.app_details is None + + +def test_malformed_description_does_not_break_trace_conversion(): + turn = Turn( + ref=REF, llm_calls=[_call([{"type": "function", "name": "search", "description": {"unexpected": "object"}}])] + ) + + (declaration,) = ( + to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations + ) + + assert declaration.name == "search" + assert declaration.description is None + + +@pytest.mark.parametrize( + "definition", + [ + {"type": "google_search", "name": "web"}, + {"type": "custom", "name": "custom"}, + {"name": "missing_type"}, + ], +) +def test_only_function_definitions_are_converted(definition): + turn = Turn(ref=REF, llm_calls=[_call([definition, {"type": "function", "name": "lookup"}])]) + + (declaration,) = ( + to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations + ) + + assert declaration.name == "lookup" diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index b7cd0cb..585d7f7 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json import os import re from collections.abc import AsyncGenerator @@ -298,3 +299,53 @@ async def test_every_prebuilt_metric_receives_invocation_events(monkeypatch, met for inv in [*recorder.actual, *recorder.expected]: assert isinstance(inv.intermediate_data, InvocationEvents) assert inv.app_details is not None + + +class _ToolUseFakeJudge(BaseLlm): + prompts: list[str] = [] + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + self.prompts.append(llm_request.contents[0].parts[0].text) + text = "ID: rubric_0\nProperty: Uses appropriate tools.\nRationale: fake\nVerdict: yes" + yield LlmResponse(content=genai_types.Content(role="model", parts=[genai_types.Part(text=text)])) + + +@pytest.mark.parametrize("metric_name", ["rubric_based_tool_use_quality_v1", "rubric_based_final_response_quality_v1"]) +@pytest.mark.parametrize("builtin", [False, True], ids=["direct-evaluator", "builtin-metric"]) +async def test_replayed_judge_sees_all_available_tools(monkeypatch, builtin, metric_name): + (turn,) = extract_conversation( + load_traces(os.path.join(os.path.dirname(__file__), "fixtures/otlp/adk_py/v2proj_span_only/S2.json")) + ).turns + invocations = to_adk_invocations([turn]) + definitions = turn.llm_calls[0].tool_definitions + assert {definition["name"] for definition in definitions} == {"get_weather", "get_time"} + assert {call.name for call in turn.tool_calls} == {"get_weather", "get_time"} + + rubrics = ["Uses appropriate tools."] + evaluator = builtin_metrics.get_evaluator( + builtin_metrics.build_eval_metric(metric_name, None, 0.5, rubrics=rubrics) + ) + judge = _ToolUseFakeJudge(model="fake-judge") + evaluator._judge_model = judge + if builtin: + monkeypatch.setattr(builtin_metrics, "get_evaluator", lambda _metric: evaluator) + result = await evaluate_builtin_metric(metric_name, invocations, None, None, 0.5, rubrics=rubrics) + assert result.error is None + assert result.score == 1.0 + else: + result = await evaluator.evaluate_invocations(actual_invocations=invocations) + assert result.overall_score == 1.0 + + available_tools = json.loads(judge.prompts[0].rsplit("\n", 1)[1].split("")[0]) + declarations = { + declaration["name"]: declaration + for tool in available_tools["tool_declarations"]["weather_agent"] + for declaration in tool["function_declarations"] + } + assert set(declarations) == {definition["name"] for definition in definitions} + for definition in definitions: + declaration = declarations[definition["name"]] + assert declaration["description"] == definition["description"] + assert declaration["parameters_json_schema"] == definition["parameters"]