From 85ec5af8c9f80ae7fc8e05cee897d2a499827b81 Mon Sep 17 00:00:00 2001 From: erauner Date: Thu, 29 Oct 2026 13:04:24 -0500 Subject: [PATCH 1/3] feat(adk-bridge): pass captured tool definitions to judges --- src/agentevals/adk_bridge.py | 31 +++++++++++ tests/test_adk_bridge.py | 96 +++++++++++++++++++++++++++++++++++ tests/test_builtin_metrics.py | 50 ++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 tests/test_adk_bridge.py diff --git a/src/agentevals/adk_bridge.py b/src/agentevals/adk_bridge.py index e9bfed6..4f0ee97 100644 --- a/src/agentevals/adk_bridge.py +++ b/src/agentevals/adk_bridge.py @@ -13,6 +13,7 @@ from collections.abc import Sequence from typing import Any +from google.adk.evaluation.app_details import AgentDetails, AppDetails from google.adk.evaluation.eval_case import ( EvalCase, IntermediateData, @@ -101,6 +102,35 @@ def _as_dict(value: Any, key: str) -> dict[str, Any]: return {key: value} +def _app_details(turn: Turn) -> AppDetails | None: + """Include all captured tools, even those the agent never called.""" + tools_by_agent: dict[str, dict[str, genai_types.FunctionDeclaration]] = {} + for call in turn.llm_calls: + for definition in call.tool_definitions: + name = definition.get("name") + if not isinstance(name, str) or not name: + continue + tools = tools_by_agent.setdefault(call.agent_name or turn.agent_name or "agent", {}) + if name not in tools: + description = definition.get("description") + tools[name] = genai_types.FunctionDeclaration( + name=name, + description=description if isinstance(description, str) else None, + parameters_json_schema=definition.get("parameters"), + ) + if not tools_by_agent: + return None + return AppDetails( + agent_details={ + name: AgentDetails( + name=name, + tool_declarations=[genai_types.Tool(function_declarations=list(tools.values()))], + ) + for name, tools in tools_by_agent.items() + } + ) + + def to_adk_invocation(turn: Turn) -> Invocation: user_text = text_of(turn.user_input) final_text = text_of(turn.final_output) @@ -153,6 +183,7 @@ def to_adk_invocation(turn: Turn) -> Invocation: if final_text else None, intermediate_data=InvocationEvents(invocation_events=events), + app_details=_app_details(turn), creation_timestamp=turn.start_ns / 1e9, **_durations(turn), ) diff --git a/tests/test_adk_bridge.py b/tests/test_adk_bridge.py new file mode 100644 index 0000000..8bc1a58 --- /dev/null +++ b/tests/test_adk_bridge.py @@ -0,0 +1,96 @@ +"""Captured tool declarations reach ADK judges without changing exported eval sets.""" + +from __future__ import annotations + +import pytest + +from agentevals.adk_bridge import invocation_to_dict, to_adk_invocation +from agentevals.builtin_metrics import _enrich_app_details +from agentevals.genai.model import LlmCall, ToolCall, Turn +from agentevals.otel.model import SpanRef + +REF = SpanRef("trace", "turn") + + +def _call(definitions, agent_name=None): + return LlmCall(ref=REF, operation="chat", agent_name=agent_name, tool_definitions=definitions) + + +@pytest.mark.parametrize("schema_type", ["object", "OBJECT"], ids=["json-schema", "adk-schema"]) +def test_captured_tools_include_uncalled_tools_and_keep_first_definition(schema_type): + schema = {"type": schema_type, "properties": {"query": {"type": "string"}}} + definitions = [ + {"name": "search", "description": "Search the available evidence.", "parameters": schema}, + {"name": "lookup", "description": "Look up an item.", "parameters": {}}, + ] + turn = Turn( + ref=REF, + agent_name="researcher", + llm_calls=[ + _call(definitions), + _call([{"name": "search", "description": "Changed", "parameters": {}}, {"name": "finish"}]), + ], + tool_calls=[ToolCall(name="search")], + ) + + invocation = to_adk_invocation(turn) + + declarations = invocation.app_details.agent_details["researcher"].tool_declarations[0].function_declarations + assert [declaration.name for declaration in declarations] == ["search", "lookup", "finish"] + assert declarations[0].description == definitions[0]["description"] + assert declarations[0].parameters_json_schema == schema + assert declarations[0].parameters is None + assert declarations[1].description == definitions[1]["description"] + assert declarations[1].parameters_json_schema == {} + assert declarations[2].description is None + assert declarations[2].parameters_json_schema is None + assert _enrich_app_details([invocation])[0] is invocation + + +@pytest.mark.parametrize( + "call_agent,turn_agent,expected", [("worker", "root", "worker"), (None, "root", "root"), (None, None, "agent")] +) +def test_tool_agent_name_falls_back_to_turn_then_agent(call_agent, turn_agent, expected): + turn = Turn(ref=REF, agent_name=turn_agent, llm_calls=[_call([{"name": "search"}], call_agent)]) + + assert list(to_adk_invocation(turn).app_details.agent_details) == [expected] + + +def test_agents_keep_separate_tools_and_do_not_include_system_instructions(): + first = _call([{"name": "lookup", "description": "Search documents."}], "researcher") + first.system_instructions = [{"type": "text", "content": "Private instructions."}] + second = _call([{"name": "lookup", "description": "Search billing records."}], "billing") + + invocation = to_adk_invocation(Turn(ref=REF, llm_calls=[first, second])) + + agents = invocation.app_details.agent_details + assert list(agents) == ["researcher", "billing"] + assert agents["researcher"].tool_declarations[0].function_declarations[0].description == "Search documents." + assert agents["billing"].tool_declarations[0].function_declarations[0].description == "Search billing records." + assert all(details.instructions == "" for details in agents.values()) + assert "app_details" not in invocation_to_dict(invocation) + + +@pytest.mark.parametrize("calls", [[], [_call([])], [_call([{}, {"name": ""}, {"name": None}, {"name": 7}])]]) +def test_missing_or_unusable_definitions_leave_names_only_fallback(calls): + turn = Turn(ref=REF, llm_calls=calls, tool_calls=[ToolCall(name="search")]) + invocation = to_adk_invocation(turn) + + assert invocation.app_details is None + (enriched,) = _enrich_app_details([invocation]) + (declaration,) = enriched.app_details.agent_details["agent"].tool_declarations[0].function_declarations + assert declaration.name == "search" + assert declaration.description is None + assert declaration.parameters_json_schema is None + assert invocation.app_details is None + + +def test_malformed_description_does_not_break_trace_conversion(): + turn = Turn(ref=REF, llm_calls=[_call([{"name": "search", "description": {"unexpected": "object"}}])]) + + (declaration,) = ( + to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations + ) + + assert declaration.name == "search" + assert declaration.description is None diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index b7cd0cb..8dcc76b 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json import os import re from collections.abc import AsyncGenerator @@ -298,3 +299,52 @@ async def test_every_prebuilt_metric_receives_invocation_events(monkeypatch, met for inv in [*recorder.actual, *recorder.expected]: assert isinstance(inv.intermediate_data, InvocationEvents) assert inv.app_details is not None + + +class _ToolUseFakeJudge(BaseLlm): + prompts: list[str] = [] + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + self.prompts.append(llm_request.contents[0].parts[0].text) + text = "ID: rubric_0\nProperty: Uses appropriate tools.\nRationale: fake\nVerdict: yes" + yield LlmResponse(content=genai_types.Content(role="model", parts=[genai_types.Part(text=text)])) + + +@pytest.mark.skipif(not os.path.exists(HELM_TRACE), reason="samples/helm.json not available") +@pytest.mark.parametrize("builtin", [False, True], ids=["direct-evaluator", "builtin-metric"]) +async def test_replayed_tool_use_judge_sees_all_available_tools(monkeypatch, builtin): + (turn,) = extract_conversation(load_traces(HELM_TRACE)).turns + invocations = to_adk_invocations([turn]) + definitions = turn.llm_calls[0].tool_definitions + assert len(definitions) == 9 + assert {call.name for call in turn.tool_calls} == {"helm_list_releases"} + + metric_name = "rubric_based_tool_use_quality_v1" + rubrics = ["Uses appropriate tools."] + evaluator = builtin_metrics.get_evaluator( + builtin_metrics.build_eval_metric(metric_name, None, 0.5, rubrics=rubrics) + ) + judge = _ToolUseFakeJudge(model="fake-judge") + evaluator._judge_model = judge + if builtin: + monkeypatch.setattr(builtin_metrics, "get_evaluator", lambda _metric: evaluator) + result = await evaluate_builtin_metric(metric_name, invocations, None, None, 0.5, rubrics=rubrics) + assert result.error is None + assert result.score == 1.0 + else: + result = await evaluator.evaluate_invocations(actual_invocations=invocations) + assert result.overall_score == 1.0 + + available_tools = json.loads(judge.prompts[0].split("\n")[1].split("")[0]) + declarations = { + declaration["name"]: declaration + for tool in available_tools["tool_declarations"]["helm_agent"] + for declaration in tool["function_declarations"] + } + assert set(declarations) == {definition["name"] for definition in definitions} + for definition in definitions: + declaration = declarations[definition["name"]] + assert declaration["description"] == definition["description"] + assert declaration["parameters_json_schema"] == definition["parameters"] From 9b27b3a123823bd12be02c0f6b24f881a28d3ec7 Mon Sep 17 00:00:00 2001 From: erauner Date: Thu, 29 Oct 2026 20:23:15 -0500 Subject: [PATCH 2/3] fix(genai): preserve coherent ADK tool rounds --- src/agentevals/genai/extract.py | 27 +++- tests/genai/test_adk_legacy_wrappers.py | 158 ++++++++++++++++++++++++ 2 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 tests/genai/test_adk_legacy_wrappers.py diff --git a/src/agentevals/genai/extract.py b/src/agentevals/genai/extract.py index 3a3dad3..b72ac96 100644 --- a/src/agentevals/genai/extract.py +++ b/src/agentevals/genai/extract.py @@ -301,6 +301,29 @@ def _logical_call(analysis: TraceAnalysis, chain_ids: list[str]) -> LlmCall: else None ), ) + inputs = _first(chain, lambda v: v.messages(sc.INPUT_MESSAGES)) or [] + outputs = _first(chain, lambda v: v.messages(sc.OUTPUT_MESSAGES)) or [] + if all(leaf.sources.get(key) == "legacy_indexed" for key in (sc.INPUT_MESSAGES, sc.OUTPUT_MESSAGES)): + for view in chain[1:]: + if not view.is_adk or any( + view.sources.get(key) != "adk" for key in (sc.INPUT_MESSAGES, sc.OUTPUT_MESSAGES) + ): + continue + adk_inputs, adk_outputs = view.messages(sc.INPUT_MESSAGES), view.messages(sc.OUTPUT_MESSAGES) + if not adk_inputs or not adk_outputs: + continue + historical = tool_calls_of(adk_inputs) + requested = [*historical, *tool_calls_of(adk_outputs)] + answered = tool_responses_of(adk_inputs) + if not requested or not all( + isinstance(part["id"], str) and part["id"].strip() for part in [*requested, *answered] + ): + continue + if not {part["id"] for part in answered} <= {part["id"] for part in historical}: + continue + # ADK remaps provider tool IDs; select its captured request and response together. + inputs, outputs = adk_inputs, adk_outputs + break return LlmCall( ref=leaf.span.ref, wrapper_ref=chain[-1].span.ref if len(chain) > 1 else None, @@ -313,8 +336,8 @@ def _logical_call(analysis: TraceAnalysis, chain_ids: list[str]) -> LlmCall: usage=_first(chain, _usage), finish_reasons=[str(r) for r in finish or []], system_instructions=_first(chain, lambda v: v.messages(sc.SYSTEM_INSTRUCTIONS)) or [], - input_messages=_first(chain, lambda v: v.messages(sc.INPUT_MESSAGES)) or [], - output_messages=_first(chain, lambda v: v.messages(sc.OUTPUT_MESSAGES)) or [], + input_messages=inputs, + output_messages=outputs, tool_definitions=_first(chain, lambda v: v.messages(sc.TOOL_DEFINITIONS)) or [], status=status, error_type=error_type, diff --git a/tests/genai/test_adk_legacy_wrappers.py b/tests/genai/test_adk_legacy_wrappers.py new file mode 100644 index 0000000..f35a573 --- /dev/null +++ b/tests/genai/test_adk_legacy_wrappers.py @@ -0,0 +1,158 @@ +"""Synthetic nested ADK/OpenLLMetry rounds; no recorded tasks, prompts or results.""" + +from __future__ import annotations + +from copy import deepcopy + +from agentevals.genai.extract import extract_conversation +from agentevals.genai.messages import text_of +from agentevals.otel.decode import decode_json_document +from agentevals.otel.model import build_traces +from otel.builders import assistant, request, sid, span, user + + +def synthetic_rounds(mode: str = "valid") -> list: + spans, history = [], [{"role": "user", "parts": [{"text": "Read four synthetic items."}]}] + for index in range(3): + start, outer, leaf = index * 3_000 + 1_000, f"adk-{index}", f"provider-{index}" + calls = ( + [{"id": f"adk-{index}-{n}", "name": "read_item", "args": {"item": index * 2 + n}} for n in range(2)] + if index < 2 + else [] + ) + output = { + "role": "model", + "parts": [{"function_call": call} for call in calls] if calls else [{"text": "Four items read."}], + } + contents = deepcopy(history) + if mode == "missing_ids": + for message in contents: + for part in message["parts"]: + if "function_response" in part: + part["function_response"].pop("id", None) + if mode == "conflicting_ids": + for message in contents: + for part in message["parts"]: + if "function_response" in part: + part["function_response"]["id"] = "unrequested-" + part["function_response"]["id"] + if mode == "conflicting_transition" and index == 1: + for message in contents: + for part in message["parts"]: + for key in ("function_call", "function_response"): + if key in part: + part[key]["id"] = "other-" + part[key]["id"] + if mode == "missing_call_ids": + output = deepcopy(output) + for message in [*contents, output]: + for part in message["parts"]: + if "function_call" in part: + part["function_call"].pop("id", None) + attrs = { + "gcp.vertex.agent.invocation_id": "synthetic-invocation", + "gcp.vertex.agent.session_id": "synthetic-session", + "gcp.vertex.agent.llm_request": {"contents": contents}, + "gcp.vertex.agent.llm_response": {"content": output}, + } + if mode == "partial_adk_pair" and index == 1: + attrs.pop("gcp.vertex.agent.llm_response") + if mode == "non_adk_outer": + attrs = { + "gen_ai.operation.name": "generate_content", + "gen_ai.input.messages": [user("Outer unrelated content")], + "gen_ai.output.messages": [assistant("Outer unrelated reply")], + } + spans.append(span(span_id=outer, name="call_llm", attrs=attrs, start=start, end=start + 2_000)) + provider = { + "gen_ai.request.model": "synthetic-model", + "gen_ai.system": "openai", + "gen_ai.prompt.0.role": "user", + "gen_ai.prompt.0.content": "Read four synthetic items.", + "gen_ai.completion.0.role": "assistant", + "gen_ai.completion.0.content": "" if calls else "Four items read.", + "gen_ai.usage.input_tokens": 10, + "gen_ai.usage.output_tokens": 2, + } + for n, call in enumerate(calls): + for key, value in {"id": f"provider-{index}-{n}", "name": call["name"], "arguments": call["args"]}.items(): + provider[f"gen_ai.completion.0.tool_calls.{n}.{key}"] = value + if mode == "canonical_leaf": + provider["gen_ai.input.messages"] = [user("Canonical leaf input")] + provider["gen_ai.output.messages"] = ( + [ + assistant( + tool_calls=[ + (f"provider-{index}-{n}", call["name"], call["args"]) for n, call in enumerate(calls) + ] + ) + ] + if calls + else [assistant("Canonical leaf answer")] + ) + spans.append( + span(span_id=leaf, parent=outer, name="openai.chat", attrs=provider, start=start + 100, end=start + 500) + ) + for n, call in enumerate(calls): + spans.append( + span( + span_id=f"tool-{index}-{n}", + parent=outer, + name="execute_tool read_item", + attrs={ + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "read_item", + "gen_ai.tool.call.id": call["id"], + "gen_ai.tool.call.arguments": call["args"], + "gen_ai.tool.call.result": {"value": index * 2 + n}, + }, + start=start + 600 + n * 100, + end=start + 650 + n * 100, + ) + ) + history.extend( + [ + output, + { + "role": "user", + "parts": [ + { + "function_response": { + "id": call["id"], + "name": call["name"], + "response": {"value": index * 2 + n}, + } + } + for n, call in enumerate(calls) + ], + }, + ] + ) + decoded = decode_json_document(request(spans), strict=True) + traces, _ = build_traces(decoded.spans, decoded.logs) + return traces + + +def test_coherent_adk_rounds_are_one_complete_turn(): + turns = extract_conversation(synthetic_rounds()).turns + assert len(turns) == 1 + turn = turns[0] + assert len(turn.llm_calls) == 3 and len(turn.tool_calls) == 4 + assert [call.ref.span_id for call in turn.llm_calls] == [sid(f"provider-{i}") for i in range(3)] + assert [call.wrapper_ref.span_id for call in turn.llm_calls] == [sid(f"adk-{i}") for i in range(3)] + assert turn.usage.input_tokens == 30 and turn.usage.output_tokens == 6 + assert [tool.call_id for tool in turn.tool_calls] == [f"adk-{i}-{j}" for i in range(2) for j in range(2)] + assert [tool.result for tool in turn.tool_calls] == [{"value": i} for i in range(4)] + assert text_of(turn.final_output) == "Four items read." and not turn.warnings + + +def test_missing_or_conflicting_adk_ids_do_not_collapse_rounds(): + for mode in ("missing_ids", "missing_call_ids", "conflicting_ids"): + assert len(extract_conversation(synthetic_rounds(mode)).turns) == 3, mode + + +def test_internally_coherent_different_ids_still_fail_transition_correlation(): + assert len(extract_conversation(synthetic_rounds("conflicting_transition")).turns) > 1 + + +def test_canonical_leaf_non_adk_and_partial_pairs_preserve_turns(): + for mode in ("canonical_leaf", "non_adk_outer", "partial_adk_pair"): + assert len(extract_conversation(synthetic_rounds(mode)).turns) == 3, mode From cfbf71fac7e532375f2a406668ed3fcedaf9b880 Mon Sep 17 00:00:00 2001 From: erauner Date: Mon, 2 Nov 2026 15:08:22 -0600 Subject: [PATCH 3/3] refactor(adk-bridge): use semconv fixtures for captured tools --- src/agentevals/adk_bridge.py | 2 + src/agentevals/genai/extract.py | 27 +--- tests/genai/test_adk_legacy_wrappers.py | 158 ------------------------ tests/test_adk_bridge.py | 62 ++++++++-- tests/test_builtin_metrics.py | 17 +-- 5 files changed, 64 insertions(+), 202 deletions(-) delete mode 100644 tests/genai/test_adk_legacy_wrappers.py diff --git a/src/agentevals/adk_bridge.py b/src/agentevals/adk_bridge.py index 4f0ee97..3a2270c 100644 --- a/src/agentevals/adk_bridge.py +++ b/src/agentevals/adk_bridge.py @@ -107,6 +107,8 @@ def _app_details(turn: Turn) -> AppDetails | None: tools_by_agent: dict[str, dict[str, genai_types.FunctionDeclaration]] = {} for call in turn.llm_calls: for definition in call.tool_definitions: + if definition.get("type") != "function": + continue name = definition.get("name") if not isinstance(name, str) or not name: continue diff --git a/src/agentevals/genai/extract.py b/src/agentevals/genai/extract.py index b72ac96..3a3dad3 100644 --- a/src/agentevals/genai/extract.py +++ b/src/agentevals/genai/extract.py @@ -301,29 +301,6 @@ def _logical_call(analysis: TraceAnalysis, chain_ids: list[str]) -> LlmCall: else None ), ) - inputs = _first(chain, lambda v: v.messages(sc.INPUT_MESSAGES)) or [] - outputs = _first(chain, lambda v: v.messages(sc.OUTPUT_MESSAGES)) or [] - if all(leaf.sources.get(key) == "legacy_indexed" for key in (sc.INPUT_MESSAGES, sc.OUTPUT_MESSAGES)): - for view in chain[1:]: - if not view.is_adk or any( - view.sources.get(key) != "adk" for key in (sc.INPUT_MESSAGES, sc.OUTPUT_MESSAGES) - ): - continue - adk_inputs, adk_outputs = view.messages(sc.INPUT_MESSAGES), view.messages(sc.OUTPUT_MESSAGES) - if not adk_inputs or not adk_outputs: - continue - historical = tool_calls_of(adk_inputs) - requested = [*historical, *tool_calls_of(adk_outputs)] - answered = tool_responses_of(adk_inputs) - if not requested or not all( - isinstance(part["id"], str) and part["id"].strip() for part in [*requested, *answered] - ): - continue - if not {part["id"] for part in answered} <= {part["id"] for part in historical}: - continue - # ADK remaps provider tool IDs; select its captured request and response together. - inputs, outputs = adk_inputs, adk_outputs - break return LlmCall( ref=leaf.span.ref, wrapper_ref=chain[-1].span.ref if len(chain) > 1 else None, @@ -336,8 +313,8 @@ def _logical_call(analysis: TraceAnalysis, chain_ids: list[str]) -> LlmCall: usage=_first(chain, _usage), finish_reasons=[str(r) for r in finish or []], system_instructions=_first(chain, lambda v: v.messages(sc.SYSTEM_INSTRUCTIONS)) or [], - input_messages=inputs, - output_messages=outputs, + input_messages=_first(chain, lambda v: v.messages(sc.INPUT_MESSAGES)) or [], + output_messages=_first(chain, lambda v: v.messages(sc.OUTPUT_MESSAGES)) or [], tool_definitions=_first(chain, lambda v: v.messages(sc.TOOL_DEFINITIONS)) or [], status=status, error_type=error_type, diff --git a/tests/genai/test_adk_legacy_wrappers.py b/tests/genai/test_adk_legacy_wrappers.py deleted file mode 100644 index f35a573..0000000 --- a/tests/genai/test_adk_legacy_wrappers.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Synthetic nested ADK/OpenLLMetry rounds; no recorded tasks, prompts or results.""" - -from __future__ import annotations - -from copy import deepcopy - -from agentevals.genai.extract import extract_conversation -from agentevals.genai.messages import text_of -from agentevals.otel.decode import decode_json_document -from agentevals.otel.model import build_traces -from otel.builders import assistant, request, sid, span, user - - -def synthetic_rounds(mode: str = "valid") -> list: - spans, history = [], [{"role": "user", "parts": [{"text": "Read four synthetic items."}]}] - for index in range(3): - start, outer, leaf = index * 3_000 + 1_000, f"adk-{index}", f"provider-{index}" - calls = ( - [{"id": f"adk-{index}-{n}", "name": "read_item", "args": {"item": index * 2 + n}} for n in range(2)] - if index < 2 - else [] - ) - output = { - "role": "model", - "parts": [{"function_call": call} for call in calls] if calls else [{"text": "Four items read."}], - } - contents = deepcopy(history) - if mode == "missing_ids": - for message in contents: - for part in message["parts"]: - if "function_response" in part: - part["function_response"].pop("id", None) - if mode == "conflicting_ids": - for message in contents: - for part in message["parts"]: - if "function_response" in part: - part["function_response"]["id"] = "unrequested-" + part["function_response"]["id"] - if mode == "conflicting_transition" and index == 1: - for message in contents: - for part in message["parts"]: - for key in ("function_call", "function_response"): - if key in part: - part[key]["id"] = "other-" + part[key]["id"] - if mode == "missing_call_ids": - output = deepcopy(output) - for message in [*contents, output]: - for part in message["parts"]: - if "function_call" in part: - part["function_call"].pop("id", None) - attrs = { - "gcp.vertex.agent.invocation_id": "synthetic-invocation", - "gcp.vertex.agent.session_id": "synthetic-session", - "gcp.vertex.agent.llm_request": {"contents": contents}, - "gcp.vertex.agent.llm_response": {"content": output}, - } - if mode == "partial_adk_pair" and index == 1: - attrs.pop("gcp.vertex.agent.llm_response") - if mode == "non_adk_outer": - attrs = { - "gen_ai.operation.name": "generate_content", - "gen_ai.input.messages": [user("Outer unrelated content")], - "gen_ai.output.messages": [assistant("Outer unrelated reply")], - } - spans.append(span(span_id=outer, name="call_llm", attrs=attrs, start=start, end=start + 2_000)) - provider = { - "gen_ai.request.model": "synthetic-model", - "gen_ai.system": "openai", - "gen_ai.prompt.0.role": "user", - "gen_ai.prompt.0.content": "Read four synthetic items.", - "gen_ai.completion.0.role": "assistant", - "gen_ai.completion.0.content": "" if calls else "Four items read.", - "gen_ai.usage.input_tokens": 10, - "gen_ai.usage.output_tokens": 2, - } - for n, call in enumerate(calls): - for key, value in {"id": f"provider-{index}-{n}", "name": call["name"], "arguments": call["args"]}.items(): - provider[f"gen_ai.completion.0.tool_calls.{n}.{key}"] = value - if mode == "canonical_leaf": - provider["gen_ai.input.messages"] = [user("Canonical leaf input")] - provider["gen_ai.output.messages"] = ( - [ - assistant( - tool_calls=[ - (f"provider-{index}-{n}", call["name"], call["args"]) for n, call in enumerate(calls) - ] - ) - ] - if calls - else [assistant("Canonical leaf answer")] - ) - spans.append( - span(span_id=leaf, parent=outer, name="openai.chat", attrs=provider, start=start + 100, end=start + 500) - ) - for n, call in enumerate(calls): - spans.append( - span( - span_id=f"tool-{index}-{n}", - parent=outer, - name="execute_tool read_item", - attrs={ - "gen_ai.operation.name": "execute_tool", - "gen_ai.tool.name": "read_item", - "gen_ai.tool.call.id": call["id"], - "gen_ai.tool.call.arguments": call["args"], - "gen_ai.tool.call.result": {"value": index * 2 + n}, - }, - start=start + 600 + n * 100, - end=start + 650 + n * 100, - ) - ) - history.extend( - [ - output, - { - "role": "user", - "parts": [ - { - "function_response": { - "id": call["id"], - "name": call["name"], - "response": {"value": index * 2 + n}, - } - } - for n, call in enumerate(calls) - ], - }, - ] - ) - decoded = decode_json_document(request(spans), strict=True) - traces, _ = build_traces(decoded.spans, decoded.logs) - return traces - - -def test_coherent_adk_rounds_are_one_complete_turn(): - turns = extract_conversation(synthetic_rounds()).turns - assert len(turns) == 1 - turn = turns[0] - assert len(turn.llm_calls) == 3 and len(turn.tool_calls) == 4 - assert [call.ref.span_id for call in turn.llm_calls] == [sid(f"provider-{i}") for i in range(3)] - assert [call.wrapper_ref.span_id for call in turn.llm_calls] == [sid(f"adk-{i}") for i in range(3)] - assert turn.usage.input_tokens == 30 and turn.usage.output_tokens == 6 - assert [tool.call_id for tool in turn.tool_calls] == [f"adk-{i}-{j}" for i in range(2) for j in range(2)] - assert [tool.result for tool in turn.tool_calls] == [{"value": i} for i in range(4)] - assert text_of(turn.final_output) == "Four items read." and not turn.warnings - - -def test_missing_or_conflicting_adk_ids_do_not_collapse_rounds(): - for mode in ("missing_ids", "missing_call_ids", "conflicting_ids"): - assert len(extract_conversation(synthetic_rounds(mode)).turns) == 3, mode - - -def test_internally_coherent_different_ids_still_fail_transition_correlation(): - assert len(extract_conversation(synthetic_rounds("conflicting_transition")).turns) > 1 - - -def test_canonical_leaf_non_adk_and_partial_pairs_preserve_turns(): - for mode in ("canonical_leaf", "non_adk_outer", "partial_adk_pair"): - assert len(extract_conversation(synthetic_rounds(mode)).turns) == 3, mode diff --git a/tests/test_adk_bridge.py b/tests/test_adk_bridge.py index 8bc1a58..d757ae5 100644 --- a/tests/test_adk_bridge.py +++ b/tests/test_adk_bridge.py @@ -16,19 +16,23 @@ def _call(definitions, agent_name=None): return LlmCall(ref=REF, operation="chat", agent_name=agent_name, tool_definitions=definitions) -@pytest.mark.parametrize("schema_type", ["object", "OBJECT"], ids=["json-schema", "adk-schema"]) -def test_captured_tools_include_uncalled_tools_and_keep_first_definition(schema_type): - schema = {"type": schema_type, "properties": {"query": {"type": "string"}}} +def test_captured_tools_include_uncalled_tools_and_keep_first_definition(): + schema = {"type": "object", "properties": {"query": {"type": "string"}}} definitions = [ - {"name": "search", "description": "Search the available evidence.", "parameters": schema}, - {"name": "lookup", "description": "Look up an item.", "parameters": {}}, + {"type": "function", "name": "search", "description": "Search the available evidence.", "parameters": schema}, + {"type": "function", "name": "lookup", "description": "Look up an item.", "parameters": {}}, ] turn = Turn( ref=REF, agent_name="researcher", llm_calls=[ _call(definitions), - _call([{"name": "search", "description": "Changed", "parameters": {}}, {"name": "finish"}]), + _call( + [ + {"type": "function", "name": "search", "description": "Changed", "parameters": {}}, + {"type": "function", "name": "finish"}, + ] + ), ], tool_calls=[ToolCall(name="search")], ) @@ -51,15 +55,15 @@ def test_captured_tools_include_uncalled_tools_and_keep_first_definition(schema_ "call_agent,turn_agent,expected", [("worker", "root", "worker"), (None, "root", "root"), (None, None, "agent")] ) def test_tool_agent_name_falls_back_to_turn_then_agent(call_agent, turn_agent, expected): - turn = Turn(ref=REF, agent_name=turn_agent, llm_calls=[_call([{"name": "search"}], call_agent)]) + turn = Turn(ref=REF, agent_name=turn_agent, llm_calls=[_call([{"type": "function", "name": "search"}], call_agent)]) assert list(to_adk_invocation(turn).app_details.agent_details) == [expected] def test_agents_keep_separate_tools_and_do_not_include_system_instructions(): - first = _call([{"name": "lookup", "description": "Search documents."}], "researcher") + first = _call([{"type": "function", "name": "lookup", "description": "Search documents."}], "researcher") first.system_instructions = [{"type": "text", "content": "Private instructions."}] - second = _call([{"name": "lookup", "description": "Search billing records."}], "billing") + second = _call([{"type": "function", "name": "lookup", "description": "Search billing records."}], "billing") invocation = to_adk_invocation(Turn(ref=REF, llm_calls=[first, second])) @@ -71,7 +75,23 @@ def test_agents_keep_separate_tools_and_do_not_include_system_instructions(): assert "app_details" not in invocation_to_dict(invocation) -@pytest.mark.parametrize("calls", [[], [_call([])], [_call([{}, {"name": ""}, {"name": None}, {"name": 7}])]]) +@pytest.mark.parametrize( + "calls", + [ + [], + [_call([])], + [ + _call( + [ + {}, + {"type": "function", "name": ""}, + {"type": "function", "name": None}, + {"type": "function", "name": 7}, + ] + ) + ], + ], +) def test_missing_or_unusable_definitions_leave_names_only_fallback(calls): turn = Turn(ref=REF, llm_calls=calls, tool_calls=[ToolCall(name="search")]) invocation = to_adk_invocation(turn) @@ -86,7 +106,9 @@ def test_missing_or_unusable_definitions_leave_names_only_fallback(calls): def test_malformed_description_does_not_break_trace_conversion(): - turn = Turn(ref=REF, llm_calls=[_call([{"name": "search", "description": {"unexpected": "object"}}])]) + turn = Turn( + ref=REF, llm_calls=[_call([{"type": "function", "name": "search", "description": {"unexpected": "object"}}])] + ) (declaration,) = ( to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations @@ -94,3 +116,21 @@ def test_malformed_description_does_not_break_trace_conversion(): assert declaration.name == "search" assert declaration.description is None + + +@pytest.mark.parametrize( + "definition", + [ + {"type": "google_search", "name": "web"}, + {"type": "custom", "name": "custom"}, + {"name": "missing_type"}, + ], +) +def test_only_function_definitions_are_converted(definition): + turn = Turn(ref=REF, llm_calls=[_call([definition, {"type": "function", "name": "lookup"}])]) + + (declaration,) = ( + to_adk_invocation(turn).app_details.agent_details["agent"].tool_declarations[0].function_declarations + ) + + assert declaration.name == "lookup" diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index 8dcc76b..585d7f7 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -312,16 +312,17 @@ async def generate_content_async( yield LlmResponse(content=genai_types.Content(role="model", parts=[genai_types.Part(text=text)])) -@pytest.mark.skipif(not os.path.exists(HELM_TRACE), reason="samples/helm.json not available") +@pytest.mark.parametrize("metric_name", ["rubric_based_tool_use_quality_v1", "rubric_based_final_response_quality_v1"]) @pytest.mark.parametrize("builtin", [False, True], ids=["direct-evaluator", "builtin-metric"]) -async def test_replayed_tool_use_judge_sees_all_available_tools(monkeypatch, builtin): - (turn,) = extract_conversation(load_traces(HELM_TRACE)).turns +async def test_replayed_judge_sees_all_available_tools(monkeypatch, builtin, metric_name): + (turn,) = extract_conversation( + load_traces(os.path.join(os.path.dirname(__file__), "fixtures/otlp/adk_py/v2proj_span_only/S2.json")) + ).turns invocations = to_adk_invocations([turn]) definitions = turn.llm_calls[0].tool_definitions - assert len(definitions) == 9 - assert {call.name for call in turn.tool_calls} == {"helm_list_releases"} + assert {definition["name"] for definition in definitions} == {"get_weather", "get_time"} + assert {call.name for call in turn.tool_calls} == {"get_weather", "get_time"} - metric_name = "rubric_based_tool_use_quality_v1" rubrics = ["Uses appropriate tools."] evaluator = builtin_metrics.get_evaluator( builtin_metrics.build_eval_metric(metric_name, None, 0.5, rubrics=rubrics) @@ -337,10 +338,10 @@ async def test_replayed_tool_use_judge_sees_all_available_tools(monkeypatch, bui result = await evaluator.evaluate_invocations(actual_invocations=invocations) assert result.overall_score == 1.0 - available_tools = json.loads(judge.prompts[0].split("\n")[1].split("")[0]) + available_tools = json.loads(judge.prompts[0].rsplit("\n", 1)[1].split("")[0]) declarations = { declaration["name"]: declaration - for tool in available_tools["tool_declarations"]["helm_agent"] + for tool in available_tools["tool_declarations"]["weather_agent"] for declaration in tool["function_declarations"] } assert set(declarations) == {definition["name"] for definition in definitions}