From e6cba3c2ff75c5a696b0d306f8a85bf4b4b6983a Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Wed, 11 Nov 2026 13:31:40 +0100 Subject: [PATCH 1/2] feat(gooddata-eval): populate best_run_latency_s for agentic evaluators best_run_latency_s was only ever populated on the single-shot path; every agentic kind (agentic_*, vis_agentic) always reported it as null. Adds per-run wall-clock timing to all agentic evaluators and wires the rank-selected run's latency into ItemReport. Three findings from review (dropped conditional checks in alert_skill's best-run detail, over-broad except Exception in anomaly_detection/what_if) are tracked separately: https://github.com/gooddata/gooddata-python-sdk/issues/1942 jira: trivial risk: low Co-Authored-By: Claude Sonnet 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 15 +++ .../gooddata_eval/core/agentic/_outcome.py | 123 ++++++++++++++++++ .../gooddata_eval/core/agentic/alert_skill.py | 66 +++++++--- .../core/agentic/anomaly_detection.py | 87 ++++++++----- .../core/agentic/conversation.py | 64 +++++---- .../core/agentic/dashboard_skill.py | 94 ++++++++++--- .../core/agentic/general_question.py | 48 ++++--- .../gooddata_eval/core/agentic/guardrail.py | 45 +++++-- .../gooddata_eval/core/agentic/kda_skill.py | 56 +++++--- .../core/agentic/metric_skill.py | 55 ++++---- .../gooddata_eval/core/agentic/search_tool.py | 44 ++++--- .../core/agentic/visualization.py | 50 ++++--- .../src/gooddata_eval/core/agentic/what_if.py | 89 ++++++++----- .../core/evaluators/_llm_judge.py | 1 + .../src/gooddata_eval/core/models.py | 7 + .../src/gooddata_eval/core/timing.py | 10 ++ .../tests/test_agentic_anomaly_detection.py | 12 +- .../tests/test_agentic_dashboard_skill.py | 56 +++++++- .../tests/test_agentic_general_question.py | 6 + .../tests/test_agentic_kda_skill.py | 2 + .../tests/test_agentic_metric_skill.py | 1 + .../tests/test_agentic_runner.py | 68 +++++++++- .../tests/test_agentic_search_tool.py | 2 + .../tests/test_agentic_what_if.py | 14 +- packages/gooddata-eval/tests/test_timing.py | 8 ++ 25 files changed, 773 insertions(+), 250 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 66bcf45c8..1aa4d3184 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -428,6 +428,18 @@ def _apply_timings(item_report: ItemReport, timings: Any) -> None: item_report.simulated_user_latency_s = timings.simulated_user_s +def _apply_best_run_latency(item_report: ItemReport, source: Any) -> None: + """Copy the rank-selected run's own wall time, mirroring the single-shot path's + ``ItemReport.best_run_latency_s`` (see ``core/runner.py``'s ``_run_one_item``). + + Kinds not yet wired to measure it pass None and keep the field at its own None + default rather than reporting an invented number. + """ + best_run_latency_s = getattr(source, "best_run_latency_s", None) + if best_run_latency_s is not None: + item_report.best_run_latency_s = best_run_latency_s + + def run_agentic_items( items: list[DatasetItem], host: str, @@ -525,6 +537,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.response_id = response_id item_report.best_detail = detail or {} _apply_timings(item_report, getattr(outcome, "timings", None)) + _apply_best_run_latency(item_report, outcome) _apply_run_counts(item_report, outcome) _apply_failed_runs(item_report, outcome) _apply_run_latencies(item_report, outcome) @@ -536,6 +549,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.response_id = getattr(exc, "response_id", None) item_report.best_detail = getattr(exc, "detail", None) or {} _apply_timings(item_report, getattr(exc, "timings", None)) + _apply_best_run_latency(item_report, exc) _apply_run_counts(item_report, exc) _apply_failed_runs(item_report, exc) _apply_run_latencies(item_report, exc) @@ -579,6 +593,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: # runs and then raised would otherwise show up in the per-response table and not in # the per-conversation one. traces.discard() + _apply_best_run_latency(item_report, exc) finally: item_report.latency_s = time.perf_counter() - t0 item_report.trace_latencies_s = traces.stop() diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py new file mode 100644 index 000000000..576aee838 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py @@ -0,0 +1,123 @@ +# (C) 2026 GoodData Corporation +"""The common tail of every ``evaluate_agentic_*`` function. + +Before this module, each of the agentic evaluators hand-wrote the same block: copy +``reasoning_steps``/``conversation_id``/``response_id``/``detail``/``runs_passed``/ +``runs_effective``/``best_run_latency_s`` (and sometimes ``timings``) either onto a +raised exception or into the returned ``AgenticEvalOutcome``. Eleven independent copies +of the same ~7 lines meant a new universal field needed eleven edits, not one, and +nothing failed loudly when a copy was missed -- exactly what happened to ``timings`` +(see ``AgenticAssertionError``'s docstring: "while it lived in eight copies, two +declared timings and six did not") and again to ``best_run_latency_s`` (every agentic +kind silently reported it as ``null`` until this was noticed and fixed kind-by-kind). + +``agentic_detail`` has the same motivation for the ``detail`` dict's timeline fields: +``timeline_detail`` builds both ``latency_breakdown`` and ``tool_calls`` from the same +events so they stay index-aligned, but roughly half the evaluators called +``build_latency_breakdown`` directly and silently never got a ``tool_calls`` key. +""" + +from __future__ import annotations + +from typing import Any, NoReturn + +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + timeline_detail, +) +from gooddata_eval.core.timing import PhaseTimings + +__all__ = ["agentic_detail", "agentic_success", "raise_agentic_failure"] + + +def agentic_detail( + tool_call_events: list[ToolCallEvent], + reasoning_step_events: list[ReasoningStepEvent] | None, + **kind_specific: Any, +) -> dict: + """A kind's full ``detail`` dict: its own fields plus the universal timeline ones. + + ``kind_specific`` comes first in the merge so a kind can never accidentally shadow + ``latency_breakdown``/``tool_calls`` with a same-named field of its own. + """ + return {**kind_specific, **timeline_detail(tool_call_events, reasoning_step_events)} + + +def raise_agentic_failure( + exception_cls: type[AgenticAssertionError], + message: str, + *, + reasoning_steps: list[str], + conversation_id: str, + response_id: str | None, + detail: dict, + runs_passed: int, + runs_effective: int, + best_run_latency_s: float | None, + timings: PhaseTimings | None = None, + failed_runs: list[dict] | None = None, + run_latencies_s: list[float] | None = None, +) -> NoReturn: + """Build ``exception_cls(message)`` with every common field attached, and raise it. + + ``exception_cls`` must be an ``AgenticAssertionError`` subclass -- that base class is + what declares these fields as legal targets (see its docstring). A kind whose own + "no verdict at all" branch raises a bare ``JudgeResponseError`` instead (not a subclass) + does not go through this helper for that branch. + + ``timings`` stays optional: only the three kinds that track per-run ``PhaseTimings`` + (general_question, metric_skill, dashboard_skill) pass one -- the rest keep their + own ``PhaseTimings()`` zero default, same as before this helper existed. ``failed_runs`` + and ``run_latencies_s`` are likewise optional: not every kind has built them by the + time it raises. + """ + exc = exception_cls(message) + exc.reasoning_steps = reasoning_steps + exc.conversation_id = conversation_id + exc.response_id = response_id + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective + exc.best_run_latency_s = best_run_latency_s + if timings is not None: + exc.timings = timings + if failed_runs is not None: + exc.failed_runs = failed_runs + if run_latencies_s is not None: + exc.run_latencies_s = run_latencies_s + raise exc + + +def agentic_success( + *, + reasoning_steps: list[str], + conversation_id: str | None, + response_id: str | None, + detail: dict, + runs_passed: int, + runs_effective: int, + best_run_latency_s: float | None, + timings: PhaseTimings | None = None, + failed_runs: list[dict] | None = None, + run_latencies_s: list[float] | None = None, +) -> AgenticEvalOutcome: + """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape.""" + kwargs: dict[str, Any] = { + "reasoning_steps": reasoning_steps, + "conversation_id": conversation_id, + "response_id": response_id, + "detail": detail, + "runs_passed": runs_passed, + "runs_effective": runs_effective, + "best_run_latency_s": best_run_latency_s, + } + if timings is not None: + kwargs["timings"] = timings + if failed_runs is not None: + kwargs["failed_runs"] = failed_runs + if run_latencies_s is not None: + kwargs["run_latencies_s"] = run_latencies_s + return AgenticEvalOutcome(**kwargs) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 6c5a55d67..f1f532af3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -23,6 +23,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -700,7 +701,8 @@ class AlertRunResult: # also report operator/threshold/metric/recipients as False. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED turns_used: int = 0 - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 @@ -975,6 +977,7 @@ def run_agentic_alert_skill( def _run_once(conv_id: str) -> AlertRunResult: alert_id_to_delete: str | None = None + run_started = time.monotonic() try: alert_id: str | None = None actual_args: dict = {} @@ -1260,9 +1263,25 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.eval - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + alert_created=ev.alert_created, + operator_correct=ev.operator_correct, + threshold_correct=ev.threshold_correct, + trigger_correct=ev.trigger_correct, + filters_correct=ev.filters_correct, + metric_correct=ev.metric_correct, + recipients_correct=ev.recipients_correct, + attributes_correct=ev.attributes_correct, + granularity_correct=ev.granularity_correct, + actual_alert_arguments=best.actual_alert_arguments, + # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + turns_used=best.turns_used, + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -1273,29 +1292,34 @@ def _write_scores(ctx: RunTraceContext) -> None: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) - exc = AlertSkillAssertionError( + raise_agentic_failure( + AlertSkillAssertionError, f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " - + ", ".join(f"{name}={value}" for name, value in ev.checks().items()) - + f". Actual args: {best.actual_alert_arguments}" + f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " + f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " + f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, " + f"recipients_correct={ev.recipients_correct}, " + f"attributes_correct={ev.attributes_correct}, " + f"granularity_correct={ev.granularity_correct}. " + f"Actual args: {best.actual_alert_arguments}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py index 1a4238983..25f2281a0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py @@ -41,6 +41,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -58,9 +59,9 @@ AgenticAssertionError, AgenticEvalOutcome, ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, - build_latency_breakdown, shift_and_index_events, ) @@ -285,8 +286,14 @@ class AnomalyRunResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from + # turn_wall_clock_sec above, which is only the final triggering turn. run_latency_s: float = 0.0 + # Why the simulated-user loop stopped -- see LoopExit. `triggered=False` alone cannot + # separate a refusal from a run that hit max_iterations while still on track (matches + # kda_skill.py/alert_skill.py). + exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @dataclass @@ -371,6 +378,7 @@ def run_agentic_anomaly_detection( ) def _run_once(conv_id: str) -> AnomalyRunResult: + run_started = time.monotonic() viz_args: dict | None = None execute_result: dict | None = None turn_wall_clock_sec: float | None = None @@ -397,6 +405,10 @@ def _accumulate(result: ChatResult) -> None: all_reasoning_step_events.extend(result.reasoning_step_events or []) run_started = time.monotonic() + # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that + # simply runs out of range() is labelled correctly with no trailing else. + exit_reason = LoopExit.BUDGET_EXHAUSTED + for iteration in range(max_iterations): try: chat_result = client.send_message(conv_id, current_question) @@ -409,6 +421,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) viz_args, execute_result = _extract_anomaly_calls(all_tool_call_events) turn_completed = False + exit_reason = LoopExit.CHAT_ERROR break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -422,8 +435,10 @@ def _accumulate(result: ChatResult) -> None: if execute_result is not None: # The turn that ran the detection, not an earlier disambiguation turn. turn_wall_clock_sec = chat_result.turn_wall_clock_sec + exit_reason = LoopExit.SUCCESS break if not response_text: + exit_reason = LoopExit.AGENT_SILENT break if iteration >= max_iterations - 1: break @@ -432,6 +447,7 @@ def _accumulate(result: ChatResult) -> None: disambiguated = True except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run _log.warning("Simulated anomaly user reply failed for conversation %s: %s", conv_id, exc) + exit_reason = LoopExit.SIMULATED_USER_FAILED break return AnomalyRunResult( @@ -440,6 +456,7 @@ def _accumulate(result: ChatResult) -> None: actual_visualization=viz_args, actual_execute_result=execute_result, turn_wall_clock_sec=turn_wall_clock_sec, + exit_reason=exit_reason, reasoning_steps=reasoning_steps, response_id=response_id, tool_call_events=all_tool_call_events, @@ -493,25 +510,29 @@ class AnomalyDetectionAssertionError(AgenticAssertionError): def _detail(best: AnomalyRunResult) -> dict[str, Any]: ev = best.evaluation - return { - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "metric_correct": ev.metric_correct, - "granularity_correct": ev.granularity_correct, + return agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + triggered=ev.triggered, + executed=ev.executed, + success=ev.success, + turn_completed=ev.turn_completed, + metric_correct=ev.metric_correct, + granularity_correct=ev.granularity_correct, # Which content checks the fixture pinned -- without it a run that verified nothing # reads the same as one where everything matched. - "asserted": ev.asserted, - "disambiguated": ev.disambiguated, - "actual_metrics": sorted(_metric_uris(best.actual_visualization)), - "actual_granularity": _inferred_granularity(best.actual_visualization), + asserted=ev.asserted, + disambiguated=ev.disambiguated, + actual_metrics=sorted(_metric_uris(best.actual_visualization)), + actual_granularity=_inferred_granularity(best.actual_visualization), # Reported, never asserted: whether a real series contains anomalies is a property # of the data, so a fixture demanding some would fail on the next warehouse refresh. - "anomaly_point_count": _point_count(best.actual_execute_result), - "actual_execute_result": best.actual_execute_result, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + anomaly_point_count=_point_count(best.actual_execute_result), + actual_execute_result=best.actual_execute_result, + # Why the loop stopped. triggered=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + ) def evaluate_agentic_anomaly_detection( @@ -648,24 +669,28 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Analysed {detail['actual_metrics']} at {detail['actual_granularity']}. " f"Actual execute result: {best.actual_execute_result}." ) - exc = AnomalyDetectionAssertionError(message) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = len(summary.run_results) - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=len(summary.run_results), + raise_agentic_failure( + AnomalyDetectionAssertionError, + message, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, + ) + + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index e980958bc..92f106340 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -56,6 +56,7 @@ summarize_visualizations, ) from gooddata_eval.core.agentic._gate import log_gate_scores +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -82,7 +83,6 @@ ReasoningStepEvent, ToolCallEvent, shift_and_index_events, - timeline_detail, ) from gooddata_eval.core.scoring import ( check_filters, @@ -729,8 +729,10 @@ class ConversationResult: stalled_turns: int = 0 mode: ConversationMode = "legacy" judge_model: str | None = None - # The whole conversation's wall time, from just before its first message to the end of its - # last turn's scoring; conversation create/delete is not part of it. + # This conversation's own wall time, across every turn -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). There is no + # K-of-N selection for this kind (see run_agentic_conversation's docstring), so + # this is simply the one conversation's own time, not a "best of" pick. run_latency_s: float = 0.0 @@ -857,6 +859,7 @@ def run_agentic_conversation( A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode neither sends ``clarifying_answers`` nor lets the clarification checks fail the item, so it raises ``ValueError``. """ + run_started = time.monotonic() resolved_mode = resolve_conversation_mode(mode) graded = [t.turn_id for t in fixture.turns if t.expects_clarification is not None] if resolved_mode == "legacy" and graded: @@ -1307,21 +1310,22 @@ def run_agentic_conversation( def _conversation_detail(result: ConversationResult) -> dict: - return { - "mode": result.mode, - "full_skill_coverage": result.full_skill_coverage, - "conversation_success": result.conversation_success, - "context_success": result.context_success, - "context_kept_rate": result.context_kept_rate, - "turns_before_first_break": result.turns_before_first_break, - "lost_context_clarifications": result.lost_context_clarifications, - "stalled_turns": result.stalled_turns, - "total_clarification_turns": result.total_clarification_turns, - "max_clarification_turns": result.max_clarification_turns, - "judge_model": result.judge_model, - "turns": [tr.detail() for tr in result.turn_results], - **timeline_detail(result.tool_call_events, result.reasoning_step_events), - } + return agentic_detail( + result.tool_call_events, + result.reasoning_step_events, + mode=result.mode, + full_skill_coverage=result.full_skill_coverage, + conversation_success=result.conversation_success, + context_success=result.context_success, + context_kept_rate=result.context_kept_rate, + turns_before_first_break=result.turns_before_first_break, + lost_context_clarifications=result.lost_context_clarifications, + stalled_turns=result.stalled_turns, + total_clarification_turns=result.total_clarification_turns, + max_clarification_turns=result.max_clarification_turns, + judge_model=result.judge_model, + turns=[tr.detail() for tr in result.turn_results], + ) class ConversationAssertionError(AgenticAssertionError): @@ -1508,18 +1512,21 @@ def _write_scores(ctx: RunTraceContext) -> None: f"full_skill_coverage={result.full_skill_coverage}. " f"Failed turns: {legacy_failed}" ) - exc = ConversationAssertionError(message) - exc.reasoning_steps = result.reasoning_steps - exc.conversation_id = result.conversation_id - exc.response_id = result.response_id - exc.detail = detail # This kind takes no k and drives its fixture exactly once, whatever --runs asks # for. Saying so explicitly stops the report claiming K runs that never happened. - exc.runs_passed = 0 - exc.runs_effective = 1 - exc.run_latencies_s = [result.run_latency_s] - raise exc - return AgenticEvalOutcome( + raise_agentic_failure( + ConversationAssertionError, + message, + reasoning_steps=result.reasoning_steps, + conversation_id=result.conversation_id, + response_id=result.response_id, + detail=detail, + runs_passed=0, + runs_effective=1, + run_latencies_s=[result.run_latency_s], + best_run_latency_s=result.run_latency_s, + ) + return agentic_success( reasoning_steps=result.reasoning_steps, conversation_id=result.conversation_id, response_id=result.response_id, @@ -1527,4 +1534,5 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_passed=1, runs_effective=1, run_latencies_s=[result.run_latency_s], + best_run_latency_s=result.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index 1c7aaf771..2fafc30ba 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -19,6 +19,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -29,20 +30,21 @@ utc_now, ) from gooddata_eval.core.chat.render import render_answer_text -from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.langfuse.item_scope import item_scope from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, shift_and_index_events, ) from gooddata_eval.core.scoring import AAC_VISUALIZATION_TYPES, same_viz_type -from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings _DEFAULT_K = 1 # Matches kda_skill and visualization. The reply this loop sends is built from the expectation @@ -1708,6 +1710,12 @@ class DashboardRunResult: timings: PhaseTimings = field(default_factory=PhaseTimings) # This run's own wall time, from just before its chat call to the end of its scoring. run_latency_s: float = 0.0 + # Why the loop stopped -- see LoopExit. A dashboard not produced alone cannot tell a + # refusal from a run that hit max_iterations while still on track (matches + # kda_skill.py/alert_skill.py). CHAT_ERROR is reachable; SIMULATED_USER_FAILED is not -- + # this loop's follow-up is built from the expectation by build_simulated_reply, with no + # judge call that could fail. + exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @dataclass @@ -2024,12 +2032,43 @@ def _execute_single_dashboard_run( turn_offset = 0.0 tool_index_offset = 0 reasoning_index_offset = 0 + # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that + # simply runs out of range() (or the is_edit "no reply of its own" branch below, which + # is the same shape of outcome -- the agent answered, just not with the dashboard tool) + # is labelled correctly with no trailing else. + exit_reason = LoopExit.BUDGET_EXHAUSTED run_started = time.monotonic() for iteration in range(max_iterations): turns += 1 agent_started = time.monotonic() - chat_result = client.send_message(conversation_id, current_question) + try: + chat_result = client.send_message(conversation_id, current_question) + except ChatError as exc: + # Recorded rather than left to escape: this runs inside one of K runs, so an + # uncaught ChatError discarded every run already completed along with this one's + # exit_reason, and the item surfaced as a bare error with no per-run diagnosis. + # Matches metric_skill.py and conversation.py, which record it the same way. + timings.agent_s += time.monotonic() - agent_started + print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}") + partial = exc.partial_result + if partial is not None: + # Shifted and indexed exactly as a completed turn is, before anything reads + # the events: left raw, a late turn's call_ts restarts near zero and + # timeline_detail reports it as overlapping the first one. + reasoning_steps.extend(partial.reasoning_steps or []) + response_id = partial.response_id or response_id + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + partial, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(partial.tool_call_events or []) + all_reasoning_step_events.extend(partial.reasoning_step_events or []) + steps += partial.reasoning_step_count + exit_reason = LoopExit.CHAT_ERROR + break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed reasoning_steps.extend(chat_result.reasoning_steps or []) @@ -2071,9 +2110,11 @@ def _execute_single_dashboard_run( dashboard_part = _extract_dashboard_part(chat_result, "dashboard") if is_edit: patch_part = _extract_dashboard_part(chat_result, _PATCH_TYPE) + exit_reason = LoopExit.SUCCESS break if not response_text and not chat_result.tool_call_events: + exit_reason = LoopExit.AGENT_SILENT break if not expects_dashboard: # A refusal case is answered in one turn. The reply this loop would otherwise send @@ -2121,6 +2162,7 @@ def _execute_single_dashboard_run( reasoning_step_events=all_reasoning_step_events, timings=timings, run_latency_s=time.monotonic() - run_started, + exit_reason=exit_reason, ) @@ -2307,7 +2349,17 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_effective = len(summary.run_results) best = summary.best - detail: dict[str, Any] = _run_detail(best) + detail: dict[str, Any] = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + **best.evaluation.strict_checks, + **best.evaluation.diagnostics, + failures=best.evaluation.failures, + notes=best.evaluation.notes, + # Why the loop stopped. A dashboard not produced alone cannot tell a refusal from + # a run that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -2335,30 +2387,32 @@ def _write_scores(ctx: RunTraceContext) -> None: " one feature flag registers both, so check that before the model." ) notes = "; ".join(best.evaluation.notes) - exc = DashboardSkillAssertionError( + raise_agentic_failure( + DashboardSkillAssertionError, f"Dashboard skill assertion failed. {gate_note}{skill_note} " f"Checks: {best.evaluation.strict_checks}. " f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." - + (f" Notes (not scored): {notes}." if notes else "") + + (f" Notes (not scored): {notes}." if notes else ""), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.timings = item_timings - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - timings=item_timings, + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 0a55b2a00..907f0567a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -15,6 +15,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -37,7 +38,7 @@ ToolCallEvent, build_latency_breakdown, ) -from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings _DEFAULT_K = 1 @@ -241,6 +242,7 @@ def _attach_diagnostics( runs_passed: int, runs_effective: int, run_latencies_s: list[float], + best_run_latency_s: float | None, ) -> None: """Hang the report payload on a raised error, whichever error it is. @@ -256,6 +258,7 @@ def _attach_diagnostics( error.runs_effective = runs_effective error.failed_runs = failed_runs error.run_latencies_s = run_latencies_s + error.best_run_latency_s = best_run_latency_s def _run_detail(run: GeneralQuestionResult) -> dict: @@ -396,13 +399,17 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best - detail = { - **_run_detail(best), + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + judge_passed=best.passed, + judge_reasoning=best.reasoning, + actual_output=best.actual_output, # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K computed over fewer runs than --runs asked for is a weaker result and # the report has to say so. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), - } + ) # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as # non-passing here so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( @@ -420,28 +427,37 @@ def _write_scores(ctx: RunTraceContext) -> None: f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + " | ".join(unscored) ) - _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) + _attach_diagnostics( + error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, run_latency_s(best.timings) + ) error.timings = item_timings raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) - exc = GeneralQuestionAssertionError( - f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" + raise_agentic_failure( + GeneralQuestionAssertionError, + f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) - _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) - exc.timings = item_timings - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - timings=item_timings, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index 81f9cb1ad..f9e281c2d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -15,6 +15,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -87,7 +88,8 @@ class GuardrailResult: # Set when the judge returned something unreadable for THIS run. Excluded from pass@K # and from Langfuse scoring rather than counted as a failure -- see score_run. judge_error: str | None = None - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time (agent call + its grading) -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 @@ -211,6 +213,7 @@ def _attach_diagnostics( runs_passed: int, runs_effective: int, run_latencies_s: list[float], + best_run_latency_s: float | None, ) -> None: """Hang the report payload on a raised error, whichever error it is. @@ -226,6 +229,7 @@ def _attach_diagnostics( error.runs_effective = runs_effective error.failed_runs = failed_runs error.run_latencies_s = run_latencies_s + error.best_run_latency_s = best_run_latency_s def _run_detail(run: GuardrailResult) -> dict: @@ -354,12 +358,16 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best - detail = { - **_run_detail(best), + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + judge_passed=best.passed, + judge_reasoning=best.reasoning, + actual_output=best.actual_output, # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K over fewer runs than --runs asked for is a weaker result. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), - } + ) # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as # non-passing here so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( @@ -375,25 +383,34 @@ def _write_scores(ctx: RunTraceContext) -> None: f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + " | ".join(unscored) ) - _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) + _attach_diagnostics( + error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, best.run_latency_s + ) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) - exc = GuardrailAssertionError( - f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" + raise_agentic_failure( + GuardrailAssertionError, + f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) - _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 7ab93c40a..683f8d42d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -19,6 +19,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -201,7 +202,9 @@ class KdaRunResult: # separate a refusal from a run that hit max_iterations while still on track. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED turns_used: int = 0 - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from + # turn_wall_clock_sec above, which is only the final create-triggering turn. run_latency_s: float = 0.0 @@ -271,6 +274,7 @@ def run_agentic_kda_skill( ) def _run_once(conv_id: str) -> KdaRunResult: + run_started = time.monotonic() create_args: dict | None = None execute_result: dict | None = None turn_wall_clock_sec: float | None = None @@ -562,9 +566,21 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.evaluation - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + triggered=ev.triggered, + executed=ev.executed, + success=ev.success, + turn_completed=ev.turn_completed, + disambiguated=ev.disambiguated, + actual_create_args=best.actual_create_args, + actual_execute_result=best.actual_execute_result, + # Why the loop stopped -- see LoopExit. + exit_reason=best.exit_reason.value, + turns_used=best.turns_used, + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -582,25 +598,27 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Actual create args: {best.actual_create_args}. " f"Actual execute result: {best.actual_execute_result}." ) - exc = KdaSkillAssertionError(message) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + raise_agentic_failure( + KdaSkillAssertionError, + message, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, + ) + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 464cf592e..f662b205b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -21,6 +21,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -44,7 +45,7 @@ shift_and_index_events, timeline_detail, ) -from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings try: from openai import OpenAI as _OpenAI @@ -630,9 +631,19 @@ def _run_detail(run: MetricRunResult) -> dict: **timeline_detail(run.tool_call_events, run.reasoning_step_events), } - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + metric_created=best.metric_created, + maql_correct=best.maql_correct, + expected_maql_candidates=[c.get("maql", "") for c in expected_outputs_list], + actual_maql=best.actual_maql, + # Why the loop stopped. metric_created=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + turns_used=best.turns_used, + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -644,32 +655,32 @@ def _run_detail(run: MetricRunResult) -> dict: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) - exc = MetricSkillAssertionError( + raise_agentic_failure( + MetricSkillAssertionError, f"Metric skill assertion failed. {gate_note} " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " - f"Actual MAQL: {best.actual_maql}." + f"Actual MAQL: {best.actual_maql}.", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.timings = item_timings - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = run_latencies_s - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - timings=item_timings, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 90dc591cf..e58a08dd3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -15,6 +15,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -114,7 +115,8 @@ class SearchResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time -- mirrors the single-shot path's best_run_latency_s + # (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 @@ -326,7 +328,13 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_effective = len(summary.run_results) best = summary.best - detail = _run_detail(best) + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + tool_selected=best.tool_selected, + tool_correct=best.tool_correct, + tool_call_names=best.tool_call_names, + ) # Same predicate pass@K and runs_passed are taken over, so an item's failed_runs and its # counts cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -337,29 +345,29 @@ def _write_scores(ctx: RunTraceContext) -> None: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) - exc = SearchToolAssertionError( + raise_agentic_failure( + SearchToolAssertionError, f"Search tool assertion failed. {gate_note} " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " - f"Tool calls made: {best.tool_call_names}" + f"Tool calls made: {best.tool_call_names}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + best_run_latency_s=best.run_latency_s, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 6b1523ae2..b06821452 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -20,6 +20,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -77,7 +78,8 @@ class RunResult: # Why the simulated-user loop stopped. visualization_created=False alone cannot separate # a refusal from a run that hit max_iterations while still on track -- see LoopExit. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED - # This run's own wall time, from just before its first chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 # Everything the final turn said, which the answer patterns are matched against. answer_text: str = "" @@ -208,6 +210,7 @@ def _execute_single_run( one_turn = not options.expects_visualization or options.visualization_optional if one_turn: max_iterations = min(max_iterations, 1) + run_started = time.monotonic() total_turns = 0 total_steps = 0 all_tool_call_events: list[ToolCallEvent] = [] @@ -643,9 +646,16 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.eval_result - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + **evaluation_result_detail(ev), + # Why the loop stopped -- see LoopExit. total_turns is already the turn count for + # this run, so it doubles as turns_used. + exit_reason=best.exit_reason.value, + turns_used=int(best.total_turns), + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. Every failing run keeps its own # expected/actual check breakdown -- for this kind that is the whole diagnosis, and @@ -667,7 +677,8 @@ def _write_scores(ctx: RunTraceContext) -> None: else {"expects_visualization": False} ) actual_dump = best.actual_output.model_dump(exclude_none=True) if best.actual_output else None - exc = VisualizationAssertionError( + raise_agentic_failure( + VisualizationAssertionError, "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" "Agentic Visualization Assertion Failed! (Critical Mode)\n" f"{gate_note}\n" @@ -699,26 +710,25 @@ def _write_scores(ctx: RunTraceContext) -> None: f" Executed : {ev.executed}{' (required)' if ev.requires_execution else ''}\n" f"{_options_lines(ev, best.answer_text)}" f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n" - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = run_latencies_s - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index baaeb201d..de124ceed 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -35,6 +35,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -53,9 +54,9 @@ AgenticAssertionError, AgenticEvalOutcome, ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, - build_latency_breakdown, shift_and_index_events, ) @@ -212,8 +213,14 @@ class WhatIfRunResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from + # turn_wall_clock_sec above, which is only the scenario-execution turn. run_latency_s: float = 0.0 + # Why the simulated-user loop stopped -- see LoopExit. `triggered=False` alone cannot + # separate a refusal from a run that hit max_iterations while still on track (matches + # kda_skill.py/alert_skill.py). + exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @dataclass @@ -328,6 +335,7 @@ def run_agentic_what_if( ) def _run_once(conv_id: str) -> WhatIfRunResult: + run_started = time.monotonic() create_args: dict | None = None execute_result: dict | None = None turn_wall_clock_sec: float | None = None @@ -354,6 +362,10 @@ def _accumulate(result: ChatResult) -> None: all_reasoning_step_events.extend(result.reasoning_step_events or []) run_started = time.monotonic() + # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that + # simply runs out of range() is labelled correctly with no trailing else. + exit_reason = LoopExit.BUDGET_EXHAUSTED + for iteration in range(max_iterations): try: chat_result = client.send_message(conv_id, current_question) @@ -366,6 +378,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) create_args, execute_result = _extract_what_if_calls(all_tool_call_events) turn_completed = False + exit_reason = LoopExit.CHAT_ERROR break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -379,8 +392,10 @@ def _accumulate(result: ChatResult) -> None: if execute_result is not None: # The turn that ran the scenario, not an earlier disambiguation turn. turn_wall_clock_sec = chat_result.turn_wall_clock_sec + exit_reason = LoopExit.SUCCESS break if not response_text: + exit_reason = LoopExit.AGENT_SILENT break if iteration >= max_iterations - 1: break @@ -389,6 +404,7 @@ def _accumulate(result: ChatResult) -> None: disambiguated = True except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run _log.warning("Simulated what-if user reply failed for conversation %s: %s", conv_id, exc) + exit_reason = LoopExit.SIMULATED_USER_FAILED break return WhatIfRunResult( @@ -396,6 +412,7 @@ def _accumulate(result: ChatResult) -> None: evaluation=_evaluate_run(create_args, execute_result, expected_output, turn_completed, disambiguated), actual_create_args=create_args, actual_execute_result=execute_result, + exit_reason=exit_reason, turn_wall_clock_sec=turn_wall_clock_sec, reasoning_steps=reasoning_steps, response_id=response_id, @@ -453,26 +470,30 @@ class WhatIfAssertionError(AgenticAssertionError): def _detail(best: WhatIfRunResult) -> dict[str, Any]: ev = best.evaluation adjustments = _adjustments(best.actual_create_args) - return { - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "metric_correct": ev.metric_correct, - "maql_correct": ev.maql_correct, - "scenario_count_correct": ev.scenario_count_correct, - "baseline_correct": ev.baseline_correct, + return agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + triggered=ev.triggered, + executed=ev.executed, + success=ev.success, + turn_completed=ev.turn_completed, + metric_correct=ev.metric_correct, + maql_correct=ev.maql_correct, + scenario_count_correct=ev.scenario_count_correct, + baseline_correct=ev.baseline_correct, # Which content checks the fixture pinned -- without it a run that verified nothing # reads the same as one where everything matched. - "asserted": ev.asserted, - "disambiguated": ev.disambiguated, - "actual_adjustments": adjustments, - "actual_scenario_labels": [ + asserted=ev.asserted, + disambiguated=ev.disambiguated, + actual_adjustments=adjustments, + actual_scenario_labels=[ s.get("label") for s in ((best.actual_create_args or {}).get("scenarios") or []) if isinstance(s, dict) ], - "actual_execute_result": best.actual_execute_result, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + actual_execute_result=best.actual_execute_result, + # Why the loop stopped. triggered=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + ) def evaluate_agentic_what_if( @@ -613,24 +634,28 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Actual adjustments: {detail['actual_adjustments']}. " f"Actual execute result: {best.actual_execute_result}." ) - exc = WhatIfAssertionError(message) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = len(summary.run_results) - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=len(summary.run_results), + raise_agentic_failure( + WhatIfAssertionError, + message, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, + ) + + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py index c32cb3f40..6bb4b9e62 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py @@ -62,6 +62,7 @@ class JudgeResponseError(RuntimeError): runs_effective: int failed_runs: list[dict] run_latencies_s: list[float] + best_run_latency_s: float | None def _message_content(response: Any) -> str | None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 85f92e74c..435ac811a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -434,6 +434,7 @@ class AgenticAssertionError(AssertionError): runs_effective: int failed_runs: list[dict] run_latencies_s: list[float] + best_run_latency_s: float | None class AgenticEvalOutcome(BaseModel): @@ -470,6 +471,12 @@ class AgenticEvalOutcome(BaseModel): # needs one per run. Empty means the kind recorded none, which the report reads as "no # sample", not as zero-second runs. run_latencies_s: list[float] = Field(default_factory=list) + # The rank-selected run's own wall time -- mirrors the single-shot path's + # ItemReport.best_run_latency_s (see core/runner.py's _run_one_item), which the + # agentic path never populated because its K runs happen inside the evaluator, + # not in a loop the runner itself can time. None for a kind that has not yet been + # wired to measure it (see AgenticAssertionError.best_run_latency_s, same default). + best_run_latency_s: float | None = None class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/timing.py b/packages/gooddata-eval/src/gooddata_eval/core/timing.py index b2d203d7b..2c7b9992f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/timing.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/timing.py @@ -182,3 +182,13 @@ def note_trace_latency(seconds: float | None) -> None: sink = _trace_latencies.get() if sink is not None and seconds is not None: sink.append(seconds) + + +def run_latency_s(timings: PhaseTimings) -> float: + """One run's own wall time, for a single run's ``PhaseTimings``. + + Excludes ``langfuse_s`` (deliberately off the critical path -- see + ``PhaseTimings.langfuse_s``) so this stays comparable to the single-shot path's + ``best_run_latency_s``, which times only the agent call and its grading. + """ + return timings.agent_s + timings.judge_s + timings.simulated_user_s diff --git a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py index d7fef2965..5c1e25d8b 100644 --- a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py +++ b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py @@ -16,7 +16,7 @@ evaluate_agentic_anomaly_detection, run_agentic_anomaly_detection, ) -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.anomaly_detection" @@ -254,18 +254,22 @@ def test_a_single_turn_detection_passes(): summary = _run([_chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is False + assert summary.best.exit_reason is LoopExit.SUCCESS def test_a_clarifying_question_is_answered_and_the_run_continues(): summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is True + assert summary.best.exit_reason is LoopExit.SUCCESS def test_the_loop_stops_at_max_iterations_without_a_detection(): + # executed=False alone cannot tell this apart from a refusal -- exit_reason can. summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) assert summary.pass_at_k is False assert summary.best.evaluation.executed is False + assert summary.best.exit_reason is LoopExit.BUDGET_EXHAUSTED def test_an_empty_response_ends_the_run_immediately(): @@ -276,17 +280,19 @@ def test_an_empty_response_ends_the_run_immediately(): patch(f"{_MODULE}.ChatClient", return_value=client), patch(f"{_MODULE}.generate_simulated_anomaly_response") as sim, ): - run_agentic_anomaly_detection( + summary = run_agentic_anomaly_detection( host="http://h", token="tok", workspace_id="ws1", question="q", expected_output=_EXPECTED ) assert client.send_message.call_count == 1 sim.assert_not_called() + assert summary.best.exit_reason is LoopExit.AGENT_SILENT def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): summary = _run([_chat(_pair()), RuntimeError("boom")], k=2) assert len(summary.run_results) == 2 assert summary.pass_at_k is True + assert summary.run_results[1].exit_reason is LoopExit.CHAT_ERROR def test_k_must_be_at_least_one(): @@ -318,6 +324,8 @@ def test_detail_reports_the_series_analysed_and_the_flag_count(): assert outcome.detail["anomaly_point_count"] == 0 assert outcome.detail["asserted"] == ["metric", "granularity"] assert "latency_breakdown" in outcome.detail + assert "tool_calls" in outcome.detail + assert outcome.detail["exit_reason"] == "success" assert outcome.runs_passed == 1 assert len(outcome.run_latencies_s) == outcome.runs_effective == 1 assert all(latency >= 0 for latency in outcome.run_latencies_s) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index c29bfbbab..2edfc58e8 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -17,7 +17,8 @@ evaluate_dashboard_response, run_agentic_dashboard_skill, ) -from gooddata_eval.core.models import ChatResult, ToolCallEvent +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent # Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of # `agent_dashboard_skill` against ecommerce_demo. @@ -2343,6 +2344,8 @@ def test_the_loop_is_capped(self): assert client.send_message.call_count == 2 assert not summary.pass_at_k assert not summary.best.evaluation.drafted + # drafted=False alone cannot tell this apart from a refusal -- exit_reason can. + assert summary.best.exit_reason is LoopExit.BUDGET_EXHAUSTED def test_an_empty_turn_stops_the_loop(self): client = MagicMock() @@ -2350,6 +2353,7 @@ def test_an_empty_turn_stops_the_loop(self): summary = _run_with(client, _DC05_EXPECTED, max_iterations=5) assert client.send_message.call_count == 1 assert not summary.pass_at_k + assert summary.best.exit_reason is LoopExit.AGENT_SILENT def test_a_caller_supplied_conversation_is_not_deleted(self): client = MagicMock() @@ -2358,6 +2362,54 @@ def test_a_caller_supplied_conversation_is_not_deleted(self): client.create_conversation.assert_not_called() client.delete_conversation.assert_not_called() + def test_a_chat_error_is_recorded_rather_than_left_to_escape(self) -> None: + """It used to escape run_agentic_dashboard_skill, taking every completed K-run with + it: no DashboardRunResult, no exit_reason, just a bare error on the item. Recorded + the way metric_skill.py and conversation.py record it, so an infrastructure fault + stays distinguishable from the agent failing to draft.""" + client = MagicMock() + client.send_message.side_effect = ChatError("gen-ai fell over") + + summary = _run_with(client, _DC05_EXPECTED) + + assert summary.run_results[0].exit_reason is LoopExit.CHAT_ERROR + assert summary.run_results[0].evaluation.strict_pass is False + assert summary.pass_at_k is False + + def test_a_chat_error_keeps_the_partial_turn_s_work(self) -> None: + """A bare catch would discard ChatError.partial_result. What the stream delivered + before it died is still this turn's work -- the steps were taken and the tools ran.""" + partial = _chat_result( + tool_calls=[{"functionName": "search_objects", "functionArguments": "{}", "callTs": 1, "durationMs": 10}], + text="Here is the draft.", + ) + partial.reasoning_steps = ["looked for the charts"] + partial.reasoning_step_count = 1 + client = MagicMock() + client.send_message.side_effect = ChatError("stream died mid-answer", partial_result=partial) + + summary = _run_with(client, _DC05_EXPECTED) + run = summary.run_results[0] + + assert run.exit_reason is LoopExit.CHAT_ERROR + assert run.reasoning_steps == ["looked for the charts"] + assert run.total_steps == 1 + assert [tc.function_name for tc in run.tool_call_events] == ["search_objects"] + + def test_a_chat_error_on_a_later_turn_keeps_the_earlier_ones(self) -> None: + """The whole point of recording it: turns already completed survive the fault.""" + client = MagicMock() + client.send_message.side_effect = [ + _chat_result(text="Which charts did you have in mind?"), + ChatError("gen-ai fell over"), + ] + + summary = _run_with(client, _DC05_EXPECTED) + run = summary.run_results[0] + + assert run.exit_reason is LoopExit.CHAT_ERROR + assert run.total_turns == 2 + class TestEvaluateEntryPoint: def test_a_failing_run_raises_with_the_failures_attached(self): @@ -2421,6 +2473,8 @@ def test_a_passing_run_returns_the_outcome(self): assert outcome.detail["charts_matched"] is True assert len(outcome.run_latencies_s) == outcome.runs_effective assert all(latency >= 0 for latency in outcome.run_latencies_s) + assert outcome.detail["exit_reason"] == "success" + assert "tool_calls" in outcome.detail def _as_events(raw: list[dict]) -> list[ToolCallEvent]: diff --git a/packages/gooddata-eval/tests/test_agentic_general_question.py b/packages/gooddata-eval/tests/test_agentic_general_question.py index e670fb352..c95caa54a 100644 --- a/packages/gooddata-eval/tests/test_agentic_general_question.py +++ b/packages/gooddata-eval/tests/test_agentic_general_question.py @@ -257,6 +257,7 @@ def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): "judge_reasoning": "Correct answer", "actual_output": "42", "latency_breakdown": [], + "tool_calls": [], } # One wall-clock sample per run, passing runs included -- the latency distribution is over # these, and a kind that forgot to report them would silently drop out of it. @@ -288,6 +289,7 @@ def test_evaluate_agentic_general_question_attaches_reasoning_steps_to_exception "judge_reasoning": "Wrong answer", "actual_output": "I don't know", "latency_breakdown": [], + "tool_calls": [], } assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1 @@ -407,6 +409,9 @@ def test_evaluate_general_question_aggregates_timings_across_k_runs(): assert outcome.timings.judge_s == 1.5 # 1.0 + 0.5 # Each run's own span, agent and judge both inside it -- not their sum across runs. assert outcome.run_latencies_s == [3.0, 3.5] + # best_run_latency_s is the rank-selected run's own time (agent_s=2.0 + judge_s=1.0 + # from the first, tied-best run), not the item total above (both runs summed). + assert outcome.best_run_latency_s == 3.0 def test_evaluate_general_question_attaches_timings_to_the_failure_exception(): @@ -430,6 +435,7 @@ def test_evaluate_general_question_attaches_timings_to_the_failure_exception(): assert exc_info.value.timings.agent_s == 4.0 assert exc_info.value.timings.judge_s == 2.0 + assert exc_info.value.best_run_latency_s == 6.0 def test_trace_link_window_is_pinned_at_submit_time_not_when_the_task_runs(): diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 07f8bcb64..6a1690ebb 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -1134,6 +1134,7 @@ def test_evaluate_agentic_kda_skill_returns_reasoning_steps_on_pass(): "turns_used": 1, "max_iterations": 1, "latency_breakdown": [], + "tool_calls": [], } assert len(outcome.run_latencies_s) == outcome.runs_effective == 1 assert all(latency >= 0 for latency in outcome.run_latencies_s) @@ -1174,6 +1175,7 @@ def test_evaluate_agentic_kda_skill_attaches_reasoning_steps_to_exception_on_fai "turns_used": 1, "max_iterations": 1, "latency_breakdown": [], + "tool_calls": [], } assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index a86a2e9c3..5efcbb0bb 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -855,6 +855,7 @@ def test_evaluate_metric_skill_surfaces_timings_on_the_outcome(): assert outcome.timings.agent_s == 3.0 assert outcome.timings.simulated_user_s == 0.0 + assert outcome.best_run_latency_s == 3.0 def test_no_timer_output_by_default(monkeypatch, capsys): diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 8682c9bec..ff5b574bf 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -416,6 +416,54 @@ def test_run_agentic_items_records_phase_timings_when_the_item_fails(): assert report.items[0].judge_latency_s == 2.0 +def test_run_agentic_items_records_best_run_latency_on_success(): + # best_run_latency_s was never populated on the agentic path -- the K-run loop and + # best-of-K selection happen inside the evaluator, not in a loop the runner itself + # times (unlike _run_one_item on the single-shot path). + outcome = AgenticEvalOutcome( + conversation_id="c1", + detail={"alert_created": True}, + best_run_latency_s=4.25, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", return_value=outcome): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert report.items[0].best_run_latency_s == 4.25 + + +def test_run_agentic_items_records_best_run_latency_when_the_item_fails(): + exc = AlertSkillAssertionError("nope") + exc.detail = {"alert_created": False} + exc.best_run_latency_s = 9.5 + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert report.items[0].pass_at_k is False + assert report.items[0].best_run_latency_s == 9.5 + + +def test_an_errored_item_without_best_run_latency_keeps_the_none_default(): + # Kinds not yet wired to measure it (or any exception with no such attribute) must + # not report an invented number -- None stays None, not 0.0. + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=RuntimeError("boom")): + report = run_agentic_items([_timed_item()], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01") + + assert report.items[0].best_run_latency_s is None + + def test_a_pending_trace_link_does_not_block_the_next_item(): """The overlap claim, proved by construction rather than by a stopwatch. @@ -1013,9 +1061,15 @@ def test_every_multi_run_kind_s_evaluator_builds_failed_runs(kind, expected_outp pytest.skip(f"{kind} runs its fixture once; no K to fail within") assert "build_failed_runs(" in source, f"{module.__name__} never builds per-run failure records" assert "failed_runs=failed_runs" in source, f"{module.__name__} never returns them on the success path" - # Either set directly on the raised error, or via the kind's own _attach_diagnostics - # helper -- the judge-based kinds raise from two places and set the payload in one. - attaches = "failed_runs = failed_runs" in source or "_attach_diagnostics(" in source + # Either set directly on the raised error, via the kind's own _attach_diagnostics + # helper (the judge-based kinds raise from two places and set the payload in one), or + # passed to raise_agentic_failure -- the shared tail every other kind's failure raise + # goes through, alongside its agentic_success mirror, so the keyword appears twice. + attaches = ( + "failed_runs = failed_runs" in source + or "_attach_diagnostics(" in source + or ("raise_agentic_failure(" in source and source.count("failed_runs=failed_runs") >= 2) + ) assert attaches, f"{module.__name__} never attaches them to its failure" @@ -1164,7 +1218,13 @@ def test_every_evaluator_hands_out_its_run_latencies(kind, expected_output, targ module = importlib.import_module(getattr(agentic_runner, target).__module__) source = inspect.getsource(module) assert "run_latencies_s=" in source, f"{module.__name__} never returns its run latencies on the success path" - assert ".run_latencies_s = " in source, f"{module.__name__} never attaches them to its failure" + # Either set directly on the raised error (the two judge-based kinds' all-ungraded + # path), or passed to raise_agentic_failure alongside its agentic_success mirror, so + # the keyword appears at least twice. + attaches = ".run_latencies_s = " in source or ( + "raise_agentic_failure(" in source and source.count("run_latencies_s=") >= 2 + ) + assert attaches, f"{module.__name__} never attaches them to its failure" @pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) diff --git a/packages/gooddata-eval/tests/test_agentic_search_tool.py b/packages/gooddata-eval/tests/test_agentic_search_tool.py index 916877bdb..05a1a1635 100644 --- a/packages/gooddata-eval/tests/test_agentic_search_tool.py +++ b/packages/gooddata-eval/tests/test_agentic_search_tool.py @@ -208,6 +208,7 @@ def test_evaluate_agentic_search_tool_returns_reasoning_steps_on_pass(): "tool_correct": True, "tool_call_names": ["search_objects"], "latency_breakdown": [], + "tool_calls": [], } # One wall-clock sample per run, passing runs included -- the latency distribution is over # these, and a kind that forgot to report them would silently drop out of it. @@ -248,6 +249,7 @@ def test_evaluate_agentic_search_tool_attaches_reasoning_steps_to_exception_on_f "tool_correct": False, "tool_call_names": [], "latency_breakdown": [], + "tool_calls": [], } assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 67a3cc84c..38b04ebf3 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -13,7 +13,7 @@ evaluate_agentic_what_if, run_agentic_what_if, ) -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.what_if" @@ -215,6 +215,7 @@ def test_a_single_turn_scenario_passes(): summary = _run([_chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is False + assert summary.best.exit_reason is LoopExit.SUCCESS def test_a_clarifying_question_is_answered_and_the_run_continues(): @@ -222,12 +223,15 @@ def test_a_clarifying_question_is_answered_and_the_run_continues(): summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is True + assert summary.best.exit_reason is LoopExit.SUCCESS def test_the_loop_stops_at_max_iterations_without_a_scenario(): + # triggered=False alone cannot tell this apart from a refusal -- exit_reason can. summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) assert summary.pass_at_k is False assert summary.best.evaluation.triggered is False + assert summary.best.exit_reason is LoopExit.BUDGET_EXHAUSTED def test_an_empty_response_ends_the_run_immediately(): @@ -238,15 +242,19 @@ def test_an_empty_response_ends_the_run_immediately(): patch(f"{_MODULE}.ChatClient", return_value=client), patch(f"{_MODULE}.generate_simulated_what_if_response") as sim, ): - run_agentic_what_if(host="http://h", token="tok", workspace_id="ws1", question="q", expected_output=_EXPECTED) + summary = run_agentic_what_if( + host="http://h", token="tok", workspace_id="ws1", question="q", expected_output=_EXPECTED + ) assert client.send_message.call_count == 1 sim.assert_not_called() + assert summary.best.exit_reason is LoopExit.AGENT_SILENT def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): summary = _run([_chat(_pair()), RuntimeError("boom")], k=2) assert len(summary.run_results) == 2 assert summary.pass_at_k is True + assert summary.run_results[1].exit_reason is LoopExit.CHAT_ERROR def test_k_must_be_at_least_one(): @@ -277,6 +285,8 @@ def test_detail_reports_the_adjustments_the_agent_asked_for(): assert outcome.detail["actual_scenario_labels"] == ["Scenario A"] assert outcome.detail["asserted"] == ["metric_id", "scenario_maql"] assert "latency_breakdown" in outcome.detail + assert "tool_calls" in outcome.detail + assert outcome.detail["exit_reason"] == "success" assert outcome.runs_passed == 1 assert len(outcome.run_latencies_s) == outcome.runs_effective == 1 assert all(latency >= 0 for latency in outcome.run_latencies_s) diff --git a/packages/gooddata-eval/tests/test_timing.py b/packages/gooddata-eval/tests/test_timing.py index b2de2e19b..527ce7be5 100644 --- a/packages/gooddata-eval/tests/test_timing.py +++ b/packages/gooddata-eval/tests/test_timing.py @@ -15,6 +15,7 @@ log_timer, note_trace_latency, percentile, + run_latency_s, sum_timings, timers_enabled, trimmed_mean, @@ -268,3 +269,10 @@ def item(base: float) -> list[float]: results = list(pool.map(item, [0.0, 100.0, 200.0, 300.0])) assert results == [[b + s for s in range(5)] for b in (0.0, 100.0, 200.0, 300.0)] + + +def test_run_latency_s_excludes_langfuse(): + # langfuse_s is deliberately off the critical path (see PhaseTimings docstring), so a + # trace-link slowdown must not inflate the run latency a worst-case table sorts on. + timings = PhaseTimings(agent_s=2.0, judge_s=1.0, simulated_user_s=0.5, langfuse_s=100.0) + assert run_latency_s(timings) == 3.5 From 5f40cd0d770401af308690e6d62fcd9d76ff1220 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Wed, 11 Nov 2026 13:31:52 +0100 Subject: [PATCH 2/2] feat(gooddata-eval): add the agentic forecasting evaluator First of three evaluators for product skills with no eval coverage so far: forecasting, what-if analysis, anomaly detection. This one checks forecasting correctness, not just completion -- the skill refuses to execute unless the visualization carries an AAC forecast config, so the requested horizon, metric, confidence and seasonality are all numbers in the tool call itself, checkable with no judge. A review finding (forecasting turns not joined to the item's Langfuse trace scope under clarification) is tracked separately: https://github.com/gooddata/gooddata-python-sdk/issues/1941 jira: trivial risk: low Co-Authored-By: Claude Sonnet 5 --- .../src/gooddata_eval/cli/agentic_runner.py | 15 + .../gooddata_eval/core/agentic/forecasting.py | 641 ++++++++++++++++++ .../src/gooddata_eval/core/interaction.py | 1 + .../tests/test_agentic_forecasting.py | 528 +++++++++++++++ .../tests/test_agentic_runner.py | 2 + .../gooddata-eval/tests/test_trace_linker.py | 1 + 6 files changed, 1188 insertions(+) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py create mode 100644 packages/gooddata-eval/tests/test_agentic_forecasting.py diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 1aa4d3184..ad1514ef9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -18,6 +18,7 @@ from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill +from gooddata_eval.core.agentic.forecasting import evaluate_agentic_forecasting from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -57,6 +58,7 @@ class _LfKw(TypedDict, total=False): "agentic_guardrail", "agentic_conversation", "agentic_kda_skill", + "agentic_forecasting", "agentic_dashboard_summary", "agentic_anomaly_detection", "agentic_what_if", @@ -305,6 +307,19 @@ def _dispatch_agentic( user_context=item.user_context, **lf_kw, ) + elif kind == "agentic_forecasting": + return evaluate_agentic_forecasting( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + gate=gate, + agent_id=agent_id, + user_context=item.user_context, + **lf_kw, + ) elif kind == "agentic_dashboard_summary": summary_input = item.summary_input if summary_input is None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py new file mode 100644 index 000000000..98f9d3b76 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -0,0 +1,641 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic forecasting-skill evaluation runner. + +Unlike kda_skill, this is not limited to "the process ran". The forecasting skill refuses to +execute unless the visualization it points at carries an AAC forecast config, and that config +is where the user's request lands: + + config.forecast_enabled must be true or execute_forecast returns an error + config.forecast_period how many periods ahead -- "next 3 months" is 3 + config.forecast_confidence confidence level, 0.95 by default + config.forecast_seasonal whether seasonality is modelled + +So a fixture can state what it asked for and have it checked exactly, with no judge: the +period the agent chose is a number in the tool call it made. The measure it forecast is +checkable the same way, off the visualization's own fields. +""" + +from __future__ import annotations + +import logging +import os +import time +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) + +_log = logging.getLogger(__name__) + +_DEFAULT_K = 1 +# The agent routinely asks which measure to forecast before it builds anything (observed +# live: "your data has two different Spend metrics"), so the budget covers a couple of +# disambiguation rounds plus slack, matching kda_skill's reasoning. +_DEFAULT_MAX_ITERATIONS = 4 + + +def _build_clarification_prompt(agent_message: str, expected_output: dict) -> str: + """The simulated-user reply, mentioning only the hints the fixture actually supplies. + + An absent hint is dropped from the prompt entirely rather than asserted as a literal + "None", which would answer a question the agent never asked with a wrong value. + """ + hints: list[str] = [] + metric = expected_output.get("metric") + if metric: + hints.append(f"the measure to forecast is {metric}") + period = expected_output.get("forecast_period") + if period is not None: + hints.append(f"the forecast horizon is {period} periods ahead") + granularity = expected_output.get("granularity") + if granularity: + hints.append(f"the time granularity is {granularity}") + # Both are scored when the fixture pins them, so withholding them here would let the + # harness fail a run on a value it refused to supply: the agent asks which confidence + # level to use, the simulated user guesses, and `confidence_correct` reports the guess + # as the agent's error. `is not None` rather than a truth test -- a pinned + # `forecast_seasonal: false` is an answer, and `0.0` would be a confidence level. + confidence = expected_output.get("forecast_confidence") + if confidence is not None: + hints.append(f"the confidence level is {confidence}") + seasonal = expected_output.get("forecast_seasonal") + if seasonal is not None: + hints.append(f"seasonality {'should' if seasonal else 'should not'} be modelled") + reference = "; ".join(hints) + return ( + f"You are simulating a user in a conversation with a BI assistant that forecasts metric " + f"values. The assistant asked: '{agent_message}'. " + + (f"For reference, {reference}. " if reference else "") + + "Reply briefly as the user, answering whichever of those the assistant actually asked about." + ) + + +def generate_simulated_forecast_response(agent_message: str, expected_output: dict) -> str: + """Generate a user reply to keep the forecasting conversation going (gpt-4o-mini). + + Always OpenAI regardless of the workspace's own model: this is harness plumbing, not the + system under test. + """ + try: + from openai import OpenAI # noqa: PLC0415 + except ImportError as exc: + raise RuntimeError("openai package is required for generate_simulated_forecast_response") from exc + + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise OSError("OPENAI_API_KEY environment variable is not set") + + client = OpenAI(api_key=api_key) + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": _build_clarification_prompt(agent_message, expected_output)}], + max_tokens=150, + temperature=0, + timeout=30, + ) + return response.choices[0].message.content or "Please proceed with the most complete option." + + +def _extract_forecast_calls( + tool_call_events: list[ToolCallEvent], +) -> tuple[dict | None, dict | None]: + """Return (visualization_args, execute_result) for the LAST create/execute pair. + + A new create_adhoc_visualization clears any earlier execute result: that result belongs + to the visualization it followed, not to this one. Picking the last of each + independently would pair a fresh chart with a stale forecast. + """ + viz_args: dict | None = None + execute_result: dict | None = None + for tc in tool_call_events: + if tc.function_name == "create_adhoc_visualization": + args = tc.parsed_arguments() or {} + viz = args.get("visualization") + viz_args = viz if isinstance(viz, dict) else args + execute_result = None + elif tc.function_name == "execute_forecast" and tc.result: + execute_result = tc.parsed_result() + return viz_args, execute_result + + +def _forecast_config(viz_args: dict | None) -> dict: + config = (viz_args or {}).get("config") + return config if isinstance(config, dict) else {} + + +def _metric_uris(viz_args: dict | None) -> set[str]: + """The metric URIs the visualization measures, resolved through its field aliases. + + Deliberately not core.scoring.get_metric_uri_set: that takes a parsed + CreatedVisualization, while this reads the raw tool-call arguments, where a field may be + a bare URI string rather than an object. + """ + query = (viz_args or {}).get("query") + fields = query.get("fields") if isinstance(query, dict) else None + if not isinstance(fields, dict): + return set() + uris: set[str] = set() + for alias in (viz_args or {}).get("metrics") or list(fields): + name = alias.get("field") if isinstance(alias, dict) else alias + field_def = fields.get(name) + if isinstance(field_def, dict) and field_def.get("using"): + uris.add(str(field_def["using"])) + elif isinstance(field_def, str): + uris.add(field_def) + return {u for u in uris if u.startswith(("metric/", "fact/"))} + + +@dataclass +class ForecastEvaluation: + """Scores for a single forecasting run. + + ``triggered``/``executed``/``success``/``turn_completed`` are the process checks every + skill kind shares. ``forecast_enabled``, ``period_correct`` and ``metric_correct`` are + content checks, and each is True when the fixture did not ask for it -- an unstated + expectation must not fail a run, and must not silently pass one either, which is why + ``asserted`` records which of them the fixture actually pinned. + """ + + triggered: bool + executed: bool + success: bool + turn_completed: bool + forecast_enabled: bool + period_correct: bool + metric_correct: bool + confidence_correct: bool + seasonal_correct: bool + asserted: list[str] = field(default_factory=list) + disambiguated: bool = False + + @property + def strict_pass(self) -> bool: + return all( + [ + self.triggered, + self.executed, + self.success, + self.turn_completed, + self.forecast_enabled, + self.period_correct, + self.metric_correct, + self.confidence_correct, + self.seasonal_correct, + ] + ) + + +@dataclass +class ForecastRunResult: + """Outcome of one run (one conversation, up to max_iterations messages).""" + + conversation_id: str + evaluation: ForecastEvaluation + actual_visualization: dict | None + actual_execute_result: dict | None + turn_wall_clock_sec: float | None = None + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # This run's own wall time, from just before its chat call to the end of its scoring. + run_latency_s: float = 0.0 + + +@dataclass +class AgenticForecastSummary: + """Aggregated outcome of K runs for one forecasting item.""" + + run_results: list[ForecastRunResult] + pass_at_k: bool + pass_power_k: bool + best: ForecastRunResult + + +def _evaluate_run( + viz_args: dict | None, + execute_result: dict | None, + expected_output: dict, + turn_completed: bool, + disambiguated: bool = False, +) -> ForecastEvaluation: + triggered = execute_result is not None or viz_args is not None + executed = execute_result is not None + success = executed and execute_result.get("success") is True + + config = _forecast_config(viz_args) + # None is "not set", which the tool treats as disabled -- only an explicit true enables it. + forecast_enabled = config.get("forecast_enabled") is True + + asserted: list[str] = [] + expected_period = expected_output.get("forecast_period") + if expected_period is None: + period_correct = True + else: + asserted.append("forecast_period") + actual_period = config.get("forecast_period") + period_correct = isinstance(actual_period, int | float) and float(actual_period) == float(expected_period) + + expected_metric = expected_output.get("metric") + if not expected_metric: + metric_correct = True + else: + asserted.append("metric") + wanted = {expected_metric} if isinstance(expected_metric, str) else set(expected_metric) + metric_correct = bool(_metric_uris(viz_args) & wanted) + + expected_confidence = expected_output.get("forecast_confidence") + if expected_confidence is None: + confidence_correct = True + else: + asserted.append("forecast_confidence") + actual_confidence = config.get("forecast_confidence") + confidence_correct = isinstance(actual_confidence, int | float) and float(actual_confidence) == float( + expected_confidence + ) + + expected_seasonal = expected_output.get("forecast_seasonal") + if expected_seasonal is None: + seasonal_correct = True + else: + asserted.append("forecast_seasonal") + # The tool defaults seasonal to False, so an absent value means "not seasonal" -- + # reading it as a mismatch would fail an agent that simply left the default alone. + actual_seasonal = config.get("forecast_seasonal") + seasonal_correct = bool(actual_seasonal) == bool(expected_seasonal) + + return ForecastEvaluation( + triggered=triggered, + executed=executed, + success=success, + turn_completed=turn_completed, + forecast_enabled=forecast_enabled, + period_correct=period_correct, + metric_correct=metric_correct, + confidence_correct=confidence_correct, + seasonal_correct=seasonal_correct, + asserted=asserted, + disambiguated=disambiguated, + ) + + +def run_agentic_forecasting( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, + user_context: dict | None = None, +) -> AgenticForecastSummary: + """Run the forecasting agentic evaluation K times and return a summary. + + A run ends as soon as execute_forecast returns, which is the goal signal. Short of that + it keeps sending simulated replies up to ``max_iterations``, with no attempt to classify + whether the agent's text was really a question: missing a genuine clarifying question + hard-fails the run, while answering a final answer costs one harmless extra turn. + """ + if k < 1: + raise ValueError(f"k must be >= 1, got {k}") + run_results: list[ForecastRunResult] = [] + client = ChatClient( + host=host, + token=token, + workspace_id=workspace_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + user_context=user_context, + ) + + def _run_once(conv_id: str) -> ForecastRunResult: + viz_args: dict | None = None + execute_result: dict | None = None + turn_wall_clock_sec: float | None = None + turn_completed = False + disambiguated = False + current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None + all_tool_call_events: list[ToolCallEvent] = [] + all_reasoning_step_events: list[ReasoningStepEvent] = [] + turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time + tool_index_offset = 0 + reasoning_index_offset = 0 + + def _accumulate(result: ChatResult) -> None: + nonlocal turn_offset, tool_index_offset, reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(result.tool_call_events or []) + all_reasoning_step_events.extend(result.reasoning_step_events or []) + + run_started = time.monotonic() + for iteration in range(max_iterations): + try: + chat_result = client.send_message(conv_id, current_question) + except Exception as exc: # noqa: BLE001 -- end this run, not the whole item + _log.warning("Forecast send_message failed for conversation %s: %s", conv_id, exc) + partial = getattr(exc, "partial_result", None) + if partial is not None: + reasoning_steps.extend(partial.reasoning_steps or []) + response_id = partial.response_id or response_id + _accumulate(partial) + viz_args, execute_result = _extract_forecast_calls(all_tool_call_events) + turn_completed = False + break + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id + _accumulate(chat_result) + # Over every turn so far, not just this one: the agent may build the chart on + # one turn and forecast on the next, and reading a single turn would drop the + # visualization the forecast actually ran on. + viz_args, execute_result = _extract_forecast_calls(all_tool_call_events) + response_text = render_answer_text(chat_result) + turn_completed = chat_result.stream_ended and bool(response_text) + if execute_result is not None: + # The turn that actually forecast, not an earlier disambiguation turn. + turn_wall_clock_sec = chat_result.turn_wall_clock_sec + break + if not response_text: + break + if iteration >= max_iterations - 1: + break + try: + current_question = generate_simulated_forecast_response(response_text, expected_output) + disambiguated = True + except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run + _log.warning("Simulated forecast user reply failed for conversation %s: %s", conv_id, exc) + break + + return ForecastRunResult( + conversation_id=conv_id, + evaluation=_evaluate_run(viz_args, execute_result, expected_output, turn_completed, disambiguated), + actual_visualization=viz_args, + actual_execute_result=execute_result, + turn_wall_clock_sec=turn_wall_clock_sec, + reasoning_steps=reasoning_steps, + response_id=response_id, + tool_call_events=all_tool_call_events, + reasoning_step_events=all_reasoning_step_events, + run_latency_s=time.monotonic() - run_started, + ) + + try: + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append(_run_once(conv_id_0)) + finally: + if initial_conversation_id is None: # only delete conversations we created + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append(_run_once(conv_id)) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + pass_at_k = any(r.evaluation.strict_pass for r in run_results) + pass_power_k = all(r.evaluation.strict_pass for r in run_results) + best = max( + run_results, + key=lambda r: sum( + [ + r.evaluation.triggered, + r.evaluation.executed, + r.evaluation.success, + r.evaluation.turn_completed, + r.evaluation.forecast_enabled, + r.evaluation.period_correct, + r.evaluation.metric_correct, + r.evaluation.confidence_correct, + r.evaluation.seasonal_correct, + ] + ), + ) + return AgenticForecastSummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +class ForecastingAssertionError(AgenticAssertionError): + """Raised when a forecasting evaluation fails.""" + + +def _detail(best: ForecastRunResult) -> dict[str, Any]: + ev = best.evaluation + return { + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "forecast_enabled": ev.forecast_enabled, + "period_correct": ev.period_correct, + "metric_correct": ev.metric_correct, + "confidence_correct": ev.confidence_correct, + "seasonal_correct": ev.seasonal_correct, + # Which content checks the fixture pinned. Without it a run where nothing was + # asserted is indistinguishable in the report from one where everything matched. + "asserted": ev.asserted, + "disambiguated": ev.disambiguated, + "actual_forecast_config": _forecast_config(best.actual_visualization), + "actual_metrics": sorted(_metric_uris(best.actual_visualization)), + "actual_execute_result": best.actual_execute_result, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } + + +def evaluate_agentic_forecasting( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + gate: EvalGate = DEFAULT_GATE, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "forecasting", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + user_context: dict | None = None, +) -> AgenticEvalOutcome: + """Run forecasting evaluation, log to Langfuse, and raise ForecastingAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_forecasting( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + k=k, + max_iterations=max_iterations, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + user_context=user_context, + ) + + if langfuse is not None and dataset_item_id: + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + ev = run.evaluation + strict_checks = { + "forecast_triggered": ev.triggered, + "forecast_executed": ev.executed, + "forecast_success": ev.success, + "forecast_turn_completed": ev.turn_completed, + "forecast_enabled": ev.forecast_enabled, + } + # Only the content checks the fixture actually pinned. An unasserted check + # is True internally so it cannot fail a run, but publishing that as a + # BOOLEAN 1 would claim the evaluator verified something it never looked at. + strict_checks.update( + { + key: value + for name, key, value in ( + ("forecast_period", "forecast_period_correct", ev.period_correct), + ("metric", "forecast_metric_correct", ev.metric_correct), + ("forecast_confidence", "forecast_confidence_correct", ev.confidence_correct), + ("forecast_seasonal", "forecast_seasonal_correct", ev.seasonal_correct), + ) + if name in ev.asserted + } + ) + with ctx.observe(pt, run_idx) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) + ctx.quality( + tid, + strict_checks=strict_checks, + # pt.latency covers the whole conversation, which is the item's real + # elapsed cost when the agent needed clarification turns to get + # there; turn_wall_clock_sec (the goal turn alone) is the fallback. + # This is what 7 of the 8 existing kinds do -- kda_skill is the + # outlier and documents its own reason. Cost is not gated on + # ev.triggered: a run that answered without ever reaching the tool + # still spent tokens, and hiding that understates what the item cost. + latency_sec=pt.latency if pt else run.turn_wall_clock_sec, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + # The question this run answered, so a score is readable without resolving the + # conversation back to its item. + item_input=question, + ) + + best = summary.best + ev = best.evaluation + detail = _detail(best) + runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. + failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_detail) + + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + message = ( + f"Forecasting assertion failed. {gate_note} strict_pass={ev.strict_pass} " + f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " + f"turn_completed={ev.turn_completed}, forecast_enabled={ev.forecast_enabled}, " + f"period_correct={ev.period_correct}, metric_correct={ev.metric_correct}, " + f"confidence_correct={ev.confidence_correct}, seasonal_correct={ev.seasonal_correct}). " + f"Actual forecast config: {detail['actual_forecast_config']}. " + f"Actual execute result: {best.actual_execute_result}." + ) + exc = ForecastingAssertionError(message) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = len(summary.run_results) + exc.failed_runs = failed_runs + exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] + raise exc + + return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py index ccf84491c..645625697 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py @@ -45,6 +45,7 @@ "agentic_kda_skill", "agentic_anomaly_detection", "agentic_what_if", + "agentic_forecasting", } ) diff --git a/packages/gooddata-eval/tests/test_agentic_forecasting.py b/packages/gooddata-eval/tests/test_agentic_forecasting.py new file mode 100644 index 000000000..ccecfa93f --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_forecasting.py @@ -0,0 +1,528 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.forecasting import ( + ForecastingAssertionError, + _build_clarification_prompt, + _evaluate_run, + _extract_forecast_calls, + _metric_uris, + evaluate_agentic_forecasting, + run_agentic_forecasting, +) +from gooddata_eval.core.models import ChatResult + +_MODULE = "gooddata_eval.core.agentic.forecasting" + +_EXPECTED = {"metric": "metric/spend", "forecast_period": 3} + + +def _viz_args(*, period=3, enabled=True, metric="metric/spend", confidence=0.95, seasonal=False) -> dict: + return { + "visualization": { + "type": "line_chart", + "title": "Spend forecast", + "query": { + "fields": { + "m_spend": {"using": metric}, + "d_month": {"using": "label/process_date.month"}, + }, + "filter_by": {}, + }, + "metrics": ["m_spend"], + "view_by": ["d_month"], + "config": { + "forecast_enabled": enabled, + "forecast_period": period, + "forecast_confidence": confidence, + "forecast_seasonal": seasonal, + }, + } + } + + +def _tc(name: str, args: dict | None = None, result: str | None = None): + tc = MagicMock() + tc.function_name = name + tc.call_ts = None + tc.result_ts = None + tc.index = None + tc.parsed_arguments = lambda a=args: a or {} + tc.result = result + tc.parsed_result = lambda r=result: __import__("json").loads(r) if r else None + return tc + + +_OK_FORECAST = '{"success": true, "data": {"points": [{"value": 1}], "truncated": false}, "error": null}' + + +def _chat(tool_calls, text="Here is the forecast.", ended=True) -> ChatResult: + """A real ChatResult, so render_answer_text sees every field it reads.""" + result = ChatResult.model_validate({"textResponse": text, "responseId": "resp-1"}) + result.stream_ended = ended + result.turn_wall_clock_sec = 4.2 + # Assigned after validation: these are hand-built tool-call doubles, not payload. + result.tool_call_events = tool_calls + return result + + +# ── extraction ────────────────────────────────────────────────────────────── + + +def test_extract_pairs_the_execute_with_the_visualization_it_followed(): + """A new chart clears the previous forecast: picking the last of each independently + would pair a fresh visualization with a stale result.""" + calls = [ + _tc("create_adhoc_visualization", _viz_args(period=1)), + _tc("execute_forecast", {}, _OK_FORECAST), + _tc("create_adhoc_visualization", _viz_args(period=3)), + ] + viz, result = _extract_forecast_calls(calls) + + assert viz["config"]["forecast_period"] == 3 + assert result is None + + +def test_extract_unwraps_the_visualization_argument(): + viz, _ = _extract_forecast_calls([_tc("create_adhoc_visualization", _viz_args())]) + assert viz["type"] == "line_chart" + + +def test_metric_uris_resolves_aliases_and_ignores_dimensions(): + viz, _ = _extract_forecast_calls([_tc("create_adhoc_visualization", _viz_args())]) + assert _metric_uris(viz) == {"metric/spend"} + + +def test_metric_uris_handles_a_bare_uri_field(): + """Tool-call arguments are raw JSON, where a field may be a plain string rather than + the object core.scoring's parsed model always has.""" + viz = {"query": {"fields": {"m": "metric/spend"}}, "metrics": ["m"]} + assert _metric_uris(viz) == {"metric/spend"} + + +# ── scoring ───────────────────────────────────────────────────────────────── + + +def _evaluate(viz=None, result=None, expected=None, turn_completed=True): + return _evaluate_run(viz, result, expected if expected is not None else _EXPECTED, turn_completed) + + +def _viz(**kw): + return _viz_args(**kw)["visualization"] + + +def test_a_correct_forecast_passes_every_check(): + ev = _evaluate(_viz(), {"success": True}) + assert ev.strict_pass is True + assert ev.asserted == ["forecast_period", "metric"] + + +def test_the_wrong_horizon_fails_on_period_alone(): + """The number the agent chose is in the tool call, so 'next 3 months' is checkable + exactly -- no judge, no paraphrase tolerance.""" + ev = _evaluate(_viz(period=6), {"success": True}) + assert ev.period_correct is False + assert ev.metric_correct is True + assert ev.strict_pass is False + + +def test_forecasting_the_wrong_measure_fails_on_metric_alone(): + ev = _evaluate(_viz(metric="metric/orders"), {"success": True}) + assert ev.metric_correct is False + assert ev.period_correct is True + + +def test_forecast_enabled_must_be_explicitly_true(): + """The tool refuses on forecast_enabled=false, and treats an unset value the same way, + so only an explicit true counts.""" + assert _evaluate(_viz(enabled=None), {"success": True}).forecast_enabled is False + assert _evaluate(_viz(enabled=False), {"success": True}).forecast_enabled is False + assert _evaluate(_viz(enabled=True), {"success": True}).forecast_enabled is True + + +def test_an_unstated_expectation_neither_fails_nor_silently_passes(): + """A fixture that pins nothing must not fail, but the report has to say nothing was + checked -- otherwise it reads identically to a run where everything matched.""" + ev = _evaluate(_viz(period=99, metric="metric/anything"), {"success": True}, expected={}) + assert ev.period_correct is True + assert ev.metric_correct is True + assert ev.asserted == [] + + +def test_a_failed_execution_is_not_a_success(): + ev = _evaluate(_viz(), {"success": False, "error": "no forecast for you"}) + assert ev.executed is True + assert ev.success is False + assert ev.strict_pass is False + + +def test_never_reaching_execute_is_not_triggered_by_a_chart_alone(): + ev = _evaluate(_viz(), None) + assert ev.executed is False + assert ev.success is False + + +def test_an_integer_valued_float_period_still_matches(): + """The AAC config is JSON, so 3 may arrive as 3.0.""" + assert _evaluate(_viz(period=3.0), {"success": True}).period_correct is True + + +# ── run loop ──────────────────────────────────────────────────────────────── + + +def _run(side_effect, *, k=1, expected=None, max_iterations=4): + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, k + 2)] + client.send_message.side_effect = side_effect + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response", return_value="use spend, 3 months"), + ): + return run_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=expected if expected is not None else _EXPECTED, + k=k, + max_iterations=max_iterations, + ) + + +def test_a_single_turn_forecast_passes(): + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat(calls)]) + assert summary.pass_at_k is True + assert summary.best.evaluation.disambiguated is False + + +def test_a_clarifying_question_is_answered_and_the_run_continues(): + """Observed live: the agent asks which Spend metric before building anything.""" + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(calls)]) + assert summary.pass_at_k is True + assert summary.best.evaluation.disambiguated is True + + +def test_the_loop_stops_at_max_iterations_without_a_forecast(): + summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) + assert summary.pass_at_k is False + assert summary.best.evaluation.executed is False + + +def test_an_empty_response_ends_the_run_immediately(): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _chat([], text="") + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response") as sim, + ): + run_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="q", + expected_output=_EXPECTED, + ) + assert client.send_message.call_count == 1 + sim.assert_not_called() + + +def test_a_chat_error_ends_the_run_and_keeps_what_the_partial_carried(): + error = RuntimeError("stream died") + error.partial_result = _chat( + [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + summary = _run([error]) + # The forecast landed before the stream broke, so it is still scored -- but the turn + # did not complete, which strict_pass requires. + assert summary.best.evaluation.executed is True + assert summary.best.evaluation.turn_completed is False + + +def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat(calls), RuntimeError("boom")], k=2) + assert len(summary.run_results) == 2 + assert summary.pass_at_k is True + + +def test_k_must_be_at_least_one(): + with pytest.raises(ValueError, match="k must be >= 1"): + run_agentic_forecasting(host="h", token="t", workspace_id="w", question="q", expected_output={}, k=0) + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate_item(calls): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _chat(calls) + with patch(f"{_MODULE}.ChatClient", return_value=client): + return evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=_EXPECTED, + ) + + +def test_detail_reports_the_config_the_agent_chose(): + outcome = _evaluate_item( + [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + assert outcome.detail["actual_forecast_config"]["forecast_period"] == 3 + assert outcome.detail["actual_metrics"] == ["metric/spend"] + assert outcome.detail["asserted"] == ["forecast_period", "metric"] + assert "latency_breakdown" in outcome.detail + assert outcome.runs_passed == 1 + + +def test_a_wrong_horizon_raises_naming_what_the_agent_actually_did(): + with pytest.raises(ForecastingAssertionError) as exc_info: + _evaluate_item( + [_tc("create_adhoc_visualization", _viz_args(period=12)), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + + error = exc_info.value + assert "period_correct=False" in str(error) + assert error.detail["actual_forecast_config"]["forecast_period"] == 12 + assert error.runs_passed == 0 + assert error.conversation_id == "conv-1" + + +# ── confidence and seasonality ────────────────────────────────────────────── + + +def test_confidence_is_checked_when_pinned(): + expected = {"forecast_confidence": 0.99} + assert _evaluate(_viz(confidence=0.99), {"success": True}, expected=expected).confidence_correct is True + assert _evaluate(_viz(confidence=0.95), {"success": True}, expected=expected).confidence_correct is False + + +def test_seasonality_is_checked_when_pinned(): + expected = {"forecast_seasonal": True} + assert _evaluate(_viz(seasonal=True), {"success": True}, expected=expected).seasonal_correct is True + assert _evaluate(_viz(seasonal=False), {"success": True}, expected=expected).seasonal_correct is False + + +def test_an_absent_seasonal_counts_as_the_tools_default(): + """The tool defaults seasonal to false, so an agent that leaves it alone has asked for + a non-seasonal forecast -- failing that would penalise correct behaviour.""" + viz = _viz() + del viz["config"]["forecast_seasonal"] + assert _evaluate(viz, {"success": True}, expected={"forecast_seasonal": False}).seasonal_correct is True + + +def test_confidence_and_seasonality_are_unasserted_by_default(): + ev = _evaluate(_viz(confidence=0.5, seasonal=True), {"success": True}) + assert ev.confidence_correct is True + assert ev.seasonal_correct is True + assert "forecast_confidence" not in ev.asserted + assert "forecast_seasonal" not in ev.asserted + + +# ── cross-turn extraction ─────────────────────────────────────────────────── + + +def test_a_chart_built_on_an_earlier_turn_is_still_the_one_scored(): + """The agent may build the chart on one turn and forecast on the next -- reading only + the current turn's calls would drop the visualization the forecast actually ran on and + fail a correct run for an empty config.""" + summary = _run( + [ + _chat([_tc("create_adhoc_visualization", _viz_args())], text="Building the chart, one moment."), + _chat([_tc("execute_forecast", {"visualization_ref": "viz_1"}, _OK_FORECAST)]), + ] + ) + + assert summary.best.evaluation.executed is True + assert summary.best.evaluation.period_correct is True + assert summary.best.evaluation.metric_correct is True + assert summary.pass_at_k is True + + +# ── Langfuse scoring ──────────────────────────────────────────────────────── + + +class _FakeCtx: + """Records what the deferred Langfuse block writes, without a Langfuse.""" + + def __init__(self): + self.scores: dict[str, float] = {} + # Not named `quality`: the method below would overwrite itself on first call. + self.quality_call: dict = {} + # stamp_gate_metadata writes the gate and K here, the way every other kind does. + self.run_metadata: dict = {} + + def trace(self, _conversation_id): + return None + + @contextmanager + def observe(self, _trace, _run_idx): + yield "trace-id" + + def score(self, _tid, *, name, value, data_type): + self.scores[name] = value + + def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): + self.quality_call = {"strict_checks": strict_checks, "latency_sec": latency_sec, "cost_usd": cost_usd} + + +def _scored(expected_output, calls=None): + """Run one item with Langfuse on, then execute the deferred block against a fake ctx.""" + client = MagicMock() + client.create_conversation.return_value = "conv-1" + default = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + client.send_message.return_value = _chat(calls if calls is not None else default) + captured = {} + + def _capture(_link, _identity, **kwargs): + captured.update(kwargs) + + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.submit_trace_scoring", side_effect=_capture), + ): + try: + evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=expected_output, + langfuse=MagicMock(), + dataset_item_id="ds-1", + ) + except ForecastingAssertionError: + pass # scores are written before the pass@K raise, which is the point + + ctx = _FakeCtx() + captured["write_scores"](ctx) + return ctx + + +def test_only_the_checks_the_fixture_pinned_are_scored(): + """An unasserted check is True internally so it cannot fail a run. Publishing that as a + BOOLEAN 1 would claim the evaluator verified something it never looked at.""" + ctx = _scored({"forecast_period": 3}) # metric, confidence, seasonality unpinned + + assert ctx.scores["forecast_period_correct"] == 1.0 + for absent in ("forecast_metric_correct", "forecast_confidence_correct", "forecast_seasonal_correct"): + assert absent not in ctx.scores + # The process checks are unconditional -- they are always actually evaluated. + assert set(ctx.scores) >= {"forecast_triggered", "forecast_executed", "forecast_enabled"} + + +def test_every_pinned_check_is_scored(): + ctx = _scored( + {"metric": "metric/spend", "forecast_period": 3, "forecast_confidence": 0.95, "forecast_seasonal": False} + ) + for name in ( + "forecast_period_correct", + "forecast_metric_correct", + "forecast_confidence_correct", + "forecast_seasonal_correct", + ): + assert ctx.scores[name] == 1.0 + + +def test_cost_is_reported_even_when_the_tool_was_never_reached(): + """A run that answered without forecasting still spent tokens; gating cost on + ev.triggered hid that and understated what the item cost.""" + ctx = _scored(_EXPECTED, calls=[]) + assert "cost_usd" in ctx.quality_call + + +# ── the gate decides the verdict, not pass@K alone ────────────────────────── +# +# Every other multi-run kind takes `gate` and asks `gate_passed(...)`. Forecasting computed +# pass^K and then ignored it, so `--gate power` on a flaky item reported a pass on the +# strength of one good run out of K -- the precise case the gate exists to catch. + + +def _two_runs(gate=None): + """One passing run then one failing one, which pass@K clears and pass^K must not.""" + passing = _chat([_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)]) + failing = _chat([]) + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2"] + client.send_message.side_effect = [passing, failing, failing, failing, failing, failing] + kwargs = {"gate": gate} if gate is not None else {} + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response", return_value="use spend, 3 months"), + ): + return evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=_EXPECTED, + k=2, + **kwargs, + ) + + +def test_power_gate_fails_an_item_where_only_some_runs_passed(): + with pytest.raises(ForecastingAssertionError) as excinfo: + _two_runs(gate="power") + assert "pass^2" in str(excinfo.value) + assert excinfo.value.runs_passed == 1 + assert excinfo.value.runs_effective == 2 + + +def test_any_gate_still_passes_the_same_item(): + """The default is unchanged: one passing run out of K clears pass@K.""" + outcome = _two_runs(gate="any") + assert outcome.runs_passed == 1 + assert outcome.runs_effective == 2 + + +def test_the_default_gate_is_pass_at_k(): + outcome = _two_runs() + assert outcome.runs_passed == 1 + + +# ── the simulated user's reply ────────────────────────────────────────────── + + +def test_the_reply_carries_every_hint_the_fixture_pins(): + """Anything the fixture pins is also scored, so withholding it would let the harness fail + a run on a value it refused to supply: the agent asks which confidence level to use, the + simulated user guesses, and `confidence_correct` reports the guess as the agent's error.""" + prompt = _build_clarification_prompt( + "Which confidence level, and should I model seasonality?", + { + "metric": "metric/spend", + "forecast_period": 3, + "granularity": "MONTH", + "forecast_confidence": 0.8, + "forecast_seasonal": True, + }, + ) + assert "metric/spend" in prompt + assert "3 periods ahead" in prompt + assert "MONTH" in prompt + assert "confidence level is 0.8" in prompt + assert "seasonality should be modelled" in prompt + + +def test_a_pinned_false_seasonal_is_an_answer_not_an_absence(): + """`is not None`, not a truth test. `forecast_seasonal: false` is scored -- the tool + defaults it to false -- so the simulated user has to be able to say so.""" + prompt = _build_clarification_prompt("Seasonal?", {**_EXPECTED, "forecast_seasonal": False}) + assert "seasonality should not be modelled" in prompt + + +def test_an_unpinned_hint_is_dropped_rather_than_asserted_as_none(): + prompt = _build_clarification_prompt("Which measure?", {"metric": "metric/spend"}) + assert "confidence" not in prompt + assert "seasonality" not in prompt + assert "None" not in prompt diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index ff5b574bf..b2dd25445 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -98,6 +98,7 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ("agentic_forecasting", {"forecast_period": 3}, "evaluate_agentic_forecasting"), ( "agentic_dashboard_summary", {"must_include": ["States the overall trend."]}, @@ -890,6 +891,7 @@ def test_an_errored_item_without_timings_keeps_its_zero_defaults(): ("agentic_what_if", "evaluate_agentic_what_if", {}), ("agentic_anomaly_detection", "evaluate_agentic_anomaly_detection", {}), ("agentic_document_skill", "evaluate_agentic_document_skill", {}), + ("agentic_forecasting", "evaluate_agentic_forecasting", {}), ("agentic_dashboard_summary", "evaluate_agentic_dashboard_summary", {}), ("agentic_conversation", "evaluate_agentic_conversation", {"id": "c1", "expected_skills": [], "turns": []}), ] diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index 5a230f3be..af4b70d65 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -120,6 +120,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"), ("conversation", "evaluate_agentic_conversation"), + ("forecasting", "evaluate_agentic_forecasting"), ("dashboard_summary", "evaluate_agentic_dashboard_summary"), ("anomaly_detection", "evaluate_agentic_anomaly_detection"), ("what_if", "evaluate_agentic_what_if"),