diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 66bcf45c8..ad1514ef9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -18,6 +18,7 @@ from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill +from gooddata_eval.core.agentic.forecasting import evaluate_agentic_forecasting from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail from gooddata_eval.core.agentic.kda_skill import evaluate_agentic_kda_skill @@ -57,6 +58,7 @@ class _LfKw(TypedDict, total=False): "agentic_guardrail", "agentic_conversation", "agentic_kda_skill", + "agentic_forecasting", "agentic_dashboard_summary", "agentic_anomaly_detection", "agentic_what_if", @@ -305,6 +307,19 @@ def _dispatch_agentic( user_context=item.user_context, **lf_kw, ) + elif kind == "agentic_forecasting": + return evaluate_agentic_forecasting( + host=host, + token=token, + workspace_id=workspace_id, + question=item.question, + expected_output=eo if isinstance(eo, dict) else {}, + k=k, + gate=gate, + agent_id=agent_id, + user_context=item.user_context, + **lf_kw, + ) elif kind == "agentic_dashboard_summary": summary_input = item.summary_input if summary_input is None: @@ -428,6 +443,18 @@ def _apply_timings(item_report: ItemReport, timings: Any) -> None: item_report.simulated_user_latency_s = timings.simulated_user_s +def _apply_best_run_latency(item_report: ItemReport, source: Any) -> None: + """Copy the rank-selected run's own wall time, mirroring the single-shot path's + ``ItemReport.best_run_latency_s`` (see ``core/runner.py``'s ``_run_one_item``). + + Kinds not yet wired to measure it pass None and keep the field at its own None + default rather than reporting an invented number. + """ + best_run_latency_s = getattr(source, "best_run_latency_s", None) + if best_run_latency_s is not None: + item_report.best_run_latency_s = best_run_latency_s + + def run_agentic_items( items: list[DatasetItem], host: str, @@ -525,6 +552,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.response_id = response_id item_report.best_detail = detail or {} _apply_timings(item_report, getattr(outcome, "timings", None)) + _apply_best_run_latency(item_report, outcome) _apply_run_counts(item_report, outcome) _apply_failed_runs(item_report, outcome) _apply_run_latencies(item_report, outcome) @@ -536,6 +564,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: item_report.response_id = getattr(exc, "response_id", None) item_report.best_detail = getattr(exc, "detail", None) or {} _apply_timings(item_report, getattr(exc, "timings", None)) + _apply_best_run_latency(item_report, exc) _apply_run_counts(item_report, exc) _apply_failed_runs(item_report, exc) _apply_run_latencies(item_report, exc) @@ -579,6 +608,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: # runs and then raised would otherwise show up in the per-response table and not in # the per-conversation one. traces.discard() + _apply_best_run_latency(item_report, exc) finally: item_report.latency_s = time.perf_counter() - t0 item_report.trace_latencies_s = traces.stop() diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py new file mode 100644 index 000000000..576aee838 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py @@ -0,0 +1,123 @@ +# (C) 2026 GoodData Corporation +"""The common tail of every ``evaluate_agentic_*`` function. + +Before this module, each of the agentic evaluators hand-wrote the same block: copy +``reasoning_steps``/``conversation_id``/``response_id``/``detail``/``runs_passed``/ +``runs_effective``/``best_run_latency_s`` (and sometimes ``timings``) either onto a +raised exception or into the returned ``AgenticEvalOutcome``. Eleven independent copies +of the same ~7 lines meant a new universal field needed eleven edits, not one, and +nothing failed loudly when a copy was missed -- exactly what happened to ``timings`` +(see ``AgenticAssertionError``'s docstring: "while it lived in eight copies, two +declared timings and six did not") and again to ``best_run_latency_s`` (every agentic +kind silently reported it as ``null`` until this was noticed and fixed kind-by-kind). + +``agentic_detail`` has the same motivation for the ``detail`` dict's timeline fields: +``timeline_detail`` builds both ``latency_breakdown`` and ``tool_calls`` from the same +events so they stay index-aligned, but roughly half the evaluators called +``build_latency_breakdown`` directly and silently never got a ``tool_calls`` key. +""" + +from __future__ import annotations + +from typing import Any, NoReturn + +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + timeline_detail, +) +from gooddata_eval.core.timing import PhaseTimings + +__all__ = ["agentic_detail", "agentic_success", "raise_agentic_failure"] + + +def agentic_detail( + tool_call_events: list[ToolCallEvent], + reasoning_step_events: list[ReasoningStepEvent] | None, + **kind_specific: Any, +) -> dict: + """A kind's full ``detail`` dict: its own fields plus the universal timeline ones. + + ``kind_specific`` comes first in the merge so a kind can never accidentally shadow + ``latency_breakdown``/``tool_calls`` with a same-named field of its own. + """ + return {**kind_specific, **timeline_detail(tool_call_events, reasoning_step_events)} + + +def raise_agentic_failure( + exception_cls: type[AgenticAssertionError], + message: str, + *, + reasoning_steps: list[str], + conversation_id: str, + response_id: str | None, + detail: dict, + runs_passed: int, + runs_effective: int, + best_run_latency_s: float | None, + timings: PhaseTimings | None = None, + failed_runs: list[dict] | None = None, + run_latencies_s: list[float] | None = None, +) -> NoReturn: + """Build ``exception_cls(message)`` with every common field attached, and raise it. + + ``exception_cls`` must be an ``AgenticAssertionError`` subclass -- that base class is + what declares these fields as legal targets (see its docstring). A kind whose own + "no verdict at all" branch raises a bare ``JudgeResponseError`` instead (not a subclass) + does not go through this helper for that branch. + + ``timings`` stays optional: only the three kinds that track per-run ``PhaseTimings`` + (general_question, metric_skill, dashboard_skill) pass one -- the rest keep their + own ``PhaseTimings()`` zero default, same as before this helper existed. ``failed_runs`` + and ``run_latencies_s`` are likewise optional: not every kind has built them by the + time it raises. + """ + exc = exception_cls(message) + exc.reasoning_steps = reasoning_steps + exc.conversation_id = conversation_id + exc.response_id = response_id + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective + exc.best_run_latency_s = best_run_latency_s + if timings is not None: + exc.timings = timings + if failed_runs is not None: + exc.failed_runs = failed_runs + if run_latencies_s is not None: + exc.run_latencies_s = run_latencies_s + raise exc + + +def agentic_success( + *, + reasoning_steps: list[str], + conversation_id: str | None, + response_id: str | None, + detail: dict, + runs_passed: int, + runs_effective: int, + best_run_latency_s: float | None, + timings: PhaseTimings | None = None, + failed_runs: list[dict] | None = None, + run_latencies_s: list[float] | None = None, +) -> AgenticEvalOutcome: + """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape.""" + kwargs: dict[str, Any] = { + "reasoning_steps": reasoning_steps, + "conversation_id": conversation_id, + "response_id": response_id, + "detail": detail, + "runs_passed": runs_passed, + "runs_effective": runs_effective, + "best_run_latency_s": best_run_latency_s, + } + if timings is not None: + kwargs["timings"] = timings + if failed_runs is not None: + kwargs["failed_runs"] = failed_runs + if run_latencies_s is not None: + kwargs["run_latencies_s"] = run_latencies_s + return AgenticEvalOutcome(**kwargs) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 6c5a55d67..f1f532af3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -23,6 +23,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -700,7 +701,8 @@ class AlertRunResult: # also report operator/threshold/metric/recipients as False. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED turns_used: int = 0 - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 @@ -975,6 +977,7 @@ def run_agentic_alert_skill( def _run_once(conv_id: str) -> AlertRunResult: alert_id_to_delete: str | None = None + run_started = time.monotonic() try: alert_id: str | None = None actual_args: dict = {} @@ -1260,9 +1263,25 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.eval - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + alert_created=ev.alert_created, + operator_correct=ev.operator_correct, + threshold_correct=ev.threshold_correct, + trigger_correct=ev.trigger_correct, + filters_correct=ev.filters_correct, + metric_correct=ev.metric_correct, + recipients_correct=ev.recipients_correct, + attributes_correct=ev.attributes_correct, + granularity_correct=ev.granularity_correct, + actual_alert_arguments=best.actual_alert_arguments, + # Why the loop stopped. alert_created=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + turns_used=best.turns_used, + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -1273,29 +1292,34 @@ def _write_scores(ctx: RunTraceContext) -> None: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) - exc = AlertSkillAssertionError( + raise_agentic_failure( + AlertSkillAssertionError, f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " - + ", ".join(f"{name}={value}" for name, value in ev.checks().items()) - + f". Actual args: {best.actual_alert_arguments}" + f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " + f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " + f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, " + f"recipients_correct={ev.recipients_correct}, " + f"attributes_correct={ev.attributes_correct}, " + f"granularity_correct={ev.granularity_correct}. " + f"Actual args: {best.actual_alert_arguments}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py index 1a4238983..25f2281a0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py @@ -41,6 +41,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -58,9 +59,9 @@ AgenticAssertionError, AgenticEvalOutcome, ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, - build_latency_breakdown, shift_and_index_events, ) @@ -285,8 +286,14 @@ class AnomalyRunResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from + # turn_wall_clock_sec above, which is only the final triggering turn. run_latency_s: float = 0.0 + # Why the simulated-user loop stopped -- see LoopExit. `triggered=False` alone cannot + # separate a refusal from a run that hit max_iterations while still on track (matches + # kda_skill.py/alert_skill.py). + exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @dataclass @@ -371,6 +378,7 @@ def run_agentic_anomaly_detection( ) def _run_once(conv_id: str) -> AnomalyRunResult: + run_started = time.monotonic() viz_args: dict | None = None execute_result: dict | None = None turn_wall_clock_sec: float | None = None @@ -397,6 +405,10 @@ def _accumulate(result: ChatResult) -> None: all_reasoning_step_events.extend(result.reasoning_step_events or []) run_started = time.monotonic() + # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that + # simply runs out of range() is labelled correctly with no trailing else. + exit_reason = LoopExit.BUDGET_EXHAUSTED + for iteration in range(max_iterations): try: chat_result = client.send_message(conv_id, current_question) @@ -409,6 +421,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) viz_args, execute_result = _extract_anomaly_calls(all_tool_call_events) turn_completed = False + exit_reason = LoopExit.CHAT_ERROR break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -422,8 +435,10 @@ def _accumulate(result: ChatResult) -> None: if execute_result is not None: # The turn that ran the detection, not an earlier disambiguation turn. turn_wall_clock_sec = chat_result.turn_wall_clock_sec + exit_reason = LoopExit.SUCCESS break if not response_text: + exit_reason = LoopExit.AGENT_SILENT break if iteration >= max_iterations - 1: break @@ -432,6 +447,7 @@ def _accumulate(result: ChatResult) -> None: disambiguated = True except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run _log.warning("Simulated anomaly user reply failed for conversation %s: %s", conv_id, exc) + exit_reason = LoopExit.SIMULATED_USER_FAILED break return AnomalyRunResult( @@ -440,6 +456,7 @@ def _accumulate(result: ChatResult) -> None: actual_visualization=viz_args, actual_execute_result=execute_result, turn_wall_clock_sec=turn_wall_clock_sec, + exit_reason=exit_reason, reasoning_steps=reasoning_steps, response_id=response_id, tool_call_events=all_tool_call_events, @@ -493,25 +510,29 @@ class AnomalyDetectionAssertionError(AgenticAssertionError): def _detail(best: AnomalyRunResult) -> dict[str, Any]: ev = best.evaluation - return { - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "metric_correct": ev.metric_correct, - "granularity_correct": ev.granularity_correct, + return agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + triggered=ev.triggered, + executed=ev.executed, + success=ev.success, + turn_completed=ev.turn_completed, + metric_correct=ev.metric_correct, + granularity_correct=ev.granularity_correct, # Which content checks the fixture pinned -- without it a run that verified nothing # reads the same as one where everything matched. - "asserted": ev.asserted, - "disambiguated": ev.disambiguated, - "actual_metrics": sorted(_metric_uris(best.actual_visualization)), - "actual_granularity": _inferred_granularity(best.actual_visualization), + asserted=ev.asserted, + disambiguated=ev.disambiguated, + actual_metrics=sorted(_metric_uris(best.actual_visualization)), + actual_granularity=_inferred_granularity(best.actual_visualization), # Reported, never asserted: whether a real series contains anomalies is a property # of the data, so a fixture demanding some would fail on the next warehouse refresh. - "anomaly_point_count": _point_count(best.actual_execute_result), - "actual_execute_result": best.actual_execute_result, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + anomaly_point_count=_point_count(best.actual_execute_result), + actual_execute_result=best.actual_execute_result, + # Why the loop stopped. triggered=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + ) def evaluate_agentic_anomaly_detection( @@ -648,24 +669,28 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Analysed {detail['actual_metrics']} at {detail['actual_granularity']}. " f"Actual execute result: {best.actual_execute_result}." ) - exc = AnomalyDetectionAssertionError(message) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = len(summary.run_results) - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=len(summary.run_results), + raise_agentic_failure( + AnomalyDetectionAssertionError, + message, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, + ) + + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index e980958bc..92f106340 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -56,6 +56,7 @@ summarize_visualizations, ) from gooddata_eval.core.agentic._gate import log_gate_scores +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -82,7 +83,6 @@ ReasoningStepEvent, ToolCallEvent, shift_and_index_events, - timeline_detail, ) from gooddata_eval.core.scoring import ( check_filters, @@ -729,8 +729,10 @@ class ConversationResult: stalled_turns: int = 0 mode: ConversationMode = "legacy" judge_model: str | None = None - # The whole conversation's wall time, from just before its first message to the end of its - # last turn's scoring; conversation create/delete is not part of it. + # This conversation's own wall time, across every turn -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). There is no + # K-of-N selection for this kind (see run_agentic_conversation's docstring), so + # this is simply the one conversation's own time, not a "best of" pick. run_latency_s: float = 0.0 @@ -857,6 +859,7 @@ def run_agentic_conversation( A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode neither sends ``clarifying_answers`` nor lets the clarification checks fail the item, so it raises ``ValueError``. """ + run_started = time.monotonic() resolved_mode = resolve_conversation_mode(mode) graded = [t.turn_id for t in fixture.turns if t.expects_clarification is not None] if resolved_mode == "legacy" and graded: @@ -1307,21 +1310,22 @@ def run_agentic_conversation( def _conversation_detail(result: ConversationResult) -> dict: - return { - "mode": result.mode, - "full_skill_coverage": result.full_skill_coverage, - "conversation_success": result.conversation_success, - "context_success": result.context_success, - "context_kept_rate": result.context_kept_rate, - "turns_before_first_break": result.turns_before_first_break, - "lost_context_clarifications": result.lost_context_clarifications, - "stalled_turns": result.stalled_turns, - "total_clarification_turns": result.total_clarification_turns, - "max_clarification_turns": result.max_clarification_turns, - "judge_model": result.judge_model, - "turns": [tr.detail() for tr in result.turn_results], - **timeline_detail(result.tool_call_events, result.reasoning_step_events), - } + return agentic_detail( + result.tool_call_events, + result.reasoning_step_events, + mode=result.mode, + full_skill_coverage=result.full_skill_coverage, + conversation_success=result.conversation_success, + context_success=result.context_success, + context_kept_rate=result.context_kept_rate, + turns_before_first_break=result.turns_before_first_break, + lost_context_clarifications=result.lost_context_clarifications, + stalled_turns=result.stalled_turns, + total_clarification_turns=result.total_clarification_turns, + max_clarification_turns=result.max_clarification_turns, + judge_model=result.judge_model, + turns=[tr.detail() for tr in result.turn_results], + ) class ConversationAssertionError(AgenticAssertionError): @@ -1508,18 +1512,21 @@ def _write_scores(ctx: RunTraceContext) -> None: f"full_skill_coverage={result.full_skill_coverage}. " f"Failed turns: {legacy_failed}" ) - exc = ConversationAssertionError(message) - exc.reasoning_steps = result.reasoning_steps - exc.conversation_id = result.conversation_id - exc.response_id = result.response_id - exc.detail = detail # This kind takes no k and drives its fixture exactly once, whatever --runs asks # for. Saying so explicitly stops the report claiming K runs that never happened. - exc.runs_passed = 0 - exc.runs_effective = 1 - exc.run_latencies_s = [result.run_latency_s] - raise exc - return AgenticEvalOutcome( + raise_agentic_failure( + ConversationAssertionError, + message, + reasoning_steps=result.reasoning_steps, + conversation_id=result.conversation_id, + response_id=result.response_id, + detail=detail, + runs_passed=0, + runs_effective=1, + run_latencies_s=[result.run_latency_s], + best_run_latency_s=result.run_latency_s, + ) + return agentic_success( reasoning_steps=result.reasoning_steps, conversation_id=result.conversation_id, response_id=result.response_id, @@ -1527,4 +1534,5 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_passed=1, runs_effective=1, run_latencies_s=[result.run_latency_s], + best_run_latency_s=result.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index 1c7aaf771..2fafc30ba 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -19,6 +19,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -29,20 +30,21 @@ utc_now, ) from gooddata_eval.core.chat.render import render_answer_text -from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.langfuse.item_scope import item_scope from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, shift_and_index_events, ) from gooddata_eval.core.scoring import AAC_VISUALIZATION_TYPES, same_viz_type -from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings _DEFAULT_K = 1 # Matches kda_skill and visualization. The reply this loop sends is built from the expectation @@ -1708,6 +1710,12 @@ class DashboardRunResult: timings: PhaseTimings = field(default_factory=PhaseTimings) # This run's own wall time, from just before its chat call to the end of its scoring. run_latency_s: float = 0.0 + # Why the loop stopped -- see LoopExit. A dashboard not produced alone cannot tell a + # refusal from a run that hit max_iterations while still on track (matches + # kda_skill.py/alert_skill.py). CHAT_ERROR is reachable; SIMULATED_USER_FAILED is not -- + # this loop's follow-up is built from the expectation by build_simulated_reply, with no + # judge call that could fail. + exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @dataclass @@ -2024,12 +2032,43 @@ def _execute_single_dashboard_run( turn_offset = 0.0 tool_index_offset = 0 reasoning_index_offset = 0 + # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that + # simply runs out of range() (or the is_edit "no reply of its own" branch below, which + # is the same shape of outcome -- the agent answered, just not with the dashboard tool) + # is labelled correctly with no trailing else. + exit_reason = LoopExit.BUDGET_EXHAUSTED run_started = time.monotonic() for iteration in range(max_iterations): turns += 1 agent_started = time.monotonic() - chat_result = client.send_message(conversation_id, current_question) + try: + chat_result = client.send_message(conversation_id, current_question) + except ChatError as exc: + # Recorded rather than left to escape: this runs inside one of K runs, so an + # uncaught ChatError discarded every run already completed along with this one's + # exit_reason, and the item surfaced as a bare error with no per-run diagnosis. + # Matches metric_skill.py and conversation.py, which record it the same way. + timings.agent_s += time.monotonic() - agent_started + print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}") + partial = exc.partial_result + if partial is not None: + # Shifted and indexed exactly as a completed turn is, before anything reads + # the events: left raw, a late turn's call_ts restarts near zero and + # timeline_detail reports it as overlapping the first one. + reasoning_steps.extend(partial.reasoning_steps or []) + response_id = partial.response_id or response_id + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + partial, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(partial.tool_call_events or []) + all_reasoning_step_events.extend(partial.reasoning_step_events or []) + steps += partial.reasoning_step_count + exit_reason = LoopExit.CHAT_ERROR + break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed reasoning_steps.extend(chat_result.reasoning_steps or []) @@ -2071,9 +2110,11 @@ def _execute_single_dashboard_run( dashboard_part = _extract_dashboard_part(chat_result, "dashboard") if is_edit: patch_part = _extract_dashboard_part(chat_result, _PATCH_TYPE) + exit_reason = LoopExit.SUCCESS break if not response_text and not chat_result.tool_call_events: + exit_reason = LoopExit.AGENT_SILENT break if not expects_dashboard: # A refusal case is answered in one turn. The reply this loop would otherwise send @@ -2121,6 +2162,7 @@ def _execute_single_dashboard_run( reasoning_step_events=all_reasoning_step_events, timings=timings, run_latency_s=time.monotonic() - run_started, + exit_reason=exit_reason, ) @@ -2307,7 +2349,17 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_effective = len(summary.run_results) best = summary.best - detail: dict[str, Any] = _run_detail(best) + detail: dict[str, Any] = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + **best.evaluation.strict_checks, + **best.evaluation.diagnostics, + failures=best.evaluation.failures, + notes=best.evaluation.notes, + # Why the loop stopped. A dashboard not produced alone cannot tell a refusal from + # a run that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -2335,30 +2387,32 @@ def _write_scores(ctx: RunTraceContext) -> None: " one feature flag registers both, so check that before the model." ) notes = "; ".join(best.evaluation.notes) - exc = DashboardSkillAssertionError( + raise_agentic_failure( + DashboardSkillAssertionError, f"Dashboard skill assertion failed. {gate_note}{skill_note} " f"Checks: {best.evaluation.strict_checks}. " f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." - + (f" Notes (not scored): {notes}." if notes else "") + + (f" Notes (not scored): {notes}." if notes else ""), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.timings = item_timings - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - timings=item_timings, + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py new file mode 100644 index 000000000..98f9d3b76 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/forecasting.py @@ -0,0 +1,641 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic forecasting-skill evaluation runner. + +Unlike kda_skill, this is not limited to "the process ran". The forecasting skill refuses to +execute unless the visualization it points at carries an AAC forecast config, and that config +is where the user's request lands: + + config.forecast_enabled must be true or execute_forecast returns an error + config.forecast_period how many periods ahead -- "next 3 months" is 3 + config.forecast_confidence confidence level, 0.95 by default + config.forecast_seasonal whether seasonality is modelled + +So a fixture can state what it asked for and have it checked exactly, with no judge: the +period the agent chose is a number in the tool call it made. The measure it forecast is +checkable the same way, off the visualization's own fields. +""" + +from __future__ import annotations + +import logging +import os +import time +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) + +_log = logging.getLogger(__name__) + +_DEFAULT_K = 1 +# The agent routinely asks which measure to forecast before it builds anything (observed +# live: "your data has two different Spend metrics"), so the budget covers a couple of +# disambiguation rounds plus slack, matching kda_skill's reasoning. +_DEFAULT_MAX_ITERATIONS = 4 + + +def _build_clarification_prompt(agent_message: str, expected_output: dict) -> str: + """The simulated-user reply, mentioning only the hints the fixture actually supplies. + + An absent hint is dropped from the prompt entirely rather than asserted as a literal + "None", which would answer a question the agent never asked with a wrong value. + """ + hints: list[str] = [] + metric = expected_output.get("metric") + if metric: + hints.append(f"the measure to forecast is {metric}") + period = expected_output.get("forecast_period") + if period is not None: + hints.append(f"the forecast horizon is {period} periods ahead") + granularity = expected_output.get("granularity") + if granularity: + hints.append(f"the time granularity is {granularity}") + # Both are scored when the fixture pins them, so withholding them here would let the + # harness fail a run on a value it refused to supply: the agent asks which confidence + # level to use, the simulated user guesses, and `confidence_correct` reports the guess + # as the agent's error. `is not None` rather than a truth test -- a pinned + # `forecast_seasonal: false` is an answer, and `0.0` would be a confidence level. + confidence = expected_output.get("forecast_confidence") + if confidence is not None: + hints.append(f"the confidence level is {confidence}") + seasonal = expected_output.get("forecast_seasonal") + if seasonal is not None: + hints.append(f"seasonality {'should' if seasonal else 'should not'} be modelled") + reference = "; ".join(hints) + return ( + f"You are simulating a user in a conversation with a BI assistant that forecasts metric " + f"values. The assistant asked: '{agent_message}'. " + + (f"For reference, {reference}. " if reference else "") + + "Reply briefly as the user, answering whichever of those the assistant actually asked about." + ) + + +def generate_simulated_forecast_response(agent_message: str, expected_output: dict) -> str: + """Generate a user reply to keep the forecasting conversation going (gpt-4o-mini). + + Always OpenAI regardless of the workspace's own model: this is harness plumbing, not the + system under test. + """ + try: + from openai import OpenAI # noqa: PLC0415 + except ImportError as exc: + raise RuntimeError("openai package is required for generate_simulated_forecast_response") from exc + + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise OSError("OPENAI_API_KEY environment variable is not set") + + client = OpenAI(api_key=api_key) + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": _build_clarification_prompt(agent_message, expected_output)}], + max_tokens=150, + temperature=0, + timeout=30, + ) + return response.choices[0].message.content or "Please proceed with the most complete option." + + +def _extract_forecast_calls( + tool_call_events: list[ToolCallEvent], +) -> tuple[dict | None, dict | None]: + """Return (visualization_args, execute_result) for the LAST create/execute pair. + + A new create_adhoc_visualization clears any earlier execute result: that result belongs + to the visualization it followed, not to this one. Picking the last of each + independently would pair a fresh chart with a stale forecast. + """ + viz_args: dict | None = None + execute_result: dict | None = None + for tc in tool_call_events: + if tc.function_name == "create_adhoc_visualization": + args = tc.parsed_arguments() or {} + viz = args.get("visualization") + viz_args = viz if isinstance(viz, dict) else args + execute_result = None + elif tc.function_name == "execute_forecast" and tc.result: + execute_result = tc.parsed_result() + return viz_args, execute_result + + +def _forecast_config(viz_args: dict | None) -> dict: + config = (viz_args or {}).get("config") + return config if isinstance(config, dict) else {} + + +def _metric_uris(viz_args: dict | None) -> set[str]: + """The metric URIs the visualization measures, resolved through its field aliases. + + Deliberately not core.scoring.get_metric_uri_set: that takes a parsed + CreatedVisualization, while this reads the raw tool-call arguments, where a field may be + a bare URI string rather than an object. + """ + query = (viz_args or {}).get("query") + fields = query.get("fields") if isinstance(query, dict) else None + if not isinstance(fields, dict): + return set() + uris: set[str] = set() + for alias in (viz_args or {}).get("metrics") or list(fields): + name = alias.get("field") if isinstance(alias, dict) else alias + field_def = fields.get(name) + if isinstance(field_def, dict) and field_def.get("using"): + uris.add(str(field_def["using"])) + elif isinstance(field_def, str): + uris.add(field_def) + return {u for u in uris if u.startswith(("metric/", "fact/"))} + + +@dataclass +class ForecastEvaluation: + """Scores for a single forecasting run. + + ``triggered``/``executed``/``success``/``turn_completed`` are the process checks every + skill kind shares. ``forecast_enabled``, ``period_correct`` and ``metric_correct`` are + content checks, and each is True when the fixture did not ask for it -- an unstated + expectation must not fail a run, and must not silently pass one either, which is why + ``asserted`` records which of them the fixture actually pinned. + """ + + triggered: bool + executed: bool + success: bool + turn_completed: bool + forecast_enabled: bool + period_correct: bool + metric_correct: bool + confidence_correct: bool + seasonal_correct: bool + asserted: list[str] = field(default_factory=list) + disambiguated: bool = False + + @property + def strict_pass(self) -> bool: + return all( + [ + self.triggered, + self.executed, + self.success, + self.turn_completed, + self.forecast_enabled, + self.period_correct, + self.metric_correct, + self.confidence_correct, + self.seasonal_correct, + ] + ) + + +@dataclass +class ForecastRunResult: + """Outcome of one run (one conversation, up to max_iterations messages).""" + + conversation_id: str + evaluation: ForecastEvaluation + actual_visualization: dict | None + actual_execute_result: dict | None + turn_wall_clock_sec: float | None = None + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # This run's own wall time, from just before its chat call to the end of its scoring. + run_latency_s: float = 0.0 + + +@dataclass +class AgenticForecastSummary: + """Aggregated outcome of K runs for one forecasting item.""" + + run_results: list[ForecastRunResult] + pass_at_k: bool + pass_power_k: bool + best: ForecastRunResult + + +def _evaluate_run( + viz_args: dict | None, + execute_result: dict | None, + expected_output: dict, + turn_completed: bool, + disambiguated: bool = False, +) -> ForecastEvaluation: + triggered = execute_result is not None or viz_args is not None + executed = execute_result is not None + success = executed and execute_result.get("success") is True + + config = _forecast_config(viz_args) + # None is "not set", which the tool treats as disabled -- only an explicit true enables it. + forecast_enabled = config.get("forecast_enabled") is True + + asserted: list[str] = [] + expected_period = expected_output.get("forecast_period") + if expected_period is None: + period_correct = True + else: + asserted.append("forecast_period") + actual_period = config.get("forecast_period") + period_correct = isinstance(actual_period, int | float) and float(actual_period) == float(expected_period) + + expected_metric = expected_output.get("metric") + if not expected_metric: + metric_correct = True + else: + asserted.append("metric") + wanted = {expected_metric} if isinstance(expected_metric, str) else set(expected_metric) + metric_correct = bool(_metric_uris(viz_args) & wanted) + + expected_confidence = expected_output.get("forecast_confidence") + if expected_confidence is None: + confidence_correct = True + else: + asserted.append("forecast_confidence") + actual_confidence = config.get("forecast_confidence") + confidence_correct = isinstance(actual_confidence, int | float) and float(actual_confidence) == float( + expected_confidence + ) + + expected_seasonal = expected_output.get("forecast_seasonal") + if expected_seasonal is None: + seasonal_correct = True + else: + asserted.append("forecast_seasonal") + # The tool defaults seasonal to False, so an absent value means "not seasonal" -- + # reading it as a mismatch would fail an agent that simply left the default alone. + actual_seasonal = config.get("forecast_seasonal") + seasonal_correct = bool(actual_seasonal) == bool(expected_seasonal) + + return ForecastEvaluation( + triggered=triggered, + executed=executed, + success=success, + turn_completed=turn_completed, + forecast_enabled=forecast_enabled, + period_correct=period_correct, + metric_correct=metric_correct, + confidence_correct=confidence_correct, + seasonal_correct=seasonal_correct, + asserted=asserted, + disambiguated=disambiguated, + ) + + +def run_agentic_forecasting( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, + user_context: dict | None = None, +) -> AgenticForecastSummary: + """Run the forecasting agentic evaluation K times and return a summary. + + A run ends as soon as execute_forecast returns, which is the goal signal. Short of that + it keeps sending simulated replies up to ``max_iterations``, with no attempt to classify + whether the agent's text was really a question: missing a genuine clarifying question + hard-fails the run, while answering a final answer costs one harmless extra turn. + """ + if k < 1: + raise ValueError(f"k must be >= 1, got {k}") + run_results: list[ForecastRunResult] = [] + client = ChatClient( + host=host, + token=token, + workspace_id=workspace_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + user_context=user_context, + ) + + def _run_once(conv_id: str) -> ForecastRunResult: + viz_args: dict | None = None + execute_result: dict | None = None + turn_wall_clock_sec: float | None = None + turn_completed = False + disambiguated = False + current_question = question + reasoning_steps: list[str] = [] + response_id: str | None = None + all_tool_call_events: list[ToolCallEvent] = [] + all_reasoning_step_events: list[ReasoningStepEvent] = [] + turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time + tool_index_offset = 0 + reasoning_index_offset = 0 + + def _accumulate(result: ChatResult) -> None: + nonlocal turn_offset, tool_index_offset, reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(result.tool_call_events or []) + all_reasoning_step_events.extend(result.reasoning_step_events or []) + + run_started = time.monotonic() + for iteration in range(max_iterations): + try: + chat_result = client.send_message(conv_id, current_question) + except Exception as exc: # noqa: BLE001 -- end this run, not the whole item + _log.warning("Forecast send_message failed for conversation %s: %s", conv_id, exc) + partial = getattr(exc, "partial_result", None) + if partial is not None: + reasoning_steps.extend(partial.reasoning_steps or []) + response_id = partial.response_id or response_id + _accumulate(partial) + viz_args, execute_result = _extract_forecast_calls(all_tool_call_events) + turn_completed = False + break + reasoning_steps.extend(chat_result.reasoning_steps or []) + response_id = chat_result.response_id or response_id + _accumulate(chat_result) + # Over every turn so far, not just this one: the agent may build the chart on + # one turn and forecast on the next, and reading a single turn would drop the + # visualization the forecast actually ran on. + viz_args, execute_result = _extract_forecast_calls(all_tool_call_events) + response_text = render_answer_text(chat_result) + turn_completed = chat_result.stream_ended and bool(response_text) + if execute_result is not None: + # The turn that actually forecast, not an earlier disambiguation turn. + turn_wall_clock_sec = chat_result.turn_wall_clock_sec + break + if not response_text: + break + if iteration >= max_iterations - 1: + break + try: + current_question = generate_simulated_forecast_response(response_text, expected_output) + disambiguated = True + except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run + _log.warning("Simulated forecast user reply failed for conversation %s: %s", conv_id, exc) + break + + return ForecastRunResult( + conversation_id=conv_id, + evaluation=_evaluate_run(viz_args, execute_result, expected_output, turn_completed, disambiguated), + actual_visualization=viz_args, + actual_execute_result=execute_result, + turn_wall_clock_sec=turn_wall_clock_sec, + reasoning_steps=reasoning_steps, + response_id=response_id, + tool_call_events=all_tool_call_events, + reasoning_step_events=all_reasoning_step_events, + run_latency_s=time.monotonic() - run_started, + ) + + try: + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append(_run_once(conv_id_0)) + finally: + if initial_conversation_id is None: # only delete conversations we created + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + conv_id = client.create_conversation() + try: + run_results.append(_run_once(conv_id)) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + pass_at_k = any(r.evaluation.strict_pass for r in run_results) + pass_power_k = all(r.evaluation.strict_pass for r in run_results) + best = max( + run_results, + key=lambda r: sum( + [ + r.evaluation.triggered, + r.evaluation.executed, + r.evaluation.success, + r.evaluation.turn_completed, + r.evaluation.forecast_enabled, + r.evaluation.period_correct, + r.evaluation.metric_correct, + r.evaluation.confidence_correct, + r.evaluation.seasonal_correct, + ] + ), + ) + return AgenticForecastSummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +class ForecastingAssertionError(AgenticAssertionError): + """Raised when a forecasting evaluation fails.""" + + +def _detail(best: ForecastRunResult) -> dict[str, Any]: + ev = best.evaluation + return { + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "forecast_enabled": ev.forecast_enabled, + "period_correct": ev.period_correct, + "metric_correct": ev.metric_correct, + "confidence_correct": ev.confidence_correct, + "seasonal_correct": ev.seasonal_correct, + # Which content checks the fixture pinned. Without it a run where nothing was + # asserted is indistinguishable in the report from one where everything matched. + "asserted": ev.asserted, + "disambiguated": ev.disambiguated, + "actual_forecast_config": _forecast_config(best.actual_visualization), + "actual_metrics": sorted(_metric_uris(best.actual_visualization)), + "actual_execute_result": best.actual_execute_result, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } + + +def evaluate_agentic_forecasting( + host: str, + token: str, + workspace_id: str, + question: str, + expected_output: dict, + k: int = _DEFAULT_K, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + initial_conversation_id: str | None = None, + gate: EvalGate = DEFAULT_GATE, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "forecasting", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + user_context: dict | None = None, +) -> AgenticEvalOutcome: + """Run forecasting evaluation, log to Langfuse, and raise ForecastingAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_forecasting( + host=host, + token=token, + workspace_id=workspace_id, + question=question, + expected_output=expected_output, + k=k, + max_iterations=max_iterations, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + user_context=user_context, + ) + + if langfuse is not None and dataset_item_id: + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + ev = run.evaluation + strict_checks = { + "forecast_triggered": ev.triggered, + "forecast_executed": ev.executed, + "forecast_success": ev.success, + "forecast_turn_completed": ev.turn_completed, + "forecast_enabled": ev.forecast_enabled, + } + # Only the content checks the fixture actually pinned. An unasserted check + # is True internally so it cannot fail a run, but publishing that as a + # BOOLEAN 1 would claim the evaluator verified something it never looked at. + strict_checks.update( + { + key: value + for name, key, value in ( + ("forecast_period", "forecast_period_correct", ev.period_correct), + ("metric", "forecast_metric_correct", ev.metric_correct), + ("forecast_confidence", "forecast_confidence_correct", ev.confidence_correct), + ("forecast_seasonal", "forecast_seasonal_correct", ev.seasonal_correct), + ) + if name in ev.asserted + } + ) + with ctx.observe(pt, run_idx) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) + ctx.quality( + tid, + strict_checks=strict_checks, + # pt.latency covers the whole conversation, which is the item's real + # elapsed cost when the agent needed clarification turns to get + # there; turn_wall_clock_sec (the goal turn alone) is the fallback. + # This is what 7 of the 8 existing kinds do -- kda_skill is the + # outlier and documents its own reason. Cost is not gated on + # ev.triggered: a run that answered without ever reaching the tool + # still spent tokens, and hiding that understates what the item cost. + latency_sec=pt.latency if pt else run.turn_wall_clock_sec, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + # The question this run answered, so a score is readable without resolving the + # conversation back to its item. + item_input=question, + ) + + best = summary.best + ev = best.evaluation + detail = _detail(best) + runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. + failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_detail) + + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + message = ( + f"Forecasting assertion failed. {gate_note} strict_pass={ev.strict_pass} " + f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " + f"turn_completed={ev.turn_completed}, forecast_enabled={ev.forecast_enabled}, " + f"period_correct={ev.period_correct}, metric_correct={ev.metric_correct}, " + f"confidence_correct={ev.confidence_correct}, seasonal_correct={ev.seasonal_correct}). " + f"Actual forecast config: {detail['actual_forecast_config']}. " + f"Actual execute result: {best.actual_execute_result}." + ) + exc = ForecastingAssertionError(message) + exc.reasoning_steps = best.reasoning_steps + exc.conversation_id = best.conversation_id + exc.response_id = best.response_id + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = len(summary.run_results) + exc.failed_runs = failed_runs + exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] + raise exc + + return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 0a55b2a00..907f0567a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -15,6 +15,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -37,7 +38,7 @@ ToolCallEvent, build_latency_breakdown, ) -from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings _DEFAULT_K = 1 @@ -241,6 +242,7 @@ def _attach_diagnostics( runs_passed: int, runs_effective: int, run_latencies_s: list[float], + best_run_latency_s: float | None, ) -> None: """Hang the report payload on a raised error, whichever error it is. @@ -256,6 +258,7 @@ def _attach_diagnostics( error.runs_effective = runs_effective error.failed_runs = failed_runs error.run_latencies_s = run_latencies_s + error.best_run_latency_s = best_run_latency_s def _run_detail(run: GeneralQuestionResult) -> dict: @@ -396,13 +399,17 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best - detail = { - **_run_detail(best), + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + judge_passed=best.passed, + judge_reasoning=best.reasoning, + actual_output=best.actual_output, # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K computed over fewer runs than --runs asked for is a weaker result and # the report has to say so. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), - } + ) # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as # non-passing here so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( @@ -420,28 +427,37 @@ def _write_scores(ctx: RunTraceContext) -> None: f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + " | ".join(unscored) ) - _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) + _attach_diagnostics( + error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, run_latency_s(best.timings) + ) error.timings = item_timings raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) - exc = GeneralQuestionAssertionError( - f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" + raise_agentic_failure( + GeneralQuestionAssertionError, + f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) - _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) - exc.timings = item_timings - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - timings=item_timings, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index 81f9cb1ad..f9e281c2d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -15,6 +15,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -87,7 +88,8 @@ class GuardrailResult: # Set when the judge returned something unreadable for THIS run. Excluded from pass@K # and from Langfuse scoring rather than counted as a failure -- see score_run. judge_error: str | None = None - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time (agent call + its grading) -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 @@ -211,6 +213,7 @@ def _attach_diagnostics( runs_passed: int, runs_effective: int, run_latencies_s: list[float], + best_run_latency_s: float | None, ) -> None: """Hang the report payload on a raised error, whichever error it is. @@ -226,6 +229,7 @@ def _attach_diagnostics( error.runs_effective = runs_effective error.failed_runs = failed_runs error.run_latencies_s = run_latencies_s + error.best_run_latency_s = best_run_latency_s def _run_detail(run: GuardrailResult) -> dict: @@ -354,12 +358,16 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best - detail = { - **_run_detail(best), + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + judge_passed=best.passed, + judge_reasoning=best.reasoning, + actual_output=best.actual_output, # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K over fewer runs than --runs asked for is a weaker result. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), - } + ) # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as # non-passing here so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( @@ -375,25 +383,34 @@ def _write_scores(ctx: RunTraceContext) -> None: f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + " | ".join(unscored) ) - _attach_diagnostics(error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) + _attach_diagnostics( + error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, best.run_latency_s + ) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) - exc = GuardrailAssertionError( - f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}" + raise_agentic_failure( + GuardrailAssertionError, + f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) - _attach_diagnostics(exc, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s) - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 7ab93c40a..683f8d42d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -19,6 +19,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -201,7 +202,9 @@ class KdaRunResult: # separate a refusal from a run that hit max_iterations while still on track. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED turns_used: int = 0 - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from + # turn_wall_clock_sec above, which is only the final create-triggering turn. run_latency_s: float = 0.0 @@ -271,6 +274,7 @@ def run_agentic_kda_skill( ) def _run_once(conv_id: str) -> KdaRunResult: + run_started = time.monotonic() create_args: dict | None = None execute_result: dict | None = None turn_wall_clock_sec: float | None = None @@ -562,9 +566,21 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.evaluation - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + triggered=ev.triggered, + executed=ev.executed, + success=ev.success, + turn_completed=ev.turn_completed, + disambiguated=ev.disambiguated, + actual_create_args=best.actual_create_args, + actual_execute_result=best.actual_execute_result, + # Why the loop stopped -- see LoopExit. + exit_reason=best.exit_reason.value, + turns_used=best.turns_used, + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -582,25 +598,27 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Actual create args: {best.actual_create_args}. " f"Actual execute result: {best.actual_execute_result}." ) - exc = KdaSkillAssertionError(message) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + raise_agentic_failure( + KdaSkillAssertionError, + message, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, + ) + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 464cf592e..f662b205b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -21,6 +21,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -44,7 +45,7 @@ shift_and_index_events, timeline_detail, ) -from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings +from gooddata_eval.core.timing import PhaseTimings, log_timer, run_latency_s, sum_timings try: from openai import OpenAI as _OpenAI @@ -630,9 +631,19 @@ def _run_detail(run: MetricRunResult) -> dict: **timeline_detail(run.tool_call_events, run.reasoning_step_events), } - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + metric_created=best.metric_created, + maql_correct=best.maql_correct, + expected_maql_candidates=[c.get("maql", "") for c in expected_outputs_list], + actual_maql=best.actual_maql, + # Why the loop stopped. metric_created=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + turns_used=best.turns_used, + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -644,32 +655,32 @@ def _run_detail(run: MetricRunResult) -> dict: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) - exc = MetricSkillAssertionError( + raise_agentic_failure( + MetricSkillAssertionError, f"Metric skill assertion failed. {gate_note} " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " - f"Actual MAQL: {best.actual_maql}." + f"Actual MAQL: {best.actual_maql}.", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.timings = item_timings - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = run_latencies_s - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - timings=item_timings, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=run_latency_s(best.timings), + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 90dc591cf..e58a08dd3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -15,6 +15,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -114,7 +115,8 @@ class SearchResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time -- mirrors the single-shot path's best_run_latency_s + # (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 @@ -326,7 +328,13 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_effective = len(summary.run_results) best = summary.best - detail = _run_detail(best) + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + tool_selected=best.tool_selected, + tool_correct=best.tool_correct, + tool_call_names=best.tool_call_names, + ) # Same predicate pass@K and runs_passed are taken over, so an item's failed_runs and its # counts cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -337,29 +345,29 @@ def _write_scores(ctx: RunTraceContext) -> None: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = gate_failure_note(gate, runs_passed, runs_effective) - exc = SearchToolAssertionError( + raise_agentic_failure( + SearchToolAssertionError, f"Search tool assertion failed. {gate_note} " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " - f"Tool calls made: {best.tool_call_names}" + f"Tool calls made: {best.tool_call_names}", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + best_run_latency_s=best.run_latency_s, + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 6b1523ae2..b06821452 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -20,6 +20,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -77,7 +78,8 @@ class RunResult: # Why the simulated-user loop stopped. visualization_created=False alone cannot separate # a refusal from a run that hit max_iterations while still on track -- see LoopExit. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED - # This run's own wall time, from just before its first chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 # Everything the final turn said, which the answer patterns are matched against. answer_text: str = "" @@ -208,6 +210,7 @@ def _execute_single_run( one_turn = not options.expects_visualization or options.visualization_optional if one_turn: max_iterations = min(max_iterations, 1) + run_started = time.monotonic() total_turns = 0 total_steps = 0 all_tool_call_events: list[ToolCallEvent] = [] @@ -643,9 +646,16 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.eval_result - # max_iterations is the same for every run, so it stays at the item level rather than - # being repeated into each failing run's detail. - detail = {**_run_detail(best), "max_iterations": max_iterations} + detail = agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + **evaluation_result_detail(ev), + # Why the loop stopped -- see LoopExit. total_turns is already the turn count for + # this run, so it doubles as turns_used. + exit_reason=best.exit_reason.value, + turns_used=int(best.total_turns), + max_iterations=max_iterations, + ) # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. Every failing run keeps its own # expected/actual check breakdown -- for this kind that is the whole diagnosis, and @@ -667,7 +677,8 @@ def _write_scores(ctx: RunTraceContext) -> None: else {"expects_visualization": False} ) actual_dump = best.actual_output.model_dump(exclude_none=True) if best.actual_output else None - exc = VisualizationAssertionError( + raise_agentic_failure( + VisualizationAssertionError, "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" "Agentic Visualization Assertion Failed! (Critical Mode)\n" f"{gate_note}\n" @@ -699,26 +710,25 @@ def _write_scores(ctx: RunTraceContext) -> None: f" Executed : {ev.executed}{' (required)' if ev.requires_execution else ''}\n" f"{_options_lines(ev, best.answer_text)}" f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n" - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=runs_effective, + failed_runs=failed_runs, + run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = runs_effective - exc.failed_runs = failed_runs - exc.run_latencies_s = run_latencies_s - raise exc - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=runs_effective, + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, - # Also on the success path: pass@K clears the gate with one passing run, so a - # 1/3 item reports success while two of its runs failed for reasons worth reading. + runs_passed=runs_passed, + runs_effective=runs_effective, failed_runs=failed_runs, run_latencies_s=run_latencies_s, + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index baaeb201d..de124ceed 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -35,6 +35,7 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -53,9 +54,9 @@ AgenticAssertionError, AgenticEvalOutcome, ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, - build_latency_breakdown, shift_and_index_events, ) @@ -212,8 +213,14 @@ class WhatIfRunResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) - # This run's own wall time, from just before its chat call to the end of its scoring. + # This run's own wall time, across all its turns -- mirrors the single-shot + # path's best_run_latency_s (see core/runner.py's _run_one_item). Distinct from + # turn_wall_clock_sec above, which is only the scenario-execution turn. run_latency_s: float = 0.0 + # Why the simulated-user loop stopped -- see LoopExit. `triggered=False` alone cannot + # separate a refusal from a run that hit max_iterations while still on track (matches + # kda_skill.py/alert_skill.py). + exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @dataclass @@ -328,6 +335,7 @@ def run_agentic_what_if( ) def _run_once(conv_id: str) -> WhatIfRunResult: + run_started = time.monotonic() create_args: dict | None = None execute_result: dict | None = None turn_wall_clock_sec: float | None = None @@ -354,6 +362,10 @@ def _accumulate(result: ChatResult) -> None: all_reasoning_step_events.extend(result.reasoning_step_events or []) run_started = time.monotonic() + # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that + # simply runs out of range() is labelled correctly with no trailing else. + exit_reason = LoopExit.BUDGET_EXHAUSTED + for iteration in range(max_iterations): try: chat_result = client.send_message(conv_id, current_question) @@ -366,6 +378,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) create_args, execute_result = _extract_what_if_calls(all_tool_call_events) turn_completed = False + exit_reason = LoopExit.CHAT_ERROR break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -379,8 +392,10 @@ def _accumulate(result: ChatResult) -> None: if execute_result is not None: # The turn that ran the scenario, not an earlier disambiguation turn. turn_wall_clock_sec = chat_result.turn_wall_clock_sec + exit_reason = LoopExit.SUCCESS break if not response_text: + exit_reason = LoopExit.AGENT_SILENT break if iteration >= max_iterations - 1: break @@ -389,6 +404,7 @@ def _accumulate(result: ChatResult) -> None: disambiguated = True except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run _log.warning("Simulated what-if user reply failed for conversation %s: %s", conv_id, exc) + exit_reason = LoopExit.SIMULATED_USER_FAILED break return WhatIfRunResult( @@ -396,6 +412,7 @@ def _accumulate(result: ChatResult) -> None: evaluation=_evaluate_run(create_args, execute_result, expected_output, turn_completed, disambiguated), actual_create_args=create_args, actual_execute_result=execute_result, + exit_reason=exit_reason, turn_wall_clock_sec=turn_wall_clock_sec, reasoning_steps=reasoning_steps, response_id=response_id, @@ -453,26 +470,30 @@ class WhatIfAssertionError(AgenticAssertionError): def _detail(best: WhatIfRunResult) -> dict[str, Any]: ev = best.evaluation adjustments = _adjustments(best.actual_create_args) - return { - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "metric_correct": ev.metric_correct, - "maql_correct": ev.maql_correct, - "scenario_count_correct": ev.scenario_count_correct, - "baseline_correct": ev.baseline_correct, + return agentic_detail( + best.tool_call_events, + best.reasoning_step_events, + triggered=ev.triggered, + executed=ev.executed, + success=ev.success, + turn_completed=ev.turn_completed, + metric_correct=ev.metric_correct, + maql_correct=ev.maql_correct, + scenario_count_correct=ev.scenario_count_correct, + baseline_correct=ev.baseline_correct, # Which content checks the fixture pinned -- without it a run that verified nothing # reads the same as one where everything matched. - "asserted": ev.asserted, - "disambiguated": ev.disambiguated, - "actual_adjustments": adjustments, - "actual_scenario_labels": [ + asserted=ev.asserted, + disambiguated=ev.disambiguated, + actual_adjustments=adjustments, + actual_scenario_labels=[ s.get("label") for s in ((best.actual_create_args or {}).get("scenarios") or []) if isinstance(s, dict) ], - "actual_execute_result": best.actual_execute_result, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + actual_execute_result=best.actual_execute_result, + # Why the loop stopped. triggered=False alone cannot tell a refusal from a run + # that hit max_iterations while still on track -- see LoopExit. + exit_reason=best.exit_reason.value, + ) def evaluate_agentic_what_if( @@ -613,24 +634,28 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Actual adjustments: {detail['actual_adjustments']}. " f"Actual execute result: {best.actual_execute_result}." ) - exc = WhatIfAssertionError(message) - exc.reasoning_steps = best.reasoning_steps - exc.conversation_id = best.conversation_id - exc.response_id = best.response_id - exc.detail = detail - exc.runs_passed = runs_passed - exc.runs_effective = len(summary.run_results) - exc.failed_runs = failed_runs - exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] - raise exc - - return AgenticEvalOutcome( - runs_passed=runs_passed, - runs_effective=len(summary.run_results), + raise_agentic_failure( + WhatIfAssertionError, + message, + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, + ) + + return agentic_success( reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], + best_run_latency_s=best.run_latency_s, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py index c32cb3f40..6bb4b9e62 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py @@ -62,6 +62,7 @@ class JudgeResponseError(RuntimeError): runs_effective: int failed_runs: list[dict] run_latencies_s: list[float] + best_run_latency_s: float | None def _message_content(response: Any) -> str | None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py index ccf84491c..645625697 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py @@ -45,6 +45,7 @@ "agentic_kda_skill", "agentic_anomaly_detection", "agentic_what_if", + "agentic_forecasting", } ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 85f92e74c..435ac811a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -434,6 +434,7 @@ class AgenticAssertionError(AssertionError): runs_effective: int failed_runs: list[dict] run_latencies_s: list[float] + best_run_latency_s: float | None class AgenticEvalOutcome(BaseModel): @@ -470,6 +471,12 @@ class AgenticEvalOutcome(BaseModel): # needs one per run. Empty means the kind recorded none, which the report reads as "no # sample", not as zero-second runs. run_latencies_s: list[float] = Field(default_factory=list) + # The rank-selected run's own wall time -- mirrors the single-shot path's + # ItemReport.best_run_latency_s (see core/runner.py's _run_one_item), which the + # agentic path never populated because its K runs happen inside the evaluator, + # not in a loop the runner itself can time. None for a kind that has not yet been + # wired to measure it (see AgenticAssertionError.best_run_latency_s, same default). + best_run_latency_s: float | None = None class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/timing.py b/packages/gooddata-eval/src/gooddata_eval/core/timing.py index b2d203d7b..2c7b9992f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/timing.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/timing.py @@ -182,3 +182,13 @@ def note_trace_latency(seconds: float | None) -> None: sink = _trace_latencies.get() if sink is not None and seconds is not None: sink.append(seconds) + + +def run_latency_s(timings: PhaseTimings) -> float: + """One run's own wall time, for a single run's ``PhaseTimings``. + + Excludes ``langfuse_s`` (deliberately off the critical path -- see + ``PhaseTimings.langfuse_s``) so this stays comparable to the single-shot path's + ``best_run_latency_s``, which times only the agent call and its grading. + """ + return timings.agent_s + timings.judge_s + timings.simulated_user_s diff --git a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py index d7fef2965..5c1e25d8b 100644 --- a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py +++ b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py @@ -16,7 +16,7 @@ evaluate_agentic_anomaly_detection, run_agentic_anomaly_detection, ) -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.anomaly_detection" @@ -254,18 +254,22 @@ def test_a_single_turn_detection_passes(): summary = _run([_chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is False + assert summary.best.exit_reason is LoopExit.SUCCESS def test_a_clarifying_question_is_answered_and_the_run_continues(): summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is True + assert summary.best.exit_reason is LoopExit.SUCCESS def test_the_loop_stops_at_max_iterations_without_a_detection(): + # executed=False alone cannot tell this apart from a refusal -- exit_reason can. summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) assert summary.pass_at_k is False assert summary.best.evaluation.executed is False + assert summary.best.exit_reason is LoopExit.BUDGET_EXHAUSTED def test_an_empty_response_ends_the_run_immediately(): @@ -276,17 +280,19 @@ def test_an_empty_response_ends_the_run_immediately(): patch(f"{_MODULE}.ChatClient", return_value=client), patch(f"{_MODULE}.generate_simulated_anomaly_response") as sim, ): - run_agentic_anomaly_detection( + summary = run_agentic_anomaly_detection( host="http://h", token="tok", workspace_id="ws1", question="q", expected_output=_EXPECTED ) assert client.send_message.call_count == 1 sim.assert_not_called() + assert summary.best.exit_reason is LoopExit.AGENT_SILENT def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): summary = _run([_chat(_pair()), RuntimeError("boom")], k=2) assert len(summary.run_results) == 2 assert summary.pass_at_k is True + assert summary.run_results[1].exit_reason is LoopExit.CHAT_ERROR def test_k_must_be_at_least_one(): @@ -318,6 +324,8 @@ def test_detail_reports_the_series_analysed_and_the_flag_count(): assert outcome.detail["anomaly_point_count"] == 0 assert outcome.detail["asserted"] == ["metric", "granularity"] assert "latency_breakdown" in outcome.detail + assert "tool_calls" in outcome.detail + assert outcome.detail["exit_reason"] == "success" assert outcome.runs_passed == 1 assert len(outcome.run_latencies_s) == outcome.runs_effective == 1 assert all(latency >= 0 for latency in outcome.run_latencies_s) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index c29bfbbab..2edfc58e8 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -17,7 +17,8 @@ evaluate_dashboard_response, run_agentic_dashboard_skill, ) -from gooddata_eval.core.models import ChatResult, ToolCallEvent +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent # Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of # `agent_dashboard_skill` against ecommerce_demo. @@ -2343,6 +2344,8 @@ def test_the_loop_is_capped(self): assert client.send_message.call_count == 2 assert not summary.pass_at_k assert not summary.best.evaluation.drafted + # drafted=False alone cannot tell this apart from a refusal -- exit_reason can. + assert summary.best.exit_reason is LoopExit.BUDGET_EXHAUSTED def test_an_empty_turn_stops_the_loop(self): client = MagicMock() @@ -2350,6 +2353,7 @@ def test_an_empty_turn_stops_the_loop(self): summary = _run_with(client, _DC05_EXPECTED, max_iterations=5) assert client.send_message.call_count == 1 assert not summary.pass_at_k + assert summary.best.exit_reason is LoopExit.AGENT_SILENT def test_a_caller_supplied_conversation_is_not_deleted(self): client = MagicMock() @@ -2358,6 +2362,54 @@ def test_a_caller_supplied_conversation_is_not_deleted(self): client.create_conversation.assert_not_called() client.delete_conversation.assert_not_called() + def test_a_chat_error_is_recorded_rather_than_left_to_escape(self) -> None: + """It used to escape run_agentic_dashboard_skill, taking every completed K-run with + it: no DashboardRunResult, no exit_reason, just a bare error on the item. Recorded + the way metric_skill.py and conversation.py record it, so an infrastructure fault + stays distinguishable from the agent failing to draft.""" + client = MagicMock() + client.send_message.side_effect = ChatError("gen-ai fell over") + + summary = _run_with(client, _DC05_EXPECTED) + + assert summary.run_results[0].exit_reason is LoopExit.CHAT_ERROR + assert summary.run_results[0].evaluation.strict_pass is False + assert summary.pass_at_k is False + + def test_a_chat_error_keeps_the_partial_turn_s_work(self) -> None: + """A bare catch would discard ChatError.partial_result. What the stream delivered + before it died is still this turn's work -- the steps were taken and the tools ran.""" + partial = _chat_result( + tool_calls=[{"functionName": "search_objects", "functionArguments": "{}", "callTs": 1, "durationMs": 10}], + text="Here is the draft.", + ) + partial.reasoning_steps = ["looked for the charts"] + partial.reasoning_step_count = 1 + client = MagicMock() + client.send_message.side_effect = ChatError("stream died mid-answer", partial_result=partial) + + summary = _run_with(client, _DC05_EXPECTED) + run = summary.run_results[0] + + assert run.exit_reason is LoopExit.CHAT_ERROR + assert run.reasoning_steps == ["looked for the charts"] + assert run.total_steps == 1 + assert [tc.function_name for tc in run.tool_call_events] == ["search_objects"] + + def test_a_chat_error_on_a_later_turn_keeps_the_earlier_ones(self) -> None: + """The whole point of recording it: turns already completed survive the fault.""" + client = MagicMock() + client.send_message.side_effect = [ + _chat_result(text="Which charts did you have in mind?"), + ChatError("gen-ai fell over"), + ] + + summary = _run_with(client, _DC05_EXPECTED) + run = summary.run_results[0] + + assert run.exit_reason is LoopExit.CHAT_ERROR + assert run.total_turns == 2 + class TestEvaluateEntryPoint: def test_a_failing_run_raises_with_the_failures_attached(self): @@ -2421,6 +2473,8 @@ def test_a_passing_run_returns_the_outcome(self): assert outcome.detail["charts_matched"] is True assert len(outcome.run_latencies_s) == outcome.runs_effective assert all(latency >= 0 for latency in outcome.run_latencies_s) + assert outcome.detail["exit_reason"] == "success" + assert "tool_calls" in outcome.detail def _as_events(raw: list[dict]) -> list[ToolCallEvent]: diff --git a/packages/gooddata-eval/tests/test_agentic_forecasting.py b/packages/gooddata-eval/tests/test_agentic_forecasting.py new file mode 100644 index 000000000..ccecfa93f --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_forecasting.py @@ -0,0 +1,528 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic.forecasting import ( + ForecastingAssertionError, + _build_clarification_prompt, + _evaluate_run, + _extract_forecast_calls, + _metric_uris, + evaluate_agentic_forecasting, + run_agentic_forecasting, +) +from gooddata_eval.core.models import ChatResult + +_MODULE = "gooddata_eval.core.agentic.forecasting" + +_EXPECTED = {"metric": "metric/spend", "forecast_period": 3} + + +def _viz_args(*, period=3, enabled=True, metric="metric/spend", confidence=0.95, seasonal=False) -> dict: + return { + "visualization": { + "type": "line_chart", + "title": "Spend forecast", + "query": { + "fields": { + "m_spend": {"using": metric}, + "d_month": {"using": "label/process_date.month"}, + }, + "filter_by": {}, + }, + "metrics": ["m_spend"], + "view_by": ["d_month"], + "config": { + "forecast_enabled": enabled, + "forecast_period": period, + "forecast_confidence": confidence, + "forecast_seasonal": seasonal, + }, + } + } + + +def _tc(name: str, args: dict | None = None, result: str | None = None): + tc = MagicMock() + tc.function_name = name + tc.call_ts = None + tc.result_ts = None + tc.index = None + tc.parsed_arguments = lambda a=args: a or {} + tc.result = result + tc.parsed_result = lambda r=result: __import__("json").loads(r) if r else None + return tc + + +_OK_FORECAST = '{"success": true, "data": {"points": [{"value": 1}], "truncated": false}, "error": null}' + + +def _chat(tool_calls, text="Here is the forecast.", ended=True) -> ChatResult: + """A real ChatResult, so render_answer_text sees every field it reads.""" + result = ChatResult.model_validate({"textResponse": text, "responseId": "resp-1"}) + result.stream_ended = ended + result.turn_wall_clock_sec = 4.2 + # Assigned after validation: these are hand-built tool-call doubles, not payload. + result.tool_call_events = tool_calls + return result + + +# ── extraction ────────────────────────────────────────────────────────────── + + +def test_extract_pairs_the_execute_with_the_visualization_it_followed(): + """A new chart clears the previous forecast: picking the last of each independently + would pair a fresh visualization with a stale result.""" + calls = [ + _tc("create_adhoc_visualization", _viz_args(period=1)), + _tc("execute_forecast", {}, _OK_FORECAST), + _tc("create_adhoc_visualization", _viz_args(period=3)), + ] + viz, result = _extract_forecast_calls(calls) + + assert viz["config"]["forecast_period"] == 3 + assert result is None + + +def test_extract_unwraps_the_visualization_argument(): + viz, _ = _extract_forecast_calls([_tc("create_adhoc_visualization", _viz_args())]) + assert viz["type"] == "line_chart" + + +def test_metric_uris_resolves_aliases_and_ignores_dimensions(): + viz, _ = _extract_forecast_calls([_tc("create_adhoc_visualization", _viz_args())]) + assert _metric_uris(viz) == {"metric/spend"} + + +def test_metric_uris_handles_a_bare_uri_field(): + """Tool-call arguments are raw JSON, where a field may be a plain string rather than + the object core.scoring's parsed model always has.""" + viz = {"query": {"fields": {"m": "metric/spend"}}, "metrics": ["m"]} + assert _metric_uris(viz) == {"metric/spend"} + + +# ── scoring ───────────────────────────────────────────────────────────────── + + +def _evaluate(viz=None, result=None, expected=None, turn_completed=True): + return _evaluate_run(viz, result, expected if expected is not None else _EXPECTED, turn_completed) + + +def _viz(**kw): + return _viz_args(**kw)["visualization"] + + +def test_a_correct_forecast_passes_every_check(): + ev = _evaluate(_viz(), {"success": True}) + assert ev.strict_pass is True + assert ev.asserted == ["forecast_period", "metric"] + + +def test_the_wrong_horizon_fails_on_period_alone(): + """The number the agent chose is in the tool call, so 'next 3 months' is checkable + exactly -- no judge, no paraphrase tolerance.""" + ev = _evaluate(_viz(period=6), {"success": True}) + assert ev.period_correct is False + assert ev.metric_correct is True + assert ev.strict_pass is False + + +def test_forecasting_the_wrong_measure_fails_on_metric_alone(): + ev = _evaluate(_viz(metric="metric/orders"), {"success": True}) + assert ev.metric_correct is False + assert ev.period_correct is True + + +def test_forecast_enabled_must_be_explicitly_true(): + """The tool refuses on forecast_enabled=false, and treats an unset value the same way, + so only an explicit true counts.""" + assert _evaluate(_viz(enabled=None), {"success": True}).forecast_enabled is False + assert _evaluate(_viz(enabled=False), {"success": True}).forecast_enabled is False + assert _evaluate(_viz(enabled=True), {"success": True}).forecast_enabled is True + + +def test_an_unstated_expectation_neither_fails_nor_silently_passes(): + """A fixture that pins nothing must not fail, but the report has to say nothing was + checked -- otherwise it reads identically to a run where everything matched.""" + ev = _evaluate(_viz(period=99, metric="metric/anything"), {"success": True}, expected={}) + assert ev.period_correct is True + assert ev.metric_correct is True + assert ev.asserted == [] + + +def test_a_failed_execution_is_not_a_success(): + ev = _evaluate(_viz(), {"success": False, "error": "no forecast for you"}) + assert ev.executed is True + assert ev.success is False + assert ev.strict_pass is False + + +def test_never_reaching_execute_is_not_triggered_by_a_chart_alone(): + ev = _evaluate(_viz(), None) + assert ev.executed is False + assert ev.success is False + + +def test_an_integer_valued_float_period_still_matches(): + """The AAC config is JSON, so 3 may arrive as 3.0.""" + assert _evaluate(_viz(period=3.0), {"success": True}).period_correct is True + + +# ── run loop ──────────────────────────────────────────────────────────────── + + +def _run(side_effect, *, k=1, expected=None, max_iterations=4): + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, k + 2)] + client.send_message.side_effect = side_effect + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response", return_value="use spend, 3 months"), + ): + return run_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=expected if expected is not None else _EXPECTED, + k=k, + max_iterations=max_iterations, + ) + + +def test_a_single_turn_forecast_passes(): + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat(calls)]) + assert summary.pass_at_k is True + assert summary.best.evaluation.disambiguated is False + + +def test_a_clarifying_question_is_answered_and_the_run_continues(): + """Observed live: the agent asks which Spend metric before building anything.""" + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(calls)]) + assert summary.pass_at_k is True + assert summary.best.evaluation.disambiguated is True + + +def test_the_loop_stops_at_max_iterations_without_a_forecast(): + summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) + assert summary.pass_at_k is False + assert summary.best.evaluation.executed is False + + +def test_an_empty_response_ends_the_run_immediately(): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _chat([], text="") + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response") as sim, + ): + run_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="q", + expected_output=_EXPECTED, + ) + assert client.send_message.call_count == 1 + sim.assert_not_called() + + +def test_a_chat_error_ends_the_run_and_keeps_what_the_partial_carried(): + error = RuntimeError("stream died") + error.partial_result = _chat( + [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + summary = _run([error]) + # The forecast landed before the stream broke, so it is still scored -- but the turn + # did not complete, which strict_pass requires. + assert summary.best.evaluation.executed is True + assert summary.best.evaluation.turn_completed is False + + +def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): + calls = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + summary = _run([_chat(calls), RuntimeError("boom")], k=2) + assert len(summary.run_results) == 2 + assert summary.pass_at_k is True + + +def test_k_must_be_at_least_one(): + with pytest.raises(ValueError, match="k must be >= 1"): + run_agentic_forecasting(host="h", token="t", workspace_id="w", question="q", expected_output={}, k=0) + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate_item(calls): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _chat(calls) + with patch(f"{_MODULE}.ChatClient", return_value=client): + return evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=_EXPECTED, + ) + + +def test_detail_reports_the_config_the_agent_chose(): + outcome = _evaluate_item( + [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + assert outcome.detail["actual_forecast_config"]["forecast_period"] == 3 + assert outcome.detail["actual_metrics"] == ["metric/spend"] + assert outcome.detail["asserted"] == ["forecast_period", "metric"] + assert "latency_breakdown" in outcome.detail + assert outcome.runs_passed == 1 + + +def test_a_wrong_horizon_raises_naming_what_the_agent_actually_did(): + with pytest.raises(ForecastingAssertionError) as exc_info: + _evaluate_item( + [_tc("create_adhoc_visualization", _viz_args(period=12)), _tc("execute_forecast", {}, _OK_FORECAST)] + ) + + error = exc_info.value + assert "period_correct=False" in str(error) + assert error.detail["actual_forecast_config"]["forecast_period"] == 12 + assert error.runs_passed == 0 + assert error.conversation_id == "conv-1" + + +# ── confidence and seasonality ────────────────────────────────────────────── + + +def test_confidence_is_checked_when_pinned(): + expected = {"forecast_confidence": 0.99} + assert _evaluate(_viz(confidence=0.99), {"success": True}, expected=expected).confidence_correct is True + assert _evaluate(_viz(confidence=0.95), {"success": True}, expected=expected).confidence_correct is False + + +def test_seasonality_is_checked_when_pinned(): + expected = {"forecast_seasonal": True} + assert _evaluate(_viz(seasonal=True), {"success": True}, expected=expected).seasonal_correct is True + assert _evaluate(_viz(seasonal=False), {"success": True}, expected=expected).seasonal_correct is False + + +def test_an_absent_seasonal_counts_as_the_tools_default(): + """The tool defaults seasonal to false, so an agent that leaves it alone has asked for + a non-seasonal forecast -- failing that would penalise correct behaviour.""" + viz = _viz() + del viz["config"]["forecast_seasonal"] + assert _evaluate(viz, {"success": True}, expected={"forecast_seasonal": False}).seasonal_correct is True + + +def test_confidence_and_seasonality_are_unasserted_by_default(): + ev = _evaluate(_viz(confidence=0.5, seasonal=True), {"success": True}) + assert ev.confidence_correct is True + assert ev.seasonal_correct is True + assert "forecast_confidence" not in ev.asserted + assert "forecast_seasonal" not in ev.asserted + + +# ── cross-turn extraction ─────────────────────────────────────────────────── + + +def test_a_chart_built_on_an_earlier_turn_is_still_the_one_scored(): + """The agent may build the chart on one turn and forecast on the next -- reading only + the current turn's calls would drop the visualization the forecast actually ran on and + fail a correct run for an empty config.""" + summary = _run( + [ + _chat([_tc("create_adhoc_visualization", _viz_args())], text="Building the chart, one moment."), + _chat([_tc("execute_forecast", {"visualization_ref": "viz_1"}, _OK_FORECAST)]), + ] + ) + + assert summary.best.evaluation.executed is True + assert summary.best.evaluation.period_correct is True + assert summary.best.evaluation.metric_correct is True + assert summary.pass_at_k is True + + +# ── Langfuse scoring ──────────────────────────────────────────────────────── + + +class _FakeCtx: + """Records what the deferred Langfuse block writes, without a Langfuse.""" + + def __init__(self): + self.scores: dict[str, float] = {} + # Not named `quality`: the method below would overwrite itself on first call. + self.quality_call: dict = {} + # stamp_gate_metadata writes the gate and K here, the way every other kind does. + self.run_metadata: dict = {} + + def trace(self, _conversation_id): + return None + + @contextmanager + def observe(self, _trace, _run_idx): + yield "trace-id" + + def score(self, _tid, *, name, value, data_type): + self.scores[name] = value + + def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): + self.quality_call = {"strict_checks": strict_checks, "latency_sec": latency_sec, "cost_usd": cost_usd} + + +def _scored(expected_output, calls=None): + """Run one item with Langfuse on, then execute the deferred block against a fake ctx.""" + client = MagicMock() + client.create_conversation.return_value = "conv-1" + default = [_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)] + client.send_message.return_value = _chat(calls if calls is not None else default) + captured = {} + + def _capture(_link, _identity, **kwargs): + captured.update(kwargs) + + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.submit_trace_scoring", side_effect=_capture), + ): + try: + evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=expected_output, + langfuse=MagicMock(), + dataset_item_id="ds-1", + ) + except ForecastingAssertionError: + pass # scores are written before the pass@K raise, which is the point + + ctx = _FakeCtx() + captured["write_scores"](ctx) + return ctx + + +def test_only_the_checks_the_fixture_pinned_are_scored(): + """An unasserted check is True internally so it cannot fail a run. Publishing that as a + BOOLEAN 1 would claim the evaluator verified something it never looked at.""" + ctx = _scored({"forecast_period": 3}) # metric, confidence, seasonality unpinned + + assert ctx.scores["forecast_period_correct"] == 1.0 + for absent in ("forecast_metric_correct", "forecast_confidence_correct", "forecast_seasonal_correct"): + assert absent not in ctx.scores + # The process checks are unconditional -- they are always actually evaluated. + assert set(ctx.scores) >= {"forecast_triggered", "forecast_executed", "forecast_enabled"} + + +def test_every_pinned_check_is_scored(): + ctx = _scored( + {"metric": "metric/spend", "forecast_period": 3, "forecast_confidence": 0.95, "forecast_seasonal": False} + ) + for name in ( + "forecast_period_correct", + "forecast_metric_correct", + "forecast_confidence_correct", + "forecast_seasonal_correct", + ): + assert ctx.scores[name] == 1.0 + + +def test_cost_is_reported_even_when_the_tool_was_never_reached(): + """A run that answered without forecasting still spent tokens; gating cost on + ev.triggered hid that and understated what the item cost.""" + ctx = _scored(_EXPECTED, calls=[]) + assert "cost_usd" in ctx.quality_call + + +# ── the gate decides the verdict, not pass@K alone ────────────────────────── +# +# Every other multi-run kind takes `gate` and asks `gate_passed(...)`. Forecasting computed +# pass^K and then ignored it, so `--gate power` on a flaky item reported a pass on the +# strength of one good run out of K -- the precise case the gate exists to catch. + + +def _two_runs(gate=None): + """One passing run then one failing one, which pass@K clears and pass^K must not.""" + passing = _chat([_tc("create_adhoc_visualization", _viz_args()), _tc("execute_forecast", {}, _OK_FORECAST)]) + failing = _chat([]) + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2"] + client.send_message.side_effect = [passing, failing, failing, failing, failing, failing] + kwargs = {"gate": gate} if gate is not None else {} + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_forecast_response", return_value="use spend, 3 months"), + ): + return evaluate_agentic_forecasting( + host="http://h", + token="tok", + workspace_id="ws1", + question="Forecast spend for 3 months", + expected_output=_EXPECTED, + k=2, + **kwargs, + ) + + +def test_power_gate_fails_an_item_where_only_some_runs_passed(): + with pytest.raises(ForecastingAssertionError) as excinfo: + _two_runs(gate="power") + assert "pass^2" in str(excinfo.value) + assert excinfo.value.runs_passed == 1 + assert excinfo.value.runs_effective == 2 + + +def test_any_gate_still_passes_the_same_item(): + """The default is unchanged: one passing run out of K clears pass@K.""" + outcome = _two_runs(gate="any") + assert outcome.runs_passed == 1 + assert outcome.runs_effective == 2 + + +def test_the_default_gate_is_pass_at_k(): + outcome = _two_runs() + assert outcome.runs_passed == 1 + + +# ── the simulated user's reply ────────────────────────────────────────────── + + +def test_the_reply_carries_every_hint_the_fixture_pins(): + """Anything the fixture pins is also scored, so withholding it would let the harness fail + a run on a value it refused to supply: the agent asks which confidence level to use, the + simulated user guesses, and `confidence_correct` reports the guess as the agent's error.""" + prompt = _build_clarification_prompt( + "Which confidence level, and should I model seasonality?", + { + "metric": "metric/spend", + "forecast_period": 3, + "granularity": "MONTH", + "forecast_confidence": 0.8, + "forecast_seasonal": True, + }, + ) + assert "metric/spend" in prompt + assert "3 periods ahead" in prompt + assert "MONTH" in prompt + assert "confidence level is 0.8" in prompt + assert "seasonality should be modelled" in prompt + + +def test_a_pinned_false_seasonal_is_an_answer_not_an_absence(): + """`is not None`, not a truth test. `forecast_seasonal: false` is scored -- the tool + defaults it to false -- so the simulated user has to be able to say so.""" + prompt = _build_clarification_prompt("Seasonal?", {**_EXPECTED, "forecast_seasonal": False}) + assert "seasonality should not be modelled" in prompt + + +def test_an_unpinned_hint_is_dropped_rather_than_asserted_as_none(): + prompt = _build_clarification_prompt("Which measure?", {"metric": "metric/spend"}) + assert "confidence" not in prompt + assert "seasonality" not in prompt + assert "None" not in prompt diff --git a/packages/gooddata-eval/tests/test_agentic_general_question.py b/packages/gooddata-eval/tests/test_agentic_general_question.py index e670fb352..c95caa54a 100644 --- a/packages/gooddata-eval/tests/test_agentic_general_question.py +++ b/packages/gooddata-eval/tests/test_agentic_general_question.py @@ -257,6 +257,7 @@ def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): "judge_reasoning": "Correct answer", "actual_output": "42", "latency_breakdown": [], + "tool_calls": [], } # One wall-clock sample per run, passing runs included -- the latency distribution is over # these, and a kind that forgot to report them would silently drop out of it. @@ -288,6 +289,7 @@ def test_evaluate_agentic_general_question_attaches_reasoning_steps_to_exception "judge_reasoning": "Wrong answer", "actual_output": "I don't know", "latency_breakdown": [], + "tool_calls": [], } assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1 @@ -407,6 +409,9 @@ def test_evaluate_general_question_aggregates_timings_across_k_runs(): assert outcome.timings.judge_s == 1.5 # 1.0 + 0.5 # Each run's own span, agent and judge both inside it -- not their sum across runs. assert outcome.run_latencies_s == [3.0, 3.5] + # best_run_latency_s is the rank-selected run's own time (agent_s=2.0 + judge_s=1.0 + # from the first, tied-best run), not the item total above (both runs summed). + assert outcome.best_run_latency_s == 3.0 def test_evaluate_general_question_attaches_timings_to_the_failure_exception(): @@ -430,6 +435,7 @@ def test_evaluate_general_question_attaches_timings_to_the_failure_exception(): assert exc_info.value.timings.agent_s == 4.0 assert exc_info.value.timings.judge_s == 2.0 + assert exc_info.value.best_run_latency_s == 6.0 def test_trace_link_window_is_pinned_at_submit_time_not_when_the_task_runs(): diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 07f8bcb64..6a1690ebb 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -1134,6 +1134,7 @@ def test_evaluate_agentic_kda_skill_returns_reasoning_steps_on_pass(): "turns_used": 1, "max_iterations": 1, "latency_breakdown": [], + "tool_calls": [], } assert len(outcome.run_latencies_s) == outcome.runs_effective == 1 assert all(latency >= 0 for latency in outcome.run_latencies_s) @@ -1174,6 +1175,7 @@ def test_evaluate_agentic_kda_skill_attaches_reasoning_steps_to_exception_on_fai "turns_used": 1, "max_iterations": 1, "latency_breakdown": [], + "tool_calls": [], } assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index a86a2e9c3..5efcbb0bb 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -855,6 +855,7 @@ def test_evaluate_metric_skill_surfaces_timings_on_the_outcome(): assert outcome.timings.agent_s == 3.0 assert outcome.timings.simulated_user_s == 0.0 + assert outcome.best_run_latency_s == 3.0 def test_no_timer_output_by_default(monkeypatch, capsys): diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 8682c9bec..b2dd25445 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -98,6 +98,7 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ("agentic_forecasting", {"forecast_period": 3}, "evaluate_agentic_forecasting"), ( "agentic_dashboard_summary", {"must_include": ["States the overall trend."]}, @@ -416,6 +417,54 @@ def test_run_agentic_items_records_phase_timings_when_the_item_fails(): assert report.items[0].judge_latency_s == 2.0 +def test_run_agentic_items_records_best_run_latency_on_success(): + # best_run_latency_s was never populated on the agentic path -- the K-run loop and + # best-of-K selection happen inside the evaluator, not in a loop the runner itself + # times (unlike _run_one_item on the single-shot path). + outcome = AgenticEvalOutcome( + conversation_id="c1", + detail={"alert_created": True}, + best_run_latency_s=4.25, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", return_value=outcome): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert report.items[0].best_run_latency_s == 4.25 + + +def test_run_agentic_items_records_best_run_latency_when_the_item_fails(): + exc = AlertSkillAssertionError("nope") + exc.detail = {"alert_created": False} + exc.best_run_latency_s = 9.5 + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert report.items[0].pass_at_k is False + assert report.items[0].best_run_latency_s == 9.5 + + +def test_an_errored_item_without_best_run_latency_keeps_the_none_default(): + # Kinds not yet wired to measure it (or any exception with no such attribute) must + # not report an invented number -- None stays None, not 0.0. + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=RuntimeError("boom")): + report = run_agentic_items([_timed_item()], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01") + + assert report.items[0].best_run_latency_s is None + + def test_a_pending_trace_link_does_not_block_the_next_item(): """The overlap claim, proved by construction rather than by a stopwatch. @@ -842,6 +891,7 @@ def test_an_errored_item_without_timings_keeps_its_zero_defaults(): ("agentic_what_if", "evaluate_agentic_what_if", {}), ("agentic_anomaly_detection", "evaluate_agentic_anomaly_detection", {}), ("agentic_document_skill", "evaluate_agentic_document_skill", {}), + ("agentic_forecasting", "evaluate_agentic_forecasting", {}), ("agentic_dashboard_summary", "evaluate_agentic_dashboard_summary", {}), ("agentic_conversation", "evaluate_agentic_conversation", {"id": "c1", "expected_skills": [], "turns": []}), ] @@ -1013,9 +1063,15 @@ def test_every_multi_run_kind_s_evaluator_builds_failed_runs(kind, expected_outp pytest.skip(f"{kind} runs its fixture once; no K to fail within") assert "build_failed_runs(" in source, f"{module.__name__} never builds per-run failure records" assert "failed_runs=failed_runs" in source, f"{module.__name__} never returns them on the success path" - # Either set directly on the raised error, or via the kind's own _attach_diagnostics - # helper -- the judge-based kinds raise from two places and set the payload in one. - attaches = "failed_runs = failed_runs" in source or "_attach_diagnostics(" in source + # Either set directly on the raised error, via the kind's own _attach_diagnostics + # helper (the judge-based kinds raise from two places and set the payload in one), or + # passed to raise_agentic_failure -- the shared tail every other kind's failure raise + # goes through, alongside its agentic_success mirror, so the keyword appears twice. + attaches = ( + "failed_runs = failed_runs" in source + or "_attach_diagnostics(" in source + or ("raise_agentic_failure(" in source and source.count("failed_runs=failed_runs") >= 2) + ) assert attaches, f"{module.__name__} never attaches them to its failure" @@ -1164,7 +1220,13 @@ def test_every_evaluator_hands_out_its_run_latencies(kind, expected_output, targ module = importlib.import_module(getattr(agentic_runner, target).__module__) source = inspect.getsource(module) assert "run_latencies_s=" in source, f"{module.__name__} never returns its run latencies on the success path" - assert ".run_latencies_s = " in source, f"{module.__name__} never attaches them to its failure" + # Either set directly on the raised error (the two judge-based kinds' all-ungraded + # path), or passed to raise_agentic_failure alongside its agentic_success mirror, so + # the keyword appears at least twice. + attaches = ".run_latencies_s = " in source or ( + "raise_agentic_failure(" in source and source.count("run_latencies_s=") >= 2 + ) + assert attaches, f"{module.__name__} never attaches them to its failure" @pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) diff --git a/packages/gooddata-eval/tests/test_agentic_search_tool.py b/packages/gooddata-eval/tests/test_agentic_search_tool.py index 916877bdb..05a1a1635 100644 --- a/packages/gooddata-eval/tests/test_agentic_search_tool.py +++ b/packages/gooddata-eval/tests/test_agentic_search_tool.py @@ -208,6 +208,7 @@ def test_evaluate_agentic_search_tool_returns_reasoning_steps_on_pass(): "tool_correct": True, "tool_call_names": ["search_objects"], "latency_breakdown": [], + "tool_calls": [], } # One wall-clock sample per run, passing runs included -- the latency distribution is over # these, and a kind that forgot to report them would silently drop out of it. @@ -248,6 +249,7 @@ def test_evaluate_agentic_search_tool_attaches_reasoning_steps_to_exception_on_f "tool_correct": False, "tool_call_names": [], "latency_breakdown": [], + "tool_calls": [], } assert len(exc_info.value.run_latencies_s) == exc_info.value.runs_effective == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 67a3cc84c..38b04ebf3 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -13,7 +13,7 @@ evaluate_agentic_what_if, run_agentic_what_if, ) -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.what_if" @@ -215,6 +215,7 @@ def test_a_single_turn_scenario_passes(): summary = _run([_chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is False + assert summary.best.exit_reason is LoopExit.SUCCESS def test_a_clarifying_question_is_answered_and_the_run_continues(): @@ -222,12 +223,15 @@ def test_a_clarifying_question_is_answered_and_the_run_continues(): summary = _run([_chat([], text="Which Spend metric did you mean?"), _chat(_pair())]) assert summary.pass_at_k is True assert summary.best.evaluation.disambiguated is True + assert summary.best.exit_reason is LoopExit.SUCCESS def test_the_loop_stops_at_max_iterations_without_a_scenario(): + # triggered=False alone cannot tell this apart from a refusal -- exit_reason can. summary = _run([_chat([], text="Still thinking")] * 4, max_iterations=4) assert summary.pass_at_k is False assert summary.best.evaluation.triggered is False + assert summary.best.exit_reason is LoopExit.BUDGET_EXHAUSTED def test_an_empty_response_ends_the_run_immediately(): @@ -238,15 +242,19 @@ def test_an_empty_response_ends_the_run_immediately(): patch(f"{_MODULE}.ChatClient", return_value=client), patch(f"{_MODULE}.generate_simulated_what_if_response") as sim, ): - run_agentic_what_if(host="http://h", token="tok", workspace_id="ws1", question="q", expected_output=_EXPECTED) + summary = run_agentic_what_if( + host="http://h", token="tok", workspace_id="ws1", question="q", expected_output=_EXPECTED + ) assert client.send_message.call_count == 1 sim.assert_not_called() + assert summary.best.exit_reason is LoopExit.AGENT_SILENT def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): summary = _run([_chat(_pair()), RuntimeError("boom")], k=2) assert len(summary.run_results) == 2 assert summary.pass_at_k is True + assert summary.run_results[1].exit_reason is LoopExit.CHAT_ERROR def test_k_must_be_at_least_one(): @@ -277,6 +285,8 @@ def test_detail_reports_the_adjustments_the_agent_asked_for(): assert outcome.detail["actual_scenario_labels"] == ["Scenario A"] assert outcome.detail["asserted"] == ["metric_id", "scenario_maql"] assert "latency_breakdown" in outcome.detail + assert "tool_calls" in outcome.detail + assert outcome.detail["exit_reason"] == "success" assert outcome.runs_passed == 1 assert len(outcome.run_latencies_s) == outcome.runs_effective == 1 assert all(latency >= 0 for latency in outcome.run_latencies_s) diff --git a/packages/gooddata-eval/tests/test_timing.py b/packages/gooddata-eval/tests/test_timing.py index b2de2e19b..527ce7be5 100644 --- a/packages/gooddata-eval/tests/test_timing.py +++ b/packages/gooddata-eval/tests/test_timing.py @@ -15,6 +15,7 @@ log_timer, note_trace_latency, percentile, + run_latency_s, sum_timings, timers_enabled, trimmed_mean, @@ -268,3 +269,10 @@ def item(base: float) -> list[float]: results = list(pool.map(item, [0.0, 100.0, 200.0, 300.0])) assert results == [[b + s for s in range(5)] for b in (0.0, 100.0, 200.0, 300.0)] + + +def test_run_latency_s_excludes_langfuse(): + # langfuse_s is deliberately off the critical path (see PhaseTimings docstring), so a + # trace-link slowdown must not inflate the run latency a worst-case table sorts on. + timings = PhaseTimings(agent_s=2.0, judge_s=1.0, simulated_user_s=0.5, langfuse_s=100.0) + assert run_latency_s(timings) == 3.5 diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index 5a230f3be..af4b70d65 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -120,6 +120,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"), ("conversation", "evaluate_agentic_conversation"), + ("forecasting", "evaluate_agentic_forecasting"), ("dashboard_summary", "evaluate_agentic_dashboard_summary"), ("anomaly_detection", "evaluate_agentic_anomaly_detection"), ("what_if", "evaluate_agentic_what_if"),