From 7b30cdb7d3b3568a5e9f868777a69e63b732d9ae Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Fri, 13 Nov 2026 16:20:55 +0700 Subject: [PATCH] feat(gooddata-eval): opt-in error outcome and inherited date filter Two opt-in changes, both off by default. QA-30213, GD_EVAL_ERROR_OUTCOME / --error-outcome: a run that ends on a backend fault (chat_error after retries) or a harness fault (simulated user failed) becomes ERROR and leaves the pass^K gate. An item with no scored run raises EvalHarnessError, which is not an AssertionError. An ERROR run's trace gets only an eval_error=1 score. Agent timeouts and incomplete turns get their own loop exits (agent_timeout, turn_incomplete) and score as FAIL. The gate reads "pass^3 (1 excluded)", and runs_scored counts the scored runs. QA-30214, GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER / --accept-context-date-filter: a date filter carried over from the dashboard context passes the filter check when the expected output omits it. A dashboard filter with `using` matches only that dataset. Always on: agentic failure messages name each failing run, failed_runs[] records gain exit_reason, outcome and failed_checks, and a timed-out turn records agent_timeout / turn_incomplete instead of chat_error. Verdicts are unchanged with the flags off. jira: QA-30213 risk: low --- packages/gooddata-eval/AGENTS.md | 3 + packages/gooddata-eval/README.md | 116 +++ .../src/gooddata_eval/__init__.py | 16 +- .../src/gooddata_eval/cli/agentic_runner.py | 22 +- .../src/gooddata_eval/cli/main.py | 40 + .../core/agentic/_failed_runs.py | 51 +- .../core/agentic/_failure_message.py | 142 +++ .../src/gooddata_eval/core/agentic/_gate.py | 22 +- .../gooddata_eval/core/agentic/_outcome.py | 230 ++++- .../core/agentic/_trace_linker.py | 5 +- .../gooddata_eval/core/agentic/alert_skill.py | 53 +- .../core/agentic/anomaly_detection.py | 49 +- .../core/agentic/conversation.py | 12 +- .../core/agentic/dashboard_skill.py | 55 +- .../gooddata_eval/core/agentic/kda_skill.py | 46 +- .../core/agentic/metric_skill.py | 63 +- .../core/agentic/visualization.py | 156 +++- .../src/gooddata_eval/core/agentic/what_if.py | 49 +- .../src/gooddata_eval/core/config.py | 23 + .../core/evaluators/visualization.py | 50 +- .../src/gooddata_eval/core/langfuse/sink.py | 21 +- .../src/gooddata_eval/core/models.py | 50 +- .../src/gooddata_eval/core/outcome.py | 197 +++++ .../gooddata_eval/core/reporting/console.py | 12 +- .../core/reporting/json_report.py | 3 + .../src/gooddata_eval/core/runner.py | 79 +- .../src/gooddata_eval/core/scoring.py | 90 +- packages/gooddata-eval/tests/conftest.py | 2 + .../tests/test_agentic_alert_skill.py | 20 +- .../tests/test_agentic_anomaly_detection.py | 13 + .../tests/test_agentic_conversation.py | 28 +- .../tests/test_agentic_dashboard_skill.py | 18 +- .../tests/test_agentic_failure_message.py | 352 ++++++++ .../tests/test_agentic_kda_skill.py | 30 +- .../tests/test_agentic_metric_skill.py | 91 +- .../tests/test_agentic_observe_experiment.py | 12 + .../tests/test_agentic_visualization.py | 34 +- .../tests/test_agentic_what_if.py | 13 + packages/gooddata-eval/tests/test_cli.py | 73 +- .../tests/test_context_date_filter.py | 400 +++++++++ .../tests/test_eval_error_scores.py | 154 ++++ .../gooddata-eval/tests/test_langfuse_sink.py | 64 ++ packages/gooddata-eval/tests/test_outcome.py | 825 ++++++++++++++++++ .../gooddata-eval/tests/test_reporting.py | 26 + 44 files changed, 3672 insertions(+), 138 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/outcome.py create mode 100644 packages/gooddata-eval/tests/test_agentic_failure_message.py create mode 100644 packages/gooddata-eval/tests/test_context_date_filter.py create mode 100644 packages/gooddata-eval/tests/test_eval_error_scores.py create mode 100644 packages/gooddata-eval/tests/test_outcome.py diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 183cb387c..1df70607d 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -34,6 +34,7 @@ experiment. The newest and most actively developed package in the repo. | `core/reporting/` | console and JSON output rendering | | `core/scoring.py`, `core/runner.py` | scoring and orchestration | | `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends | +| `core/outcome.py` | `Outcome` (PASS / FAIL / ERROR), the one `LoopExit` → `Outcome` mapping, and the gate over scored runs behind `GD_EVAL_ERROR_OUTCOME`; the agentic kinds reach it through `core/agentic/_outcome.py`'s `resolve_runs` | **Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai` @@ -161,6 +162,8 @@ Run this before guessing a `--model` string. | `GOODDATA_EVAL_TRACE_LABELS` | `key=value,...` labels stamped on every observation of a chat conversation via W3C baggage; `model_version` also sets the observation `version`, so set it only for a one-model run: every `--model` in one process gets the same labels. Keys must match `[A-Za-z0-9_-]+`; others are skipped with a warning. Dashboard-summary and document-takeaway runs are not labelled | | `GOODDATA_EVAL_JOIN_GENAI_TRACE` | on unless set to `0`, `false`, `no`, `off` or empty. An agentic item's chat turns ask gen-ai to open them under gd-eval's experiment root, so each item is one Langfuse trace with its scores written once. A gen-ai that drops caller baggage keeps today's separate traces; see the gotcha below | | `GD_EVAL_TIMERS` | same as `--timers` | +| `GD_EVAL_ERROR_OUTCOME` | same as `--error-outcome`; off by default. A run that ended on a harness or backend error (`core/outcome.py`) leaves the gate's K instead of failing, and an item with no scored run raises `EvalHarnessError` (not an `AssertionError`). Changes verdicts, so keep it opt-in | +| `GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER` | same as `--accept-context-date-filter`; off by default. Visualization scoring accepts a date filter equal to one of the dashboard's in the item's `user_context` when the expected output has none (`check_filters`, see the README) | A gitignored `.env` at the repo root is the normal place for these; load it with `set -a && . ./.env && set +a` before the command. diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfda520f4..35bc4eea6 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -133,6 +133,8 @@ gd-eval run \ | `--gate` | `any` | Which verdict decides an item: `any` = pass@K (a run passing is enough), `power` = pass^K (every run must pass, so the verdict measures stability). Identical at `--runs 1`. Kinds that repeat K runs only — `power` is refused when the dataset also has non-agentic items or `agentic_conversation`, which are always decided on pass@K. | | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | | `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail`, `dashboard_summary` and `document_takeaway`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | +| `--error-outcome` | off | Report a run that ended on a harness or backend error as an **error**, not a failure. Changes verdicts, so it is opt-in; also settable via `GD_EVAL_ERROR_OUTCOME=1`. See *Harness and backend errors* below. | +| `--accept-context-date-filter` | off | Visualization scoring also accepts a date filter the agent carried over from the dashboard in the item's `user_context` when the expected output has none. Changes verdicts, so it is opt-in; also settable via `GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER=1`. See *Date filter carried over from the dashboard* below. | | `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` and `document_takeaway` items go through their own endpoints, which have no such option. | **Concurrency and workspace safety.** Agentic kinds that create workspace objects @@ -265,6 +267,95 @@ the same reason. When some runs were ungraded the note says so instead of callin `agentic_conversation` has no K gate, so `--gate power` is refused for a dataset containing one rather than labelling a report `power` that only part of the dataset was decided under. +**Which runs failed, and why.** A failure message names every failing run right after the gate note, with the +reason its loop stopped and only the checks it failed (expected/actual where the kind knows them), then the best +run's summary as before: + +``` +Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure. Failing runs: run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual ...]. +``` + +The same data is in each `failed_runs[]` record: `exit_reason` (the run's `LoopExit`, `null` for a kind without +one), `outcome` (see below) and, for the visualization, alert, metric and dashboard skills, `failed_checks` +(`[{"name", "label"?, "expected"?, "actual"?}]`, each `expected`/`actual` longer than 160 characters stored cut to +that length; the run's `detail` keeps the whole value). + +These are **always on**, independent of `GD_EVAL_ERROR_OUTCOME`: the "Failing runs" part of an agentic failure +message, the `failed_runs[]` fields `exit_reason`, `outcome` and `failed_checks`, and the dashboard skill's +per-run `exit_reason` in its run detail. They add information only; no verdict, score or exception type changes. +With the flag off, a run's `outcome` is what the gate counted it as, so every failing run reads `FAIL` — also one +that ended on `chat_error` or `simulated_user_failed` (its exit is still named). + +### Harness and backend errors (`GD_EVAL_ERROR_OUTCOME`) + +A run can end for a reason that is not the agent's: the simulated user's own model call fails +(`simulated_user_failed`) or the chat call still fails after its retries (`chat_error`). Each run gets an outcome, +from one mapping (`gooddata_eval.outcome_from_loop_exit`): + +| Loop exit | Outcome | +|---|---| +| `success`, `budget_exhausted`, `agent_silent`, `agent_timeout`, `turn_incomplete` | scored: `pass` or `fail` | +| `simulated_user_failed`, `not_run` | `error`, source `harness` | +| `chat_error` | `error`, source `backend` | + +`agent_timeout` (the agent exceeded the turn or item time budget) and `turn_incomplete` (the agent ended its turn +without a final answer) are the agent's own failures, so they stay scored. `chat_error` is what is left: a +transport fault or a 5xx that still fails after its retries. `not_run` is mapped for completeness, but only an +`agentic_conversation` turn records it, and that kind does not use outcomes (see below), so it never decides +anything today. + +The item budget (`--item-timeout`, `GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S`) runs from conversation creation, so on a +multi-turn item it also covers the simulated user's replies and the chat retries' backoff in between. An +`agent_timeout` caused by the item budget can therefore include harness time; the per-turn budget +(`GOODDATA_EVAL_CHAT_TURN_TIMEOUT_S`) times one agent turn only. + +**Changed with the flag off too:** a run whose turn timed out or ended without a final answer used to record +`exit_reason` `chat_error`; it now records `agent_timeout` or `turn_incomplete`, in the run detail and in +`failed_runs[]` (every agentic kind, and the turn results of `agentic_conversation`). Verdicts, scores and +exceptions do not change; only that value does, so filters on `exit_reason == "chat_error"` see fewer runs. + +A run that passed its checks is `pass` whatever its exit. By default the outcome decides nothing: such a run counts +as a failure, exactly as in earlier releases, and is recorded in `failed_runs[].outcome` +(`{"status", "source", "reason"}`) as the `fail` it was counted as. + +With `--error-outcome` or `GD_EVAL_ERROR_OUTCOME=1` (`0`, `false`, `no`, `off` and empty mean off): + +- **Agentic kinds** (visualization, alert, metric, dashboard, KDA, anomaly detection, what-if): `error` runs are + left out of K and the gate is applied to the scored runs. `[error, pass, pass]` passes under `--gate power`; + the excluded runs are listed in `detail.excluded_runs` and, on a failure, once, after the gate note + (`Gate pass^3 (1 excluded) failed: 1/2 scored runs passed ... 1 run excluded (harness error): run 1/3 the + simulated user failed to reply.`), not again under "Failing runs". `runs_effective` and the report's `runs` stay + the runs executed (so `avg_latency_s` divides by them); the new `runs_scored` is the runs the gate counted. An + item with **no** scored run raises `gooddata_eval.EvalHarnessError` — deliberately not an `AssertionError`, so a + runner can tell it apart from an agent failure — and is reported with `error` set, its runs and `failed_runs` + kept. Its message says the item could not be scored rather than that an assertion failed. Note that pytest + itself does not make this a JUnit ``: an exception raised in a test body is written as `` (and + Tavern turns any exception into a failure). Recording it as an error is the runner's job, e.g. a pytest hook + that matches `EvalHarnessError`. Only the seven kinds above use outcomes; the other agentic kinds + (`agentic_conversation`, `agentic_general_question`, `agentic_search`, `agentic_guardrail`, + `agentic_document_skill`, `agentic_forecasting`, `agentic_dashboard_summary`) are unaffected, even where their + runs record a loop exit. +- **Langfuse, agentic kinds**: a run left out of K gets one score, `eval_error` = 1 (NUMERIC, commented + `: `), and no pass, gate, strict-check or quality score. Scored runs keep their usual scores, + so an item whose runs all errored has no `gate_passed` at all. Read `eval_error` = 1 as the error marker, not + a missing `gate_passed`: a score write that failed also leaves `gate_passed` missing. +- **Single-shot kinds**: an item that raised is errored only — it no longer also reports `pass_at_k: true` from a + run that passed before the exception (`runs_passed` still counts that run) — and it is still sent to the + Langfuse sink, with the error in its item metadata, `eval_error` = 1 commented with the error, the `latency_s` + score when a run completed, and no pass or quality score. + +**Single-shot and agentic kinds treat a chat failure differently.** A single-shot item (`visualization`, +`search_tool`, ...) stops at the first `chat_error` (or any other exception) from its chat call and the whole item +is errored (`error` set), with or without the flag; the flag only stops it from also reporting a pass and sends it +to Langfuse. An agentic item records the failure on that run, carries on with its remaining runs, and with the +flag on leaves only that run out of K. + +An agent timeout or incomplete turn is the agent's own failure. With the flag on, a single-shot run that ends that +way is a failed run (`failed_runs[]` with `exit_reason` `agent_timeout` / `turn_incomplete`, outcome `fail`), the +item's other runs go on, and the item gets its normal pass and quality scores; an agentic run that ends that way +is scored as a failure too. With the flag off a single-shot item still errors on it, exactly as in earlier +releases. + Each item additionally carries a per-phase breakdown: ```json @@ -902,6 +993,31 @@ In a Langfuse dataset, put it in the item `metadata` (or in the `input` object) `null` value in `input` falls back to `metadata`. The value used must be an object or `null`; anything else fails the dataset load. +#### Date filter carried over from the dashboard (opt-in) + +A user who asks for a new chart from an open dashboard without naming a period may get +the dashboard's date filter on the chart, and the product accepts that. By default the +visualization scoring still requires the chart's date filters to equal the expected ones. +Pass `--accept-context-date-filter` to `gd-eval run`, set `GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER=1` +(`0`, `false`, `no`, `off` and empty mean off), or pass `accept_context_date_filter=True` to +`check_filters` to also accept, for the date filter only, a chart that: + +- has exactly one date filter, whose period equals a `date_filter` in + `user_context.view.dashboard.filters` (resolved against the same day), and +- belongs to an item whose expected output has no date filter. + +The dashboard's common date filter (no `using`) accepts any date dataset, one that names a +dataset accepts only that one; the one the chart used is reported in the run detail as +`filter_date_context: {"inherited": true, "dataset": ...}`, and a failure summary shows +`date : True (inherited from dashboard context)`. A chart with no date filter passes as +before; an expected date filter, a different period, ranking and attribute filters stay +strict. Applies to `visualization`, `vis_agentic` and `agentic_visualization` items; with the +variable unset the scores and the detail are exactly as without it. + +With the flag on, an expected output cannot say "all time, on purpose": an expected output +with no date filter then also accepts a chart that copied the dashboard's period. An item +that must reject the dashboard's period needs to run without the flag. + `agentic_conversation` items take the item's `user_context` as the context of the first turn. A turn in `expected_output.turns` can set its own `user_context`, which applies from that turn on until another turn changes it, as the attached context does in the UI: diff --git a/packages/gooddata-eval/src/gooddata_eval/__init__.py b/packages/gooddata-eval/src/gooddata_eval/__init__.py index 5e1a8d0fc..462b6a404 100644 --- a/packages/gooddata-eval/src/gooddata_eval/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/__init__.py @@ -2,5 +2,19 @@ """gooddata-eval: evaluate the GoodData AI agent against your own datasets.""" from gooddata_eval._version import __version__ +from gooddata_eval.core.outcome import ( + ErrorSource, + EvalHarnessError, + Outcome, + OutcomeStatus, + outcome_from_loop_exit, +) -__all__ = ["__version__"] +__all__ = [ + "ErrorSource", + "EvalHarnessError", + "Outcome", + "OutcomeStatus", + "__version__", + "outcome_from_loop_exit", +] diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index ad1514ef9..ad156db48 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -29,7 +29,7 @@ from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.evaluators.visualization import expectation_options -from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem +from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem, EvalHarnessError from gooddata_eval.core.runner import EvalReport, ItemReport from gooddata_eval.core.timing import TraceRecorder @@ -397,6 +397,10 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None: if effective: # Only when the kind knows better than K -- agentic_conversation runs once. item_report.runs_effective = effective + runs_scored = getattr(source, "runs_scored", None) + if isinstance(runs_scored, int): + # Only with GD_EVAL_ERROR_OUTCOME on: the runs the gate counted, beside those executed. + item_report.runs_scored = runs_scored # The agentic kinds record their unscored runs in the detail; the report field is the # one place every kind's count is read from. unscored = (getattr(source, "detail", None) or {}).get("unscored_runs") @@ -594,6 +598,22 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: # kind that attaches no diagnostics would leave at 0. item_report.runs_ungraded = item_report.runs print(f"[agentic] {item.id} UNGRADED: {exc}", flush=True) + except EvalHarnessError as exc: + # Raised only with GD_EVAL_ERROR_OUTCOME on: every run ended for a reason that is not + # the agent's. Errored, with the runs it drove and their records kept, like the + # ungraded branch above. + item_report.error = f"{type(exc).__name__}: {exc}" + item_report.runs = k + item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or [] + item_report.conversation_id = getattr(exc, "conversation_id", None) + item_report.response_id = getattr(exc, "response_id", None) + item_report.best_detail = getattr(exc, "detail", None) or {} + _apply_timings(item_report, getattr(exc, "timings", None)) + _apply_best_run_latency(item_report, exc) + _apply_run_counts(item_report, exc) + _apply_failed_runs(item_report, exc) + _apply_run_latencies(item_report, exc) + print(f"[agentic] {item.id} ERROR: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" item_report.runs = 0 diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 89a770030..a324db6ff 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -17,8 +17,10 @@ from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.chat.sse_client import ChatClient, set_default_item_timeout, set_default_turn_timeout from gooddata_eval.core.config import ( + ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR, DEFAULT_GATE, DEFAULT_JUDGE_MODEL, + ERROR_OUTCOME_ENV_VAR, JUDGE_MODEL_ENV_VAR, EvalGate, ReasoningEffort, @@ -136,6 +138,24 @@ def _build_parser() -> argparse.ArgumentParser: "Off by default because a large run emits hundreds of lines; the same measurements " "are always in the JSON report's latency_breakdown_s. Equivalent to GD_EVAL_TIMERS=1.", ) + run.add_argument( + "--error-outcome", + dest="error_outcome", + action="store_true", + help="Report a run that ended on a harness or backend error (simulated user failed, chat " + "call failed after retries) as an error instead of a failure: it is left out of the gate's " + "K, and an item with no other run errors instead of failing. Off by default because it " + "changes verdicts. Equivalent to GD_EVAL_ERROR_OUTCOME=1.", + ) + run.add_argument( + "--accept-context-date-filter", + dest="accept_context_date_filter", + action="store_true", + help="Visualization scoring also accepts a chart whose single date filter equals one of the " + "dashboard's in the item's user_context, when the expected output has no date filter. " + "Off by default because it changes verdicts. Equivalent to " + "GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER=1.", + ) run.add_argument( "--turn-timeout", dest="turn_timeout", @@ -309,6 +329,24 @@ def _apply_timer_flag(enabled: bool) -> None: os.environ[TIMERS_ENV_VAR] = "1" +def _apply_error_outcome_flag(enabled: bool) -> None: + """Publish --error-outcome for the agentic evaluators and the runner, like --timers. + + Only ever sets it -- never clears a value the caller exported themselves. + """ + if enabled: + os.environ[ERROR_OUTCOME_ENV_VAR] = "1" + + +def _apply_accept_context_date_filter_flag(enabled: bool) -> None: + """Publish --accept-context-date-filter for check_filters, like --error-outcome. + + Only ever sets it -- never clears a value the caller exported themselves. + """ + if enabled: + os.environ[ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR] = "1" + + def _reject_power_gate_on_ungated_items(config: RunConfig, items: list) -> None: """Refuse a pass^K request the run cannot honour for every item. @@ -722,6 +760,8 @@ def sdk_factory(): def main(argv: list[str] | None = None) -> int: args = parse_args(argv if argv is not None else sys.argv[1:]) _apply_timer_flag(getattr(args, "timers", False)) + _apply_error_outcome_flag(getattr(args, "error_outcome", False)) + _apply_accept_context_date_filter_flag(getattr(args, "accept_context_date_filter", False)) _apply_judge_model(getattr(args, "judge_model", None)) if hasattr(args, "concurrency") and args.concurrency < 1: print("error: --concurrency must be >= 1.", file=sys.stderr) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py index a324978d5..c0a4e234e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py @@ -18,14 +18,26 @@ from __future__ import annotations from collections.abc import Callable, Sequence +from enum import Enum from typing import Any +from gooddata_eval.core.agentic._failure_message import cap_failed_checks +from gooddata_eval.core.agentic._outcome import recorded_run_outcome + + +def _exit_reason(run: Any) -> str | None: + value = getattr(run, "exit_reason", None) + if isinstance(value, Enum): + return str(value.value) + return value if isinstance(value, str) else None + def build_failed_runs( run_results: Sequence[Any], *, passed: Callable[[Any], bool], detail: Callable[[Any], dict], + failed_checks: Callable[[Any], list[dict]] | None = None, ) -> list[dict]: """One record per run that did not pass, in the order the runs happened. @@ -38,6 +50,14 @@ def build_failed_runs( A run the judge could not grade is recorded like any other non-passing run, with its ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``. + + ``exit_reason`` is the run's ``LoopExit`` value, or null for a kind whose loop records + none, and ``outcome`` its ``core.outcome.Outcome`` record: what the gate counted the run + as, so ERROR for a harness or backend exit only with ``GD_EVAL_ERROR_OUTCOME`` on and + FAIL otherwise. ``failed_checks`` is written only when the caller supplies it: a missing + key means the kind does not name its failing checks, while an empty list would claim the + run failed none. Its expected/actual values are capped in size; the run's ``detail`` + keeps them whole. """ records: list[dict] = [] for run_index, run in enumerate(run_results, start=1): @@ -45,18 +65,21 @@ def build_failed_runs( continue tool_calls = list(getattr(run, "tool_call_events", None) or []) reasoning_steps = list(getattr(run, "reasoning_steps", None) or []) - records.append( - { - "run_index": run_index, - "passed": False, - "error": getattr(run, "judge_error", None), - "detail": detail(run), - "conversation_id": getattr(run, "conversation_id", None), - "response_id": getattr(run, "response_id", None), - "reasoning_step_count": len(reasoning_steps), - "reasoning_steps": reasoning_steps, - "tool_call_count": len(tool_calls), - "tool_names": [getattr(e, "function_name", None) for e in tool_calls], - } - ) + record: dict[str, Any] = { + "run_index": run_index, + "passed": False, + "error": getattr(run, "judge_error", None), + "detail": detail(run), + "conversation_id": getattr(run, "conversation_id", None), + "response_id": getattr(run, "response_id", None), + "reasoning_step_count": len(reasoning_steps), + "reasoning_steps": reasoning_steps, + "tool_call_count": len(tool_calls), + "tool_names": [getattr(e, "function_name", None) for e in tool_calls], + "exit_reason": _exit_reason(run), + "outcome": recorded_run_outcome(run).as_record(), + } + if failed_checks is not None: + record["failed_checks"] = cap_failed_checks(failed_checks(run)) + records.append(record) return records diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py new file mode 100644 index 000000000..5e1930082 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py @@ -0,0 +1,142 @@ +# (C) 2026 GoodData Corporation +"""The head of an agentic kind's failure message: the gate result, then every failing run. + +Each kind's assertion message describes its BEST run, which on a partial gate failure (some +of the K runs passed) is a passing run: every check reads True and the reason the item +failed is nowhere in the message. This module renders the part that is missing -- for each +failing run its index, why its loop stopped and the checks it failed, with expected/actual +where the kind knows them -- from the same ``failed_runs`` records the item reports, so the +message and the triage data cannot disagree about which runs failed or why. + +One renderer for every kind on purpose: the readers of these messages (people and the +triage tooling) look for one shape, whatever the capability under test. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +__all__ = ["FailedCheck", "cap_failed_checks", "failed_check_records", "failing_runs_note", "failure_head"] + +# A value longer than this is cut: the message is one line per run, and the whole value is in +# the run's detail anyway. +_MAX_VALUE_CHARS = 160 + + +class _Unset(Enum): + UNSET = "unset" + + +_UNSET = _Unset.UNSET + + +@dataclass(frozen=True) +class FailedCheck: + """One check a run failed, with the expected/actual pair when the kind knows it. + + ``label`` narrows the check when one name covers several parts, e.g. the ``date`` part of + ``filters_correct``. + """ + + name: str + expected: Any = _UNSET + actual: Any = _UNSET + label: str | None = None + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into a failing run's ``failed_checks``.""" + record: dict[str, Any] = {"name": self.name} + if self.label is not None: + record["label"] = self.label + if self.expected is not _UNSET: + record["expected"] = self.expected + if self.actual is not _UNSET: + record["actual"] = self.actual + return record + + +def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]: + """``checks`` as the records ``failed_runs[].failed_checks`` holds.""" + return [check.as_record() for check in checks] + + +def _text(value: Any) -> str: + text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True) + return " ".join(text.split()) + + +def _value(value: Any) -> str: + if value is None or value == [] or value == {} or value == "": + return "none" + text = _text(value) + return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..." + + +def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """``records`` with every expected/actual value cut to what the message shows of it. + + A check's values can be whole filter or bucket structures, repeated for every failing run + of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value + that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's + ``detail`` holds the full values. + """ + capped: list[dict[str, Any]] = [] + for record in records: + entry = dict(record) + for key in ("expected", "actual"): + if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS: + entry[key] = _value(entry[key]) + capped.append(entry) + return capped + + +def _render_check(record: Mapping[str, Any]) -> str: + name = str(record.get("name", "?")) + parts: list[str] = [] + if "expected" in record: + parts.append(f"expected {_value(record['expected'])}") + if "actual" in record: + parts.append(f"actual {_value(record['actual'])}") + label = record.get("label") + if not parts and label is None: + return name + inner = ", ".join(parts) + if label is not None: + inner = f"{label}: {inner}" if inner else str(label) + return f"{name}[{inner}]" + + +def _render_run(record: Mapping[str, Any], runs_total: int) -> str: + outcome = record.get("outcome") + status = "FAIL" + if isinstance(outcome, Mapping) and outcome.get("status") == "error": + status = f"ERROR {outcome.get('source') or ''}".rstrip() + exit_reason = record.get("exit_reason") + head = f"run {record.get('run_index', '?')}/{runs_total} {status}" + if exit_reason: + head += f" (loop exit: {exit_reason})" + checks = record.get("failed_checks") + if checks: + return f"{head}: {', '.join(_render_check(c) for c in checks)}" + if checks is not None: + return f"{head}: no failing check recorded" + return head + + +def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """Every failing run on one line each, joined with ``; ``. Empty when no run failed.""" + return "; ".join(_render_run(record, runs_total) for record in failed_runs) + + +def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """``gate_note`` followed by the failing runs, ready to start a kind's assertion message. + + Returns ``gate_note`` unchanged when there is no failing run to list, so a kind that + builds no ``failed_runs`` keeps its message exactly as before. + """ + note = failing_runs_note(failed_runs, runs_total) + return f"{gate_note} Failing runs: {note}." if note else gate_note diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py index 63980bb07..8057433e7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py @@ -10,6 +10,7 @@ __all__ = [ "DEFAULT_GATE", "EvalGate", + "gate_count_note", "gate_failure_note", "gate_label", "gate_passed", @@ -29,7 +30,19 @@ def gate_label(gate: str | None, k: int) -> str: return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}" -def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str: +def gate_count_note(gate: str | None, runs_passed: int, runs_total: int, *, runs_excluded: int = 0) -> str: + """E.g. ``pass^3 failed: 2/3 runs passed``, or with runs left out of the gate + ``pass^3 (1 excluded) failed: 1/2 scored runs passed`` (``runs_total`` is then the scored + runs). Shared by the assertion message and the console, so the two cannot disagree.""" + if runs_excluded: + label = f"{gate_label(gate, runs_total + runs_excluded)} ({runs_excluded} excluded)" + return f"{label} failed: {runs_passed}/{runs_total} scored runs passed" + return f"{gate_label(gate, runs_total)} failed: {runs_passed}/{runs_total} runs passed" + + +def gate_failure_note( + gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0, *, runs_excluded: int = 0 +) -> str: """Gate and how many runs met it, for the assertion message. Needed because the message body describes the BEST run, which under pass^K can be a run @@ -37,9 +50,12 @@ def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ An ungraded run counts in runs_total but can never count in runs_passed, so the remainder is not evidence of instability — saying so would blame the agent for a judge outage. + + ``runs_excluded`` (only with ``GD_EVAL_ERROR_OUTCOME`` on) counts the runs left out of the + gate; ``runs_total`` is then the scored runs. The label keeps the K that ran, e.g. + ``pass^3 (1 excluded) failed: 1/2 scored runs passed``. """ - label = gate_label(gate, runs_total) - note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed" + note = f"Gate {gate_count_note(gate, runs_passed, runs_total, runs_excluded=runs_excluded)}" if runs_ungraded: return f"{note}, {runs_ungraded} ungraded." if normalize_gate(gate) == "power" and runs_passed: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py index 576aee838..75c658157 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py @@ -19,18 +19,229 @@ from __future__ import annotations +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from enum import Enum from typing import Any, NoReturn +from gooddata_eval.core.agentic._failure_message import failure_head +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.config import error_outcome_enabled from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + EvalHarnessError, + LoopExit, ReasoningStepEvent, ToolCallEvent, timeline_detail, ) +from gooddata_eval.core.outcome import ( + EVAL_ERROR_SCORE, + Outcome, + OutcomeStatus, + excluded_runs_note, + gate_outcomes, + outcome_from_loop_exit, +) from gooddata_eval.core.timing import PhaseTimings -__all__ = ["agentic_detail", "agentic_success", "raise_agentic_failure"] +__all__ = [ + "CHAT_FAILURE_EXITS", + "EVAL_ERROR_SCORE", + "RunsVerdict", + "agentic_detail", + "agentic_success", + "chat_failure_exit", + "gate_head", + "log_eval_error_score", + "raise_agentic_failure", + "recorded_run_outcome", + "resolve_runs", + "run_loop_exit", + "run_outcome", +] + + +CHAT_FAILURE_EXITS = frozenset({LoopExit.CHAT_ERROR, LoopExit.AGENT_TIMEOUT, LoopExit.TURN_INCOMPLETE}) +"""Every exit ``chat_failure_exit`` can return: the loop stopped because a chat call raised.""" + + +def chat_failure_exit(exc: BaseException) -> LoopExit: + """The ``LoopExit`` for a run whose chat call raised ``exc``. + + ``TurnTimeoutError`` and ``TurnIncompleteError`` subclass ``ChatError`` but are the + agent's own failures -- too slow, or no final answer -- so they get exits the run is + scored on. Anything else a kind catches there (a transport fault, a 5xx after its + retries) is ``CHAT_ERROR``, which ``core/outcome.py`` makes an ERROR(backend) when + ``GD_EVAL_ERROR_OUTCOME`` is on. + """ + if isinstance(exc, TurnTimeoutError): + return LoopExit.AGENT_TIMEOUT + if isinstance(exc, TurnIncompleteError): + return LoopExit.TURN_INCOMPLETE + return LoopExit.CHAT_ERROR + + +def run_loop_exit(run: Any) -> LoopExit | None: + """The run's ``LoopExit``, or None for a kind whose run records none.""" + value = getattr(run, "exit_reason", None) + if isinstance(value, LoopExit): + return value + if isinstance(value, Enum): + value = value.value + try: + return LoopExit(value) if isinstance(value, str) else None + except ValueError: + return None + + +def run_outcome(run: Any, *, passed: bool) -> Outcome: + """One run's ``Outcome``: ``outcome_from_loop_exit`` over the run's own exit.""" + return outcome_from_loop_exit(run_loop_exit(run), passed=passed) + + +def recorded_run_outcome(run: Any) -> Outcome: + """The outcome a failing run's record carries: what the gate counted it as. + + With ``GD_EVAL_ERROR_OUTCOME`` on that is ``run_outcome``. Off, the gate counts every run, + so a run that ended on a harness or backend exit was a failure like any other and is + recorded as FAIL, its exit kept in the reason. + """ + outcome = run_outcome(run, passed=False) + if outcome.status is OutcomeStatus.ERROR and not error_outcome_enabled(): + loop_exit = run_loop_exit(run) + return Outcome.failed(f"loop exit: {loop_exit.value}" if loop_exit is not None else None) + return outcome + + +@dataclass(frozen=True) +class RunsVerdict: + """What an agentic kind gates on, with ``GD_EVAL_ERROR_OUTCOME`` applied or not. + + Off, every field is what the kind computed before outcomes existed: the counts are over + every run, ``pass_at_k``/``pass_power_k`` are the kind's own and ``runs_scored`` is None. + On, ERROR runs are left out of the gate: ``runs_scored`` is the number of scored runs, and + ``errored`` is set when there is none. ``runs_effective`` is the runs executed either way, + so per-run latency and the gate's K keep counting the runs that took time. + """ + + pass_at_k: bool + pass_power_k: bool + runs_passed: int + runs_effective: int + runs_total: int + errored: bool = False + excluded_note: str = "" + item_outcome: Outcome | None = None + excluded_runs: list[dict] = field(default_factory=list) + runs_scored: int | None = None + excluded: tuple[tuple[int, Outcome], ...] = () + """``(run_index, outcome)`` of every run left out of the gate, 1-based. Empty when off.""" + + @property + def excluded_indexes(self) -> frozenset[int]: + """The 1-based indexes of the runs left out of the gate. Empty with the flag off.""" + return frozenset(index for index, _ in self.excluded) + + def excluded_outcome(self, run_index: int) -> Outcome | None: + """The ERROR outcome of run ``run_index`` (1-based) when it was left out, else None.""" + return dict(self.excluded).get(run_index) + + def exception_cls( + self, assertion_cls: type[AgenticAssertionError] + ) -> type[AgenticAssertionError | EvalHarnessError]: + """``EvalHarnessError`` for an item with no scored run, ``assertion_cls`` otherwise.""" + return EvalHarnessError if self.errored else assertion_cls + + def message_lead(self, assertion_lead: str, kind: str) -> str: + """How a kind's failure message starts: ``assertion_lead``, or for an item with no + scored run (``EvalHarnessError``) ``" item could not be scored."``, since no + assertion about the agent was made.""" + return f"{kind} item could not be scored." if self.errored else assertion_lead + + def detail(self) -> dict[str, Any]: + """The extra ``detail`` field the flag adds: the runs it left out. Empty when off.""" + return {"excluded_runs": self.excluded_runs} if self.excluded_runs else {} + + @property + def outcome_record(self) -> dict[str, Any] | None: + return self.item_outcome.as_record() if self.item_outcome is not None else None + + +def resolve_runs( + run_results: Sequence[Any], + *, + passed: Callable[[Any], bool], + gate: str | None, + pass_at_k: bool, + pass_power_k: bool, +) -> RunsVerdict: + """The verdict inputs of an item, from its runs; see ``RunsVerdict``. + + ``pass_at_k``/``pass_power_k`` are the kind's own, returned unchanged with the flag off. + """ + total = len(run_results) + if not error_outcome_enabled(): + return RunsVerdict( + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + runs_passed=sum(1 for r in run_results if passed(r)), + runs_effective=total, + runs_total=total, + ) + verdict = gate_outcomes([run_outcome(r, passed=passed(r)) for r in run_results], gate) + return RunsVerdict( + pass_at_k=verdict.pass_at_k, + pass_power_k=verdict.pass_power_k, + runs_passed=verdict.runs_passed, + runs_effective=total, + runs_total=total, + runs_scored=verdict.runs_scored, + excluded=verdict.excluded, + errored=verdict.outcome.status is OutcomeStatus.ERROR, + excluded_note=excluded_runs_note(verdict), + item_outcome=verdict.outcome, + excluded_runs=[ + { + "run_index": index, + "exit_reason": getattr(run_loop_exit(run_results[index - 1]), "value", None), + "outcome": outcome.as_record(), + } + for index, outcome in verdict.excluded + ], + ) + + +def gate_head(gate: str | None, runs: RunsVerdict, failed_runs: Sequence[dict]) -> str: + """The start of a kind's failure message: the gate result, then every failing run. + + With the flag off this is ``gate_failure_note`` followed by the failing runs. On, the runs + left out of K follow the gate note, and an item with no scored run says so instead. + """ + if runs.errored: + note = f"No scored run, item errored: {runs.excluded_note}" + else: + excluded = len(runs.excluded) + note = gate_failure_note(gate, runs.runs_passed, runs.runs_total - excluded, runs_excluded=excluded) + if runs.excluded_note: + note = f"{note} {runs.excluded_note}" + # A run left out of the gate is named once, in the excluded note, not again as failing. + excluded_indexes = runs.excluded_indexes + failing = [r for r in failed_runs if r.get("run_index") not in excluded_indexes] + return failure_head(note, failing, runs.runs_total) + + +def log_eval_error_score(ctx: Any, trace_id: Any, outcome: Outcome) -> None: + """Mark one run left out of the gate (``GD_EVAL_ERROR_OUTCOME`` on) as an error. + + The run gets ``eval_error`` = 1, its comment ``: ``, and no pass, gate, + strict-check or quality score: any of those would read as a verdict on the agent, which + the run is not. An item whose runs all errored therefore has no gate score at all. + """ + source = outcome.source.value if outcome.source is not None else "unknown" + ctx.score(trace_id, name=EVAL_ERROR_SCORE, value=1, data_type="NUMERIC", comment=f"{source}: {outcome.reason}") def agentic_detail( @@ -47,7 +258,7 @@ def agentic_detail( def raise_agentic_failure( - exception_cls: type[AgenticAssertionError], + exception_cls: type[AgenticAssertionError | EvalHarnessError], message: str, *, reasoning_steps: list[str], @@ -60,14 +271,20 @@ def raise_agentic_failure( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + outcome: dict[str, Any] | None = None, + runs_scored: int | None = None, ) -> NoReturn: """Build ``exception_cls(message)`` with every common field attached, and raise it. ``exception_cls`` must be an ``AgenticAssertionError`` subclass -- that base class is - what declares these fields as legal targets (see its docstring). A kind whose own + what declares these fields as legal targets (see its docstring) -- or ``EvalHarnessError``, + which declares the same ones plus ``outcome``. A kind whose own "no verdict at all" branch raises a bare ``JudgeResponseError`` instead (not a subclass) does not go through this helper for that branch. + ``runs_scored`` is set only with ``GD_EVAL_ERROR_OUTCOME`` on: the runs the gate was + applied to, where ``runs_effective`` stays the runs executed. + ``timings`` stays optional: only the three kinds that track per-run ``PhaseTimings`` (general_question, metric_skill, dashboard_skill) pass one -- the rest keep their own ``PhaseTimings()`` zero default, same as before this helper existed. ``failed_runs`` @@ -88,6 +305,10 @@ def raise_agentic_failure( exc.failed_runs = failed_runs if run_latencies_s is not None: exc.run_latencies_s = run_latencies_s + if outcome is not None and isinstance(exc, EvalHarnessError): + exc.outcome = outcome + if runs_scored is not None: + exc.runs_scored = runs_scored raise exc @@ -103,6 +324,7 @@ def agentic_success( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + runs_scored: int | None = None, ) -> AgenticEvalOutcome: """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape.""" kwargs: dict[str, Any] = { @@ -120,4 +342,6 @@ def agentic_success( kwargs["failed_runs"] = failed_runs if run_latencies_s is not None: kwargs["run_latencies_s"] = run_latencies_s + if runs_scored is not None: + kwargs["runs_scored"] = runs_scored return AgenticEvalOutcome(**kwargs) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py index 73eebbbf2..28e0e98a4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py @@ -217,9 +217,10 @@ def export_unscored(self, run_idx: int, conversation_id: str) -> None: with self.observe(self.trace(conversation_id), run_idx, conversation_id=conversation_id): pass - def score(self, trace_id: Any, *, name: str, value: Any, data_type: str) -> None: + def score(self, trace_id: Any, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: """Write one score, swallowing Langfuse failures the way ``score_safe`` always has.""" - self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type) + extra = {"comment": comment} if comment is not None else {} + self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type, **extra) def quality(self, trace_id: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: """Write the derived quality/value scores for one run.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index f1f532af3..031602638 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -7,7 +7,7 @@ import os import re import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any import httpx @@ -15,15 +15,23 @@ from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1021,7 +1029,7 @@ def _run_once(conv_id: str) -> AlertRunResult: partial_alert_id, _, partial_tool_called = _extract_alert_call(partial.tool_call_events or []) if partial_tool_called: alert_id_to_delete = partial_alert_id - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break # `turns_used` counts attempts (set above, so a failed send still shows the # loop reached this iteration); `turns`/`steps` count completed work, so they @@ -1164,6 +1172,11 @@ def _run_detail(run: AlertRunResult) -> dict: } +def _failed_checks(run: AlertRunResult) -> list[dict]: + """The checks this run failed, by name. The arguments the run sent are in its detail.""" + return failed_check_records(FailedCheck(name) for name, ok in run.eval.checks().items() if not ok) + + def evaluate_agentic_alert_skill( host: str, token: str, @@ -1220,6 +1233,16 @@ def evaluate_agentic_alert_skill( user_context=user_context, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.eval.strict_pass, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1231,6 +1254,10 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.eval.checks() with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -1258,14 +1285,16 @@ def _write_scores(ctx: RunTraceContext) -> None: item_input=question, ) - runs_passed = sum(1 for r in summary.run_results if r.eval.strict_pass) - runs_effective = len(summary.run_results) + runs_passed = runs_verdict.runs_passed + runs_effective = runs_verdict.runs_effective best = summary.best ev = best.eval detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, + # Empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + **runs_verdict.detail(), alert_created=ev.alert_created, operator_correct=ev.operator_correct, threshold_correct=ev.threshold_correct, @@ -1288,13 +1317,14 @@ def _write_scores(ctx: RunTraceContext) -> None: summary.run_results, passed=lambda r: r.eval.strict_pass, detail=_run_detail, + failed_checks=_failed_checks, ) - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) raise_agentic_failure( - AlertSkillAssertionError, - f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " + runs_verdict.exception_cls(AlertSkillAssertionError), + f"{runs_verdict.message_lead('Alert skill assertion failed.', 'Alert skill')} {gate_note} strict_pass={ev.strict_pass}. " f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, " f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, " @@ -1302,12 +1332,14 @@ def _write_scores(ctx: RunTraceContext) -> None: f"attributes_correct={ev.attributes_correct}, " f"granularity_correct={ev.granularity_correct}. " f"Actual args: {best.actual_alert_arguments}", + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, @@ -1319,6 +1351,7 @@ def _write_scores(ctx: RunTraceContext) -> None: detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py index 25f2281a0..33e654d9e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py @@ -29,19 +29,26 @@ import os import re import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -421,7 +428,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) viz_args, execute_result = _extract_anomaly_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -583,6 +590,16 @@ def evaluate_agentic_anomaly_detection( user_context=user_context, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.evaluation.strict_pass, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -613,6 +630,10 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -649,7 +670,8 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.evaluation - detail = _detail(best) + # excluded_runs is empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + detail = {**_detail(best), **runs_verdict.detail()} # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. failed_runs = build_failed_runs( @@ -657,12 +679,12 @@ def _write_scores(ctx: RunTraceContext) -> None: passed=lambda r: r.evaluation.strict_pass, detail=_detail, ) - runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + runs_passed = runs_verdict.runs_passed - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) message = ( - f"Anomaly detection assertion failed. {gate_note} strict_pass={ev.strict_pass} " + f"{runs_verdict.message_lead('Anomaly detection assertion failed.', 'Anomaly detection')} {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " f"turn_completed={ev.turn_completed}, metric_correct={ev.metric_correct}, " f"granularity_correct={ev.granularity_correct}). " @@ -670,14 +692,16 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Actual execute result: {best.actual_execute_result}." ) raise_agentic_failure( - AnomalyDetectionAssertionError, + runs_verdict.exception_cls(AnomalyDetectionAssertionError), message, + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, - runs_effective=len(summary.run_results), + runs_effective=runs_verdict.runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, @@ -689,7 +713,8 @@ def _write_scores(ctx: RunTraceContext) -> None: response_id=best.response_id, detail=detail, runs_passed=runs_passed, - runs_effective=len(summary.run_results), + runs_effective=runs_verdict.runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 92f106340..5d227c9e8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -56,7 +56,13 @@ summarize_visualizations, ) from gooddata_eval.core.agentic._gate import log_gate_scores -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1038,7 +1044,7 @@ def run_agentic_conversation( conversation_reasoning_step_events.extend(partial.reasoning_step_events or []) reasoning_steps.extend(partial.reasoning_steps or []) response_id = partial.response_id or response_id - turn_exit = LoopExit.CHAT_ERROR + turn_exit = chat_failure_exit(exc) break final_result = chat_result total_steps += chat_result.reasoning_step_count @@ -1255,7 +1261,7 @@ def run_agentic_conversation( # A chat fault used to escape the whole run, so reaching here did mean no # error. Now that it is caught and recorded, this has to read it back -- # otherwise a turn whose chat call failed reports no_error=True. - no_error=turn_exit is not LoopExit.CHAT_ERROR, + no_error=turn_exit not in CHAT_FAILURE_EXITS, activated_skills=declared or [], active_skills=sorted(active_skills), clarification_turns_used=clarification_turns, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index 2fafc30ba..91060716c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -5,21 +5,29 @@ import re import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any import jsonpatch from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -2067,7 +2075,7 @@ def _execute_single_dashboard_run( all_tool_call_events.extend(partial.tool_call_events or []) all_reasoning_step_events.extend(partial.reasoning_step_events or []) steps += partial.reasoning_step_count - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -2178,10 +2186,17 @@ def _run_detail(run: DashboardRunResult) -> dict[str, Any]: **run.evaluation.diagnostics, "failures": run.evaluation.failures, "notes": run.evaluation.notes, + # Why the loop stopped -- see LoopExit. + "exit_reason": run.exit_reason.value, "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), } +def _failed_checks(run: DashboardRunResult) -> list[dict]: + """The checks this run failed, by name. What each one found is in the run's ``failures``.""" + return failed_check_records(FailedCheck(name) for name, ok in run.evaluation.strict_checks.items() if not ok) + + def run_agentic_dashboard_skill( host: str, token: str, @@ -2304,6 +2319,16 @@ def evaluate_agentic_dashboard_skill( user_context=user_context, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.evaluation.strict_pass, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -2315,6 +2340,10 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") for name, value in run.evaluation.diagnostics.items(): @@ -2345,13 +2374,15 @@ def _write_scores(ctx: RunTraceContext) -> None: ) item_timings = sum_timings([r.timings for r in summary.run_results]) - runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) - runs_effective = len(summary.run_results) + runs_passed = runs_verdict.runs_passed + runs_effective = runs_verdict.runs_effective best = summary.best detail: dict[str, Any] = agentic_detail( best.tool_call_events, best.reasoning_step_events, + # Empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + **runs_verdict.detail(), **best.evaluation.strict_checks, **best.evaluation.diagnostics, failures=best.evaluation.failures, @@ -2366,10 +2397,11 @@ def _write_scores(ctx: RunTraceContext) -> None: summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_run_detail, + failed_checks=_failed_checks, ) - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) # Two failures wear the same missing skill, and the advice differs. If the other # dashboard skill was activated, the flag that registers both is plainly on and the # model simply routed the wrong way. If neither was, the flag is off or the agent has @@ -2388,17 +2420,19 @@ def _write_scores(ctx: RunTraceContext) -> None: ) notes = "; ".join(best.evaluation.notes) raise_agentic_failure( - DashboardSkillAssertionError, - f"Dashboard skill assertion failed. {gate_note}{skill_note} " + runs_verdict.exception_cls(DashboardSkillAssertionError), + f"{runs_verdict.message_lead('Dashboard skill assertion failed.', 'Dashboard skill')} {gate_note}{skill_note} " f"Checks: {best.evaluation.strict_checks}. " f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." + (f" Notes (not scored): {notes}." if notes else ""), + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, best_run_latency_s=run_latency_s(best.timings), timings=item_timings, failed_runs=failed_runs, @@ -2411,6 +2445,7 @@ def _write_scores(ctx: RunTraceContext) -> None: detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=run_latency_s(best.timings), diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 683f8d42d..0f250bfe3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -6,7 +6,7 @@ import logging import os import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace import httpx @@ -14,12 +14,19 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -329,7 +336,7 @@ def _accumulate(result: ChatResult) -> None: if create_args is not None: turn_wall_clock_sec = partial.turn_wall_clock_sec turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break turns += 1 steps += chat_result.reasoning_step_count @@ -499,6 +506,16 @@ def evaluate_agentic_kda_skill( user_context=user_context, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.evaluation.strict_pass, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -525,6 +542,10 @@ def _write_scores(ctx: RunTraceContext) -> None: "[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") @@ -561,14 +582,16 @@ def _write_scores(ctx: RunTraceContext) -> None: item_input=question, ) - runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) - runs_effective = len(summary.run_results) + runs_passed = runs_verdict.runs_passed + runs_effective = runs_verdict.runs_effective best = summary.best ev = best.evaluation detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, + # Empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + **runs_verdict.detail(), triggered=ev.triggered, executed=ev.executed, success=ev.success, @@ -589,24 +612,26 @@ def _write_scores(ctx: RunTraceContext) -> None: detail=_run_detail, ) - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) message = ( - f"KDA skill assertion failed. {gate_note} strict_pass={ev.strict_pass} " + f"{runs_verdict.message_lead('KDA skill assertion failed.', 'KDA skill')} {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, " f"success={ev.success}, turn_completed={ev.turn_completed}). " f"Actual create args: {best.actual_create_args}. " f"Actual execute result: {best.actual_execute_result}." ) raise_agentic_failure( - KdaSkillAssertionError, + runs_verdict.exception_cls(KdaSkillAssertionError), message, + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, @@ -618,6 +643,7 @@ def _write_scores(ctx: RunTraceContext) -> None: detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index f662b205b..65b64a81f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -6,22 +6,30 @@ import os import re import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from gooddata_api_client.exceptions import NotFoundException from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -352,7 +360,7 @@ def _execute_single_metric_run( for metric_id in _extract_created_metric_ids(partial.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -562,6 +570,16 @@ def evaluate_agentic_metric_skill( user_context=user_context, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.metric_created and r.maql_correct, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -577,6 +595,10 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=run.conversation_id, output={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, ) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -606,8 +628,8 @@ def _write_scores(ctx: RunTraceContext) -> None: item_timings = sum_timings([r.timings for r in summary.run_results]) - runs_passed = sum(1 for r in summary.run_results if r.metric_created and r.maql_correct) - runs_effective = len(summary.run_results) + runs_passed = runs_verdict.runs_passed + runs_effective = runs_verdict.runs_effective run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best @@ -631,9 +653,26 @@ def _run_detail(run: MetricRunResult) -> dict: **timeline_detail(run.tool_call_events, run.reasoning_step_events), } + def _failed_checks(run: MetricRunResult) -> list[dict]: + """The checks this run failed; the MAQL pair is the whole diagnosis for this kind.""" + checks: list[FailedCheck] = [] + if not run.metric_created: + checks.append(FailedCheck("metric_created", expected=True, actual=False)) + if not run.maql_correct: + checks.append( + FailedCheck( + "maql_correct", + expected=[c.get("maql", "") for c in expected_outputs_list], + actual=run.actual_maql, + ) + ) + return failed_check_records(checks) + detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, + # Empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + **runs_verdict.detail(), metric_created=best.metric_created, maql_correct=best.maql_correct, expected_maql_candidates=[c.get("maql", "") for c in expected_outputs_list], @@ -650,23 +689,26 @@ def _run_detail(run: MetricRunResult) -> dict: summary.run_results, passed=lambda r: r.metric_created and r.maql_correct, detail=_run_detail, + failed_checks=_failed_checks, ) - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) raise_agentic_failure( - MetricSkillAssertionError, - f"Metric skill assertion failed. {gate_note} " + runs_verdict.exception_cls(MetricSkillAssertionError), + f"{runs_verdict.message_lead('Metric skill assertion failed.', 'Metric skill')} {gate_note} " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}.", + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, best_run_latency_s=run_latency_s(best.timings), timings=item_timings, failed_runs=failed_runs, @@ -679,6 +721,7 @@ def _run_detail(run: MetricRunResult) -> dict: detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=run_latencies_s, best_run_latency_s=run_latency_s(best.timings), diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b06821452..09005e726 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -12,15 +12,24 @@ from dataclasses import dataclass, field, replace from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -204,6 +213,7 @@ def _execute_single_run( max_iterations: int = _DEFAULT_MAX_ITERATIONS, requires_execution: bool = False, options: ExpectationOptions | None = None, + user_context: dict | None = None, ) -> RunResult: """Drive one full multi-turn conversation and evaluate the result.""" options = _effective_options(options, requires_execution) @@ -232,21 +242,21 @@ def _execute_single_run( current_result = client.send_message(conversation_id, question) except ChatError as exc: # The opening request, so there is no partial conversation to evaluate: fall through - # to the empty-result path with CHAT_ERROR recorded. Raising here would discard every + # to the empty-result path with its exit recorded. Raising here would discard every # K-run already completed along with any exit_reason. print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}") current_result = getattr(exc, "partial_result", None) or ChatResult() - # The loop that collects events does not run after CHAT_ERROR, so a partial result's + # The loop that collects events does not run after a failed opening request, so a partial result's # chart executions are added here. all_tool_call_events.extend(current_result.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) # Counted here, not at the top of the loop: this request is sent unconditionally, so # with max_iterations=0 the loop body never runs and total_turns would report 0 turns # for a conversation the agent did receive. The loop's own increment is skipped on its # first pass to compensate, keeping total_turns == number of send_message calls. total_turns += 1 - for iteration in range(max_iterations if exit_reason is not LoopExit.CHAT_ERROR else 0): + for iteration in range(max_iterations if exit_reason not in CHAT_FAILURE_EXITS else 0): if iteration: total_turns += 1 total_steps += current_result.reasoning_step_count @@ -297,7 +307,7 @@ def _execute_single_run( # The loop adds a turn's events at its next pass, which a break skips. The # partial result is scored, so its chart executions count too. all_tool_call_events.extend(partial.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break skill_activated = _check_visualization_skill_activated(all_tool_call_events) @@ -305,7 +315,9 @@ def _execute_single_run( if current_result.created_visualizations and current_result.created_visualizations.objects: actual_output = current_result.created_visualizations.objects[0] - eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated) + eval_result, best_expected = _evaluate_against_candidates( + expected_outputs, actual_output, skill_activated, user_context=user_context + ) eval_result = with_execution( eval_result, all_tool_call_events, current_result.text_response, options.requires_execution ) @@ -316,7 +328,7 @@ def _execute_single_run( actual_output, all_tool_call_events, answer_text, - interrupted=exit_reason is LoopExit.CHAT_ERROR, + interrupted=exit_reason in CHAT_FAILURE_EXITS, ) return RunResult( @@ -392,7 +404,14 @@ def run_agentic_visualization( try: run_results.append( _execute_single_run( - client, conv_id_0, question, expected_outputs, max_iterations, requires_execution, options + client, + conv_id_0, + question, + expected_outputs, + max_iterations, + requires_execution, + options, + user_context=user_context, ) ) finally: @@ -404,7 +423,14 @@ def run_agentic_visualization( try: run_results.append( _execute_single_run( - client, conv_id, question, expected_outputs, max_iterations, requires_execution, options + client, + conv_id, + question, + expected_outputs, + max_iterations, + requires_execution, + options, + user_context=user_context, ) ) finally: @@ -445,6 +471,76 @@ def _run_detail(run: RunResult) -> dict: } +def _failed_checks(run: RunResult) -> list[dict]: + """The checks this run failed, by their ``detail`` names, with expected/actual where known. + + Follows ``EvaluationResult.strict_pass`` term by term, so the list is empty exactly when + the run passed. + """ + ev = run.eval_result + checks: list[FailedCheck] = [] + if ev.scored_without_chart: + if ev.visualization_created: + actual_type = run.actual_output.type if run.actual_output else None + checks.append(FailedCheck("no_chart", expected="no chart", actual=actual_type)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + return failed_check_records(checks) + if not ev.visualization_created: + checks.append(FailedCheck("visualization_created", expected=True, actual=False)) + if not ev.cross_ref_valid: + checks.append(FailedCheck("cross_ref_valid", actual=ev.cross_ref_errors)) + if not ev.metrics_correct: + if ev.expected_metric_uris == ev.actual_metric_uris and ev.metric_layout_problems: + # The metric set matched; the order or axis did not, and only these say which. + checks.append(FailedCheck("metrics_correct", label="layout", actual=ev.metric_layout_problems)) + else: + checks.append( + FailedCheck( + "metrics_correct", expected=sorted(ev.expected_metric_uris), actual=sorted(ev.actual_metric_uris) + ) + ) + if not ev.dimensions_correct: + checks.append( + FailedCheck("dimensions_correct", expected=sorted(ev.expected_dim_uris), actual=sorted(ev.actual_dim_uris)) + ) + if not ev.filters_correct: + differing = [ + FailedCheck( + "filters_correct", + label=category, + expected=ev.expected_filters.get(category, []), + actual=ev.actual_filters.get(category, []), + ) + for category in sorted(set(ev.expected_filters) | set(ev.actual_filters)) + if ev.expected_filters.get(category, []) != ev.actual_filters.get(category, []) + # An inherited date filter was accepted, so it is not why the filters failed. + and not (category == "date" and ev.filter_date_inherited) + ] + checks.extend(differing or [FailedCheck("filters_correct")]) + if not ev.sorts_correct: + checks.append(FailedCheck("sorts_correct", expected=ev.expected_sorts, actual=ev.actual_sorts)) + if not ev.viz_type_hard: + checks.append( + FailedCheck( + "viz_type_hard", + expected=run.best_expected.type if run.best_expected else None, + actual=run.actual_output.type if run.actual_output else None, + ) + ) + if ev.requires_execution and not ev.executed: + checks.append(FailedCheck("executed", expected=True, actual=False)) + if ev.buckets_correct is False: + checks.append(FailedCheck("buckets_correct", expected=ev.expected_buckets, actual=ev.actual_buckets)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + if ev.format_correct is False: + checks.append(FailedCheck("format_correct", actual=ev.format_problems)) + if ev.part_present is False: + checks.append(FailedCheck("part_present", expected=True, actual=False)) + return failed_check_records(checks) + + def _strict_checks(ev: EvaluationResult) -> dict[str, bool]: """The scores one run is gated on, by their Langfuse names. @@ -579,6 +675,16 @@ def evaluate_agentic_visualization( options=options, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.eval_result.strict_pass, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -592,6 +698,10 @@ def _write_scores(ctx: RunTraceContext) -> None: ev = run.eval_result strict_checks = _strict_checks(ev) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue for score_name, value in strict_checks.items(): # Scored below on every item, gating or not. if score_name != "assertion-vis-executed": @@ -640,8 +750,8 @@ def _write_scores(ctx: RunTraceContext) -> None: with open(record_output_path, "w") as _f: _j.dump(_fixture, _f, indent=2) - runs_passed = sum(1 for r in summary.run_results if r.eval_result.strict_pass) - runs_effective = len(summary.run_results) + runs_passed = runs_verdict.runs_passed + runs_effective = runs_verdict.runs_effective run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best @@ -649,6 +759,8 @@ def _write_scores(ctx: RunTraceContext) -> None: detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, + # Empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + **runs_verdict.detail(), **evaluation_result_detail(ev), # Why the loop stopped -- see LoopExit. total_turns is already the turn count for # this run, so it doubles as turns_used. @@ -664,10 +776,11 @@ def _write_scores(ctx: RunTraceContext) -> None: summary.run_results, passed=lambda r: r.eval_result.strict_pass, detail=_run_detail, + failed_checks=_failed_checks, ) - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) n = len(expected_outputs) candidate_note = f" (closest of {n} candidates)" if n > 1 else "" cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" @@ -678,9 +791,10 @@ def _write_scores(ctx: RunTraceContext) -> None: ) actual_dump = best.actual_output.model_dump(exclude_none=True) if best.actual_output else None raise_agentic_failure( - VisualizationAssertionError, + runs_verdict.exception_cls(VisualizationAssertionError), "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" - "Agentic Visualization Assertion Failed! (Critical Mode)\n" + f"{runs_verdict.message_lead('Agentic Visualization Assertion Failed!', 'Agentic Visualization')}" + " (Critical Mode)\n" f"{gate_note}\n" "------------------------------------------\n" f"Question:\n{question}\n" @@ -699,8 +813,9 @@ def _write_scores(ctx: RunTraceContext) -> None: f" expected : {sorted(ev.expected_dim_uris)}\n" f" actual : {sorted(ev.actual_dim_uris)}\n" f" Filters Correct : {ev.filters_correct}\n" - f" date : {ev.filter_date_score}\n" - f"{_filter_diff('date', ev)}" + f" date : {ev.filter_date_score}" + f"{' (inherited from dashboard context)' if ev.filter_date_inherited else ''}\n" + f"{'' if ev.filter_date_inherited else _filter_diff('date', ev)}" f" ranking : {ev.filter_ranking_score}\n" f"{_filter_diff('ranking', ev)}" f" attribute : {ev.filter_attribute_score}\n" @@ -711,12 +826,14 @@ def _write_scores(ctx: RunTraceContext) -> None: f"{_options_lines(ev, best.answer_text)}" f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=run_latencies_s, best_run_latency_s=best.run_latency_s, @@ -728,6 +845,7 @@ def _write_scores(ctx: RunTraceContext) -> None: detail=detail, runs_passed=runs_passed, runs_effective=runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=run_latencies_s, best_run_latency_s=best.run_latency_s, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index de124ceed..8ea9c6801 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -23,19 +23,26 @@ import logging import os import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + log_eval_error_score, + raise_agentic_failure, + resolve_runs, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -378,7 +385,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) create_args, execute_result = _extract_what_if_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -544,6 +551,16 @@ def evaluate_agentic_what_if( user_context=user_context, ) + # Before the Langfuse scores, so the gate they record is the one that decides the item. + runs_verdict = resolve_runs( + summary.run_results, + passed=lambda r: r.evaluation.strict_pass, + gate=gate, + pass_at_k=summary.pass_at_k, + pass_power_k=summary.pass_power_k, + ) + summary = replace(summary, pass_at_k=runs_verdict.pass_at_k, pass_power_k=runs_verdict.pass_power_k) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -576,6 +593,10 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + # Left out of the gate: eval_error only, nothing that reads as a verdict. + if (run_error := runs_verdict.excluded_outcome(run_idx + 1)) is not None: + log_eval_error_score(ctx, tid, run_error) + continue for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -612,7 +633,8 @@ def _write_scores(ctx: RunTraceContext) -> None: best = summary.best ev = best.evaluation - detail = _detail(best) + # excluded_runs is empty unless GD_EVAL_ERROR_OUTCOME left runs out of the gate. + detail = {**_detail(best), **runs_verdict.detail()} # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. _detail describes one run, so a failing # run is reported with exactly the keys the winning one is. @@ -621,12 +643,12 @@ def _write_scores(ctx: RunTraceContext) -> None: passed=lambda r: r.evaluation.strict_pass, detail=_detail, ) - runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + runs_passed = runs_verdict.runs_passed - if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + if runs_verdict.errored or not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_head(gate, runs_verdict, failed_runs) message = ( - f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} " + f"{runs_verdict.message_lead('What-if assertion failed.', 'What-if')} {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " f"turn_completed={ev.turn_completed}, metric_correct={ev.metric_correct}, " f"maql_correct={ev.maql_correct}, scenario_count_correct={ev.scenario_count_correct}, " @@ -635,14 +657,16 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Actual execute result: {best.actual_execute_result}." ) raise_agentic_failure( - WhatIfAssertionError, + runs_verdict.exception_cls(WhatIfAssertionError), message, + outcome=runs_verdict.outcome_record, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, detail=detail, runs_passed=runs_passed, - runs_effective=len(summary.run_results), + runs_effective=runs_verdict.runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, @@ -654,7 +678,8 @@ def _write_scores(ctx: RunTraceContext) -> None: response_id=best.response_id, detail=detail, runs_passed=runs_passed, - runs_effective=len(summary.run_results), + runs_effective=runs_verdict.runs_effective, + runs_scored=runs_verdict.runs_scored, failed_runs=failed_runs, run_latencies_s=[r.run_latency_s for r in summary.run_results], best_run_latency_s=best.run_latency_s, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index bb5f838ae..583e01c4a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -17,6 +17,29 @@ def env_flag(name: str) -> bool: return os.environ.get(name, "").strip().lower() not in _ENV_FALSE +# Off by default: on, a run that ended for a reason that is not the agent's (see +# core/outcome.py) is left out of the gate instead of counting as a failure, and an item with +# no scored run raises EvalHarnessError instead of its kind's *AssertionError. That changes +# verdicts, which is why it is opt-in. +ERROR_OUTCOME_ENV_VAR = "GD_EVAL_ERROR_OUTCOME" + + +def error_outcome_enabled() -> bool: + """Whether harness and backend errors are reported as errors rather than scored failures.""" + return env_flag(ERROR_OUTCOME_ENV_VAR) + + +# Opt-in: accept a date filter the agent carried over from the dashboard open in +# the item's `user_context`, when the expected output states no date filter. Off by default: +# a caller that does not ask for it gets the strict date comparison. +ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR = "GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER" + + +def accept_context_date_filter() -> bool: + """Whether `check_filters` may accept a date filter inherited from the dashboard context.""" + return env_flag(ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR) + + # The judge model, overridable for comparison experiments. The default is deliberate on # two counts: gpt-4o honours temperature=0, so a given response always gets the same # verdict; and it is NOT the family under test, so the judge is not grading its own diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py index 10806d047..59880c6dd 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py @@ -83,6 +83,15 @@ class EvaluationResult: format_correct: bool | None = None format_problems: list[str] = field(default_factory=list) part_present: bool | None = None + # Set only when `filter_date_score` holds because the actual date filter is the one the + # dashboard in the item's `user_context` applies (opt-in, see `check_filters`): the date + # dataset that filter used. Reported in the detail only when set. + filter_date_inherited_dataset: str | None = None + + @property + def filter_date_inherited(self) -> bool: + """Whether `filter_date_score` holds only through the dashboard-context rule.""" + return self.filter_date_inherited_dataset is not None @property def scored_without_chart(self) -> bool: @@ -453,6 +462,9 @@ def _evaluate_visualization( actual: CreatedVisualization | None, skill_activated: bool = False, today: date | None = None, + *, + user_context: Mapping | None = None, + accept_context_date_filter: bool | None = None, ) -> EvaluationResult: # One anchor for the scoring and for the filters reported beside it. check_filters # already shares an anchor across its two sides, but resolving the reported filters @@ -489,7 +501,13 @@ def _evaluate_visualization( cross_ref_valid, cross_ref_errors = validate_cross_references(actual) act_metric_uris = get_metric_uri_set(actual) act_dim_uris = get_dimension_uri_set(actual) - filter_scores = check_filters(expected, actual, today) + filter_scores = check_filters( + expected, + actual, + today, + user_context=user_context, + accept_context_date_filter=accept_context_date_filter, + ) metric_layout_problems = check_metric_layout(expected, actual) return EvaluationResult( visualization_created=True, @@ -516,6 +534,7 @@ def _evaluate_visualization( expected_buckets=bucket_layout(expected) if expected.buckets_strict else None, actual_buckets=bucket_layout(actual) if expected.buckets_strict else None, metric_layout_problems=metric_layout_problems, + filter_date_inherited_dataset=filter_scores.date_inherited_dataset, ) @@ -551,13 +570,29 @@ def _evaluate_against_candidates( actual: CreatedVisualization | None, skill_activated: bool = False, today: date | None = None, + *, + user_context: Mapping | None = None, + accept_context_date_filter: bool | None = None, ) -> tuple[EvaluationResult, CreatedVisualization | None]: if not expected_outputs: return _no_candidate_result(actual, skill_activated), None # Anchored once here too: the candidates are ranked against each other, so letting # each resolve its own today could rank them on spans from different days. today = today or date.today() - pairs = [(_evaluate_visualization(exp, actual, skill_activated, today), exp) for exp in expected_outputs] + pairs = [ + ( + _evaluate_visualization( + exp, + actual, + skill_activated, + today, + user_context=user_context, + accept_context_date_filter=accept_context_date_filter, + ), + exp, + ) + for exp in expected_outputs + ] best_result, best_expected = max(pairs, key=lambda p: (p[0].strict_pass, p[0].strict_checks_passed_count)) return best_result, best_expected @@ -624,6 +659,13 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict: "actual_dim_uris": sorted(ev.actual_dim_uris), "expected_filters": ev.expected_filters, "actual_filters": ev.actual_filters, + # Nested and only when it applies: `quality_score` counts every top-level boolean, and + # a run scored without the opt-in rule must not carry the key at all. + **( + {"filter_date_context": {"inherited": True, "dataset": ev.filter_date_inherited_dataset}} + if ev.filter_date_inherited + else {} + ), "expected_sorts": ev.expected_sorts, "actual_sorts": ev.actual_sorts, **({"metric_layout_problems": ev.metric_layout_problems} if ev.metric_layout_problems else {}), @@ -658,7 +700,9 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation candidates = _parse_expected(item.expected_output) actual = _extract_actual(chat_result) skill_activated = _check_visualization_skill_activated(chat_result.tool_call_events) - ev, _best_expected = _evaluate_against_candidates(candidates, actual, skill_activated) + ev, _best_expected = _evaluate_against_candidates( + candidates, actual, skill_activated, user_context=item.user_context + ) ev = with_execution(ev, chat_result.tool_call_events, chat_result.text_response, options.requires_execution) ev = with_options(ev, options, actual, chat_result.tool_call_events, render_answer_text(chat_result)) return ItemEvaluation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py index 35367fe58..36a6d424c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py @@ -8,6 +8,7 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any +from gooddata_eval.core.config import error_outcome_enabled from gooddata_eval.core.langfuse.client import HttpxLangfuseClient from gooddata_eval.core.langfuse.experiment import ( ExperimentItem, @@ -15,6 +16,7 @@ build_experiment_root_span, trace_tags, ) +from gooddata_eval.core.outcome import EVAL_ERROR_SCORE _MAX_LATENCY_S = 60.0 _QUALITY_WEIGHT = 0.6 @@ -132,7 +134,7 @@ def _build_span(self, report: ItemReport, dataset_item_id: str, run: ExperimentR dataset_item_id, input={"question": report.question}, output=report.best_detail, - metadata={"test_kind": report.test_kind}, + metadata={"test_kind": report.test_kind, **({"error": report.error} if report.error else {})}, ), start=start, end=end, @@ -153,7 +155,10 @@ def _build_span(self, report: ItemReport, dataset_item_id: str, run: ExperimentR def log_item(self, report: ItemReport, *, dataset_item_id: str) -> None: """Export one experiment root span plus its four scores for an evaluated item. - Swallows all errors — Langfuse failures never abort the eval run. + Swallows all errors — Langfuse failures never abort the eval run. With + GD_EVAL_ERROR_OUTCOME on, an errored item gets its error in the item metadata, no pass + or quality score, ``eval_error`` = 1 with the error as its comment, and the latency + score when it completed a run. With the flag off the scores are written as always. """ scores = compute_scores( pass_at_k=report.pass_at_k, @@ -175,7 +180,18 @@ def log_item(self, report: ItemReport, *, dataset_item_id: str) -> None: ("value_score", scores["value_score"], "NUMERIC"), ("latency_s", scores["latency_s"], "NUMERIC"), ] + comments: dict[str, str] = {} + if report.error is not None and error_outcome_enabled(): + # An errored item has no verdict, so it gets no pass or quality score: a 0 would + # count it as a failure of the agent. Nor a latency one when it raised before + # completing a run: avg_latency_s is then a 0.0 that measured nothing and would + # drag the latency averages down. It gets eval_error = 1 instead, the score an + # agentic run left out of the gate gets too (core/agentic/_outcome.py). + score_defs = [d for d in score_defs if d[0] == "latency_s" and report.runs_total > 0] + score_defs.append((EVAL_ERROR_SCORE, 1, "NUMERIC")) + comments[EVAL_ERROR_SCORE] = report.error for name, value, data_type in score_defs: + extra = {"comment": comments[name]} if name in comments else {} try: self._client.create_score( trace_id=span.trace_id, @@ -183,6 +199,7 @@ def log_item(self, report: ItemReport, *, dataset_item_id: str) -> None: name=name, value=value, data_type=data_type, + **extra, ) except Exception as exc: # noqa: PERF203 — one score's failure must not skip the rest print(f"warning: Langfuse score '{name}' failed for item '{report.id}': {exc}", file=sys.stderr) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 435ac811a..26052f2da 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -28,6 +28,11 @@ class LoopExit(str, Enum): This does not change any verdict -- an exhausted run still fails. It makes the three cases countable, so "is the turn budget too tight" becomes a question the data can answer rather than one that has to be argued. + + Which members say nothing about the agent (an ERROR rather than a scored run) is decided + in one place, ``core/outcome.py``, and decides a verdict only with + ``GD_EVAL_ERROR_OUTCOME`` on. ``AGENT_TIMEOUT`` and ``TURN_INCOMPLETE`` are deliberately + not among them: they are raised as ``ChatError`` subclasses, but the agent caused them. """ SUCCESS = "success" @@ -45,8 +50,18 @@ class LoopExit(str, Enum): fault, not an agent one -- reporting should treat it as an error, not a scored failure.""" CHAT_ERROR = "chat_error" - """The chat call itself raised mid-conversation. Like SIMULATED_USER_FAILED this is - infrastructure rather than agent capability, but it comes from the GoodData side.""" + """The chat call itself raised mid-conversation and was not one of the agent's own + failures below: a transport fault, or a 5xx still failing after its retries. Like + SIMULATED_USER_FAILED this is infrastructure rather than agent capability, but it comes + from the GoodData side.""" + + AGENT_TIMEOUT = "agent_timeout" + """The agent exceeded the per-turn or per-item wall-clock budget (``TurnTimeoutError``). + The agent's own failure to answer in time, so the run is scored, not an error.""" + + TURN_INCOMPLETE = "turn_incomplete" + """The agent ended its turn without a final answer, e.g. at its own iteration limit + (``TurnIncompleteError``). The agent's own failure, so the run is scored, not an error.""" NOT_RUN = "not_run" """The loop never started -- e.g. an agentic_conversation turn whose $ref to an earlier @@ -435,6 +450,33 @@ class AgenticAssertionError(AssertionError): failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None + runs_scored: int + + +class EvalHarnessError(Exception): + """An item with no scored run: every run ended for a reason that is not the agent's. + + Raised only with ``GD_EVAL_ERROR_OUTCOME`` on (see ``core/outcome.py``). Deliberately not + an ``AssertionError``: it is a distinct type so a runner can tell a harness or backend + outage apart from an agent failure and keep it out of the agent's pass rate. Turning it + into a JUnit error is the runner's job (e.g. a pytest hook matching this type); pytest + alone writes any exception raised in a test body as a failure. Carries the same fields + as ``AgenticAssertionError`` so a caller reads both the same way, plus ``outcome``: the + item's ``Outcome`` as a record (``{"status": "error", "source": ..., "reason": ...}``). + """ + + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict + timings: PhaseTimings + runs_passed: int + runs_effective: int + failed_runs: list[dict] + run_latencies_s: list[float] + best_run_latency_s: float | None + runs_scored: int + outcome: dict class AgenticEvalOutcome(BaseModel): @@ -477,6 +519,10 @@ class AgenticEvalOutcome(BaseModel): # not in a loop the runner itself can time. None for a kind that has not yet been # wired to measure it (see AgenticAssertionError.best_run_latency_s, same default). best_run_latency_s: float | None = None + # The runs the gate was applied to, set only with GD_EVAL_ERROR_OUTCOME on: the runs that + # ended on a harness or backend error are left out. ``runs_effective`` stays the runs + # executed, which per-run latency divides by. None means every executed run was gated. + runs_scored: int | None = None class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py new file mode 100644 index 000000000..e6fd2481d --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py @@ -0,0 +1,197 @@ +# (C) 2026 GoodData Corporation +"""One outcome per run: PASS, FAIL, or ERROR when the run says nothing about the agent. + +A run of an agentic kind can end in a way that is the harness's or the platform's fault, +not the agent's: the simulated user's own model call fails, or the chat call still fails +after its retries. Scored as usual, such a run reads as an agent failure. ``Outcome`` names +the difference, and ``outcome_from_loop_exit`` is the one place a ``LoopExit`` is mapped to +it, for every kind. + +The mapping is always computed and always recorded in ``failed_runs[].outcome``. It decides +a verdict only when ``GD_EVAL_ERROR_OUTCOME`` is on (``error_outcome_enabled``); see +``gate_outcomes`` for what changes then. Off, every verdict is the one the kind computed +before this module existed. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from gooddata_eval.core.config import error_outcome_enabled, normalize_gate +from gooddata_eval.core.models import EvalHarnessError, LoopExit + +__all__ = [ + "EVAL_ERROR_SCORE", + "ErrorSource", + "EvalHarnessError", + "GateVerdict", + "Outcome", + "OutcomeStatus", + "error_outcome_enabled", + "excluded_runs_note", + "gate_outcomes", + "outcome_from_loop_exit", +] + + +class OutcomeStatus(str, Enum): + """What one run, or one item, amounts to.""" + + PASS = "pass" + FAIL = "fail" + ERROR = "error" + """Says nothing about the agent: the run is not scored.""" + + +class ErrorSource(str, Enum): + """Whose fault an ERROR is.""" + + HARNESS = "harness" + """gooddata-eval itself, e.g. its simulated user.""" + + INFRA = "infra" + """Something between the eval and the platform: tracing, obfuscation, the network.""" + + BACKEND = "backend" + """The GoodData side answered with an error after every retry.""" + + +@dataclass(frozen=True) +class Outcome: + """PASS, FAIL with an optional reason, or ERROR with its source and reason.""" + + status: OutcomeStatus + source: ErrorSource | None = None + reason: str | None = None + + @classmethod + def passed(cls) -> Outcome: + return cls(OutcomeStatus.PASS) + + @classmethod + def failed(cls, reason: str | None = None) -> Outcome: + return cls(OutcomeStatus.FAIL, reason=reason) + + @classmethod + def error(cls, source: ErrorSource, reason: str) -> Outcome: + return cls(OutcomeStatus.ERROR, source=source, reason=reason) + + @property + def is_scored(self) -> bool: + """Whether the run counts towards the gate's K.""" + return self.status is not OutcomeStatus.ERROR + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into ``failed_runs[].outcome``.""" + return { + "status": self.status.value, + "source": self.source.value if self.source is not None else None, + "reason": self.reason, + } + + +EVAL_ERROR_SCORE = "eval_error" +"""The Langfuse score (NUMERIC, 1) an ERROR run or errored item gets instead of any verdict +score, with ``GD_EVAL_ERROR_OUTCOME`` on. Readers of the experiment rely on the name.""" + + +# Every LoopExit that makes a run an ERROR, with its source and reason. A member not listed +# here ends a run the agent is accountable for, which is then scored as PASS or FAIL. +_ERROR_EXITS: dict[LoopExit, tuple[ErrorSource, str]] = { + LoopExit.SIMULATED_USER_FAILED: (ErrorSource.HARNESS, "the simulated user failed to reply"), + LoopExit.CHAT_ERROR: (ErrorSource.BACKEND, "the chat call failed after its retries"), + LoopExit.NOT_RUN: (ErrorSource.HARNESS, "the run never started"), +} + + +def outcome_from_loop_exit(exit_reason: LoopExit | str | None, *, passed: bool) -> Outcome: + """The outcome of one run, from why its loop stopped and whether it passed its checks. + + ``SUCCESS``, ``BUDGET_EXHAUSTED``, ``AGENT_SILENT``, ``AGENT_TIMEOUT`` and + ``TURN_INCOMPLETE`` are scored: PASS or FAIL. + ``SIMULATED_USER_FAILED`` and ``NOT_RUN`` are ERROR(harness), ``CHAT_ERROR`` is + ERROR(backend). ``None`` (a kind whose loop records no exit) is scored. + + A run that passed is PASS whatever its exit: its checks held on what the agent produced, + so the run is evidence about the agent even when the conversation ended abnormally. + + Raises: + ValueError: ``exit_reason`` is a string that is not a ``LoopExit`` value. + """ + if passed: + return Outcome.passed() + loop_exit = LoopExit(exit_reason) if isinstance(exit_reason, str) else exit_reason + error = _ERROR_EXITS.get(loop_exit) if loop_exit is not None else None + if error is not None: + return Outcome.error(*error) + return Outcome.failed(f"loop exit: {loop_exit.value}" if loop_exit is not None else None) + + +@dataclass(frozen=True) +class GateVerdict: + """An item's verdict over its runs' outcomes, ERROR runs left out of K.""" + + outcome: Outcome + """PASS or FAIL from the gate over the scored runs; ERROR when no run was scored.""" + runs_total: int + runs_scored: int + runs_passed: int + pass_at_k: bool + pass_power_k: bool + excluded: tuple[tuple[int, Outcome], ...] = field(default=()) + """``(run_index, outcome)`` of every ERROR run, 1-based, in run order.""" + + +def gate_outcomes(outcomes: Sequence[Outcome], gate: str | None) -> GateVerdict: + """Apply ``gate`` to the scored runs only. + + ERROR runs are left out of K: ``[ERROR, PASS, PASS]`` under pass^K is pass^2 over the two + scored runs. With no scored run left the item is ERROR, whatever the gate. + """ + excluded = tuple((index, o) for index, o in enumerate(outcomes, start=1) if not o.is_scored) + scored = [o for o in outcomes if o.is_scored] + runs_passed = sum(1 for o in scored if o.status is OutcomeStatus.PASS) + pass_at_k = runs_passed > 0 + pass_power_k = bool(scored) and runs_passed == len(scored) + if not scored: + reasons = "; ".join(sorted({o.reason or "" for _, o in excluded})) + item = Outcome.error(_dominant_source(excluded), f"no scored run: {reasons}") + elif pass_power_k if normalize_gate(gate) == "power" else pass_at_k: + item = Outcome.passed() + else: + item = Outcome.failed() + return GateVerdict( + outcome=item, + runs_total=len(outcomes), + runs_scored=len(scored), + runs_passed=runs_passed, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + excluded=excluded, + ) + + +def _dominant_source(excluded: Sequence[tuple[int, Outcome]]) -> ErrorSource: + counts = Counter(o.source for _, o in excluded if o.source is not None) + return counts.most_common(1)[0][0] if counts else ErrorSource.HARNESS + + +def excluded_runs_note(verdict: GateVerdict) -> str: + """E.g. ``1 run excluded (harness error): run 1/3 the simulated user failed to reply.`` + + Empty when no run was excluded. + """ + if not verdict.excluded: + return "" + sources = Counter(o.source.value if o.source is not None else "unknown" for _, o in verdict.excluded) + if len(sources) == 1: + kinds = f"{next(iter(sources))} error" + else: + kinds = ", ".join(f"{n} {source} error" for source, n in sorted(sources.items())) + n = len(verdict.excluded) + runs = "; ".join(f"run {index}/{verdict.runs_total} {o.reason}" for index, o in verdict.excluded) + return f"{n} run{'s' if n != 1 else ''} excluded ({kinds}): {runs}." diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py index ebf9a09c9..f1d50cd24 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py @@ -4,7 +4,7 @@ from rich.console import Console from rich.table import Table -from gooddata_eval.core.agentic._gate import gate_label +from gooddata_eval.core.agentic._gate import gate_count_note from gooddata_eval.core.interaction import MULTI_TURN, SINGLE_SHOT from gooddata_eval.core.runner import EvalReport, ItemReport from gooddata_eval.core.timing import LATENCY_PERCENTILES @@ -89,7 +89,7 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str # was, and every other column looks identical for the two: quality_score reads # best_detail, which describes the winning run alone. So say it here. result = "PASS" - notes = "" if item.pass_power_k else f"{item.runs_passed}/{item.runs_total} runs passed" + notes = "" if item.pass_power_k else f"{item.runs_passed}/{item.runs_gated} runs passed" else: # Evaluator-agnostic: report whichever boolean checks came back False # (visualization uses metrics_correct/…; dashboard_summary uses @@ -99,8 +99,12 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str # Only pass^K fails an item whose runs passed, and best_detail then # describes one of those -- so no check reads False and Quality prints # 100%. The count is the whole reason the row is a FAIL. - label = gate_label(report.gate, item.runs_total) - notes = f"{label} failed: {item.runs_passed}/{item.runs_total} runs passed" + notes = gate_count_note( + report.gate, + item.runs_passed, + item.runs_gated, + runs_excluded=item.runs_total - item.runs_gated, + ) elif failing: notes = "failed: " + ", ".join(failing) else: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 71df0abf8..2a8db10eb 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -71,6 +71,9 @@ def _build_run_dict(report: EvalReport) -> dict: # runs_ungraded says how many runs pass@K was NOT computed over. "runs_passed": item.runs_passed, "runs_ungraded": item.runs_ungraded, + # Only with GD_EVAL_ERROR_OUTCOME on: the runs the gate counted, `runs` minus + # those that ended on a harness or backend error. + **({"runs_scored": item.runs_scored} if item.runs_scored is not None else {}), "pass_power_k": item.pass_power_k, "best_run_latency_s": ( round(item.best_run_latency_s, 3) if item.best_run_latency_s is not None else None diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 9643f9f42..c92db19ca 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -9,11 +9,13 @@ from functools import cached_property, partial from typing import Callable, Protocol -from gooddata_eval.core.config import DEFAULT_GATE, EvalGate +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.config import DEFAULT_GATE, EvalGate, error_outcome_enabled from gooddata_eval.core.evaluators import get_evaluator, supported_test_kinds from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.interaction import INTERACTION_MODES, MULTI_TURN, InteractionMode, interaction_mode -from gooddata_eval.core.models import ChatResult, DatasetItem +from gooddata_eval.core.models import ChatResult, DatasetItem, LoopExit +from gooddata_eval.core.outcome import Outcome from gooddata_eval.core.timing import latency_stats @@ -102,6 +104,10 @@ class ItemReport: # agentic_conversation drives its fixture exactly once whatever --runs says, so # trusting K there reports four runs that never happened. runs_effective: int | None = None + # The runs the gate was applied to, when GD_EVAL_ERROR_OUTCOME left some out (agentic + # kinds only). runs_total stays the runs executed, the divisor of avg_latency_s; None + # means every executed run was gated. + runs_scored: int | None = None @property def interaction_mode(self) -> InteractionMode: @@ -130,7 +136,12 @@ def pass_power_k(self) -> bool: a later one raised before returning a verdict). That is not unanimity, it is an item whose last run has no verdict at all, so an error disqualifies the claim. """ - return self.error is None and self.runs_total > 0 and self.runs_passed == self.runs_total + return self.error is None and self.runs_gated > 0 and self.runs_passed == self.runs_gated + + @property + def runs_gated(self) -> int: + """The runs the gate was applied to: ``runs_scored`` when set, else every run executed.""" + return self.runs_total if self.runs_scored is None else self.runs_scored @property def avg_latency_s(self) -> float: @@ -273,6 +284,32 @@ def _failed_run_record(run_index: int, evaluation: ItemEvaluation, chat_result: } +def _agent_failed_run_record( + run_index: int, exc: TurnTimeoutError | TurnIncompleteError, loop_exit: LoopExit, latency: float +) -> dict: + """``_failed_run_record`` for a run whose turn the agent itself timed out or left unanswered. + + Nothing was evaluated, so ``detail`` holds why the turn ended; the ids and turn time come + from the partial result when the stream delivered one. + """ + partial = exc.partial_result + return { + "run_index": run_index, + "passed": False, + "error": None, + "detail": {"exit_reason": loop_exit.value, "chat_error": f"{type(exc).__name__}: {exc}"}, + "conversation_id": getattr(partial, "conversation_id", None) or getattr(exc, "conversation_id", None), + "response_id": getattr(partial, "response_id", None), + "stream_ended": getattr(partial, "stream_ended", None), + "turn_wall_clock_sec": getattr(partial, "turn_wall_clock_sec", None), + "latency_s": round(latency, 3), + "reasoning_step_count": getattr(partial, "reasoning_step_count", 0), + "reasoning_steps": list(getattr(partial, "reasoning_steps", None) or []), + "exit_reason": loop_exit.value, + "outcome": Outcome.failed(f"loop exit: {loop_exit.value}").as_record(), + } + + def _run_one_item( item: DatasetItem, backend: ChatBackend, runs: int, on_run_done: RunCallback | None = None ) -> ItemReport: @@ -291,9 +328,29 @@ def _run_one_item( best_run_latency: float | None = None judge_errors: list[str] = [] try: + scores_agent_failures = error_outcome_enabled() for run_index in range(1, runs + 1): t0 = time.perf_counter() - chat_result = backend.ask(item) + try: + chat_result = backend.ask(item) + except (TurnTimeoutError, TurnIncompleteError) as exc: + if not scores_agent_failures: + raise + # With GD_EVAL_ERROR_OUTCOME on, an error is reserved for faults that say nothing + # about the agent. Running out of time, or ending the turn without an answer, is + # the agent's own failure: a failed run, scored, and the item's other runs go on. + loop_exit = LoopExit.AGENT_TIMEOUT if isinstance(exc, TurnTimeoutError) else LoopExit.TURN_INCOMPLETE + latency = time.perf_counter() - t0 + turn_latency = getattr(exc.partial_result, "turn_wall_clock_sec", None) + if isinstance(turn_latency, (int, float)): + report.trace_latencies_s.append(turn_latency) + report.runs += 1 + report.latency_s += latency + report.run_latencies_s.append(latency) + report.failed_runs.append(_agent_failed_run_record(run_index, exc, loop_exit, latency)) + if on_run_done is not None: + on_run_done(run_index, runs, False, latency) + continue trace_latency = getattr(chat_result, "turn_wall_clock_sec", None) if isinstance(trace_latency, (int, float)): report.trace_latencies_s.append(trace_latency) @@ -322,6 +379,10 @@ def _run_one_item( conv_id = getattr(e, "conversation_id", None) report.conversation_id = conv_id or report.conversation_id report.error = f"{type(e).__name__}: {e}" + (f" [conversation_id={conv_id}]" if conv_id else "") + if error_outcome_enabled(): + # An item is passed or errored, never both: a run that passed before the error + # stays counted in runs_passed, but the item did not complete its K runs. + report.pass_at_k = False if best is not None: report.best_detail = best.detail report.best_run_latency_s = best_run_latency @@ -381,6 +442,7 @@ def run_items( - on_run_done(index, total, run_index, runs, passed, latency) after each individual run - on_item_done(index, total, report) after an item is fully evaluated - on_langfuse_item_done(index, total, report) after non-skipped, non-errored items only + (errored ones too with GD_EVAL_ERROR_OUTCOME on) concurrency > 1 dispatches items to a ThreadPoolExecutor so multiple questions are sent to the agent simultaneously. Each item still runs @@ -403,7 +465,14 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: try: if on_item_done is not None: on_item_done(index, total, item_report) - if on_langfuse_item_done is not None and not item_report.skipped and item_report.error is None: + # With GD_EVAL_ERROR_OUTCOME on, an errored item is logged too: leaving it out would + # make an outage invisible in the experiment instead of an error in it. + logs_errors = error_outcome_enabled() + if ( + on_langfuse_item_done is not None + and not item_report.skipped + and (item_report.error is None or logs_errors) + ): on_langfuse_item_done(index, total, item_report) except Exception: # non-fatal — log but don't abort traceback.print_exc() diff --git a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py index 07e6c389e..b6fbc99cd 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/scoring.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/scoring.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from datetime import date, timedelta +from gooddata_eval.core.config import accept_context_date_filter as _accept_context_date_filter_from_env from gooddata_eval.core.granularity import canonical_date_uri from gooddata_eval.core.models import AacBucketRef, AacQueryField, CreatedVisualization @@ -56,6 +57,16 @@ class FilterScores: date_ok: bool ranking_ok: bool attribute_ok: bool + # Set only when `date_ok` holds because the actual date filter is the one the dashboard + # in the item's `user_context` applies (see `check_filters`), not because it matched the + # expected output: the date dataset (`using`) of that filter. The dashboard filter may name + # none, so any date dataset is accepted and the one the agent picked is recorded here. + date_inherited_dataset: str | None = None + + @property + def date_inherited(self) -> bool: + """Whether `date_ok` holds only through the dashboard-context rule.""" + return self.date_inherited_dataset is not None @property def all_ok(self) -> bool: @@ -503,18 +514,93 @@ def normalized_filters(viz: CreatedVisualization, today: date | None = None) -> return {"date": sorted(date_set), "ranking": sorted(ranking_set), "attribute": sorted(attr_set)} +def dashboard_date_filters(user_context: object) -> list[dict]: + """The date filters of the dashboard open in ``user_context`` (``view.dashboard.filters``). + + Anything that is not shaped like that -- no context, no dashboard, a non-list ``filters``, + an entry that is not a ``date_filter`` object -- contributes nothing rather than raising: + the context is the chat API's schema and is not validated by the dataset loader. + """ + if not isinstance(user_context, Mapping): + return [] + view = user_context.get("view") + dashboard = view.get("dashboard") if isinstance(view, Mapping) else None + filters = dashboard.get("filters") if isinstance(dashboard, Mapping) else None + if not isinstance(filters, list): + return [] + return [f for f in filters if isinstance(f, dict) and f.get("type") == "date_filter"] + + +def _inherited_date_filter_dataset( + exp_date: set[str], + act_date: set[str], + actual: CreatedVisualization, + dashboard_filters: list[dict], + today: date, +) -> str | None: + """The dataset of the actual date filter when it is the dashboard's own, else None. + + Holds only when the expected output states no date filter and the actual chart has + exactly one, whose span equals that of one of the dashboard's date filters resolved + against the same ``today``. The dashboard's common date filter names no dataset (it + resolves per widget), so it is compared on the actual filter's dataset and accepts any + date dataset. A dashboard date filter that names one in ``using`` accepts only that one. + """ + if exp_date or not dashboard_filters: + return None + actual_filters = [f for f in actual.query.filter_by.values() if f.get("type") == "date_filter"] + if len(actual_filters) != 1: + return None + dataset = actual_filters[0].get("using", "") + fields = actual.query.fields + for dashboard_filter in dashboard_filters: + candidate = _normalize_date_filter( + {**dashboard_filter, "using": dashboard_filter.get("using") or dataset}, fields, today + ) + if json.dumps(candidate, sort_keys=True) in act_date: + return dataset if isinstance(dataset, str) else "" + return None + + def check_filters( - expected: CreatedVisualization, actual: CreatedVisualization, today: date | None = None + expected: CreatedVisualization, + actual: CreatedVisualization, + today: date | None = None, + *, + user_context: Mapping | None = None, + accept_context_date_filter: bool | None = None, ) -> FilterScores: + """Compare the filters of two visualizations, per category. + + Every category compares exactly. One opt-in exception, for the date category only: + with ``accept_context_date_filter`` (None reads ``GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER``, + off by default), an actual date filter equal to a date filter of the dashboard in + ``user_context`` is accepted when the expected output has none -- the agent carried the + open dashboard's period into the new chart, which the product accepts. The result then + has ``date_inherited`` set. Without the flag, or without a dashboard date filter in + ``user_context``, scoring is exactly the strict comparison. + """ # One anchor for both sides: resolving each against its own date.today() would # score inconsistently for a run that straddles midnight. today = today or date.today() exp_date, exp_rank, exp_attr = _split_and_normalize_filters(expected, today) act_date, act_rank, act_attr = _split_and_normalize_filters(actual, today) + date_ok = act_date == exp_date + inherited_dataset: str | None = None + if not date_ok and user_context is not None: + enabled = ( + _accept_context_date_filter_from_env() if accept_context_date_filter is None else accept_context_date_filter + ) + if enabled: + inherited_dataset = _inherited_date_filter_dataset( + exp_date, act_date, actual, dashboard_date_filters(user_context), today + ) + date_ok = inherited_dataset is not None return FilterScores( - date_ok=act_date == exp_date, + date_ok=date_ok, ranking_ok=act_rank == exp_rank, attribute_ok=act_attr == exp_attr, + date_inherited_dataset=inherited_dataset, ) diff --git a/packages/gooddata-eval/tests/conftest.py b/packages/gooddata-eval/tests/conftest.py index 0f77eaa8a..b5fb98be2 100644 --- a/packages/gooddata-eval/tests/conftest.py +++ b/packages/gooddata-eval/tests/conftest.py @@ -26,6 +26,8 @@ def fixtures_dir() -> Path: "GOODDATA_EVAL_CHAT_TURN_TIMEOUT_S", "GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S", "GOODDATA_EVAL_JOIN_GENAI_TRACE", + "GD_EVAL_ERROR_OUTCOME", + "GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER", ) diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 09cfab387..9513102e3 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -26,7 +26,7 @@ render_turn_for_simulated_user, run_agentic_alert_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _DATE_FILTER = { @@ -1932,3 +1932,21 @@ def test_an_alert_created_before_the_stream_broke_is_still_cleaned_up(): assert mock_delete.call_count == 1 assert mock_delete.call_args.args[-1] == "alert-1" + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + ], +) +def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = exc + + detail = _run_alert(mock_client, max_iterations=3) + + assert detail["exit_reason"] == exit_value + assert detail["alert_created"] is False diff --git a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py index 5c1e25d8b..8043fe51e 100644 --- a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py +++ b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py @@ -16,6 +16,7 @@ evaluate_agentic_anomaly_detection, run_agentic_anomaly_detection, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.anomaly_detection" @@ -539,3 +540,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index ca0f2dc2d..f08b0d63f 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -53,7 +53,7 @@ run_agentic_conversation, ) from gooddata_eval.core.chat.render import render_answer_text -from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.chat.sse_client import ChatError as ChatError_ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent @@ -1209,6 +1209,32 @@ def test_a_partial_result_joins_the_shared_timeline_like_any_other_turn(): assert result.total_steps == 5 +def test_an_agent_timeout_ends_its_turn_with_its_own_exit_and_still_reads_as_an_error(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _metric_turn_result([_skills_tc("metric"), _create_metric_tc("m1")]), + TurnTimeoutError("agent exceeded the 60s turn budget"), + ] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_two_metric_turn_fixture(), + ) + + assert result.turn_results[0].exit_reason is LoopExit.SUCCESS + assert result.turn_results[1].exit_reason is LoopExit.AGENT_TIMEOUT + # The turn's no_error reads exactly as it did when a timeout was recorded as CHAT_ERROR. + assert result.turn_results[1].no_error is False + assert result.turn_results[1].output_present is False + + def test_a_non_chat_exception_still_propagates(): """Only chat faults are absorbed. A programming error must not be relabelled CHAT_ERROR.""" mock_client = MagicMock() diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index 2edfc58e8..0de765e16 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -17,7 +17,7 @@ evaluate_dashboard_response, run_agentic_dashboard_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent # Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of @@ -2362,6 +2362,22 @@ def test_a_caller_supplied_conversation_is_not_deleted(self): client.create_conversation.assert_not_called() client.delete_conversation.assert_not_called() + @pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], + ) + def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(self, exc, loop_exit) -> None: + client = MagicMock() + client.send_message.side_effect = exc + + summary = _run_with(client, _DC05_EXPECTED) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + def test_a_chat_error_is_recorded_rather_than_left_to_escape(self) -> None: """It used to escape run_agentic_dashboard_skill, taking every completed K-run with it: no DashboardRunResult, no exit_reason, just a bare error on the item. Recorded diff --git a/packages/gooddata-eval/tests/test_agentic_failure_message.py b/packages/gooddata-eval/tests/test_agentic_failure_message.py new file mode 100644 index 000000000..8c7ac9c8f --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_failure_message.py @@ -0,0 +1,352 @@ +# (C) 2026 GoodData Corporation +"""The failing-runs head of an agentic failure message, and the per-run records it reads.""" + +import importlib +import inspect +from dataclasses import dataclass, field +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import metric_skill as metric_skill_mod +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import ( + FailedCheck, + failed_check_records, + failing_runs_note, + failure_head, +) +from gooddata_eval.core.agentic.metric_skill import MetricSkillAssertionError, evaluate_agentic_metric_skill +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.config import ERROR_OUTCOME_ENV_VAR +from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit +from gooddata_eval.core.outcome import outcome_from_loop_exit + +# ── the renderer ───────────────────────────────────────────────────────────── + + +def test_a_failed_check_records_only_what_the_kind_knows() -> None: + assert FailedCheck("metric_correct").as_record() == {"name": "metric_correct"} + assert FailedCheck("filters_correct", label="date", expected=[], actual=["MONTH -11..0"]).as_record() == { + "name": "filters_correct", + "label": "date", + "expected": [], + "actual": ["MONTH -11..0"], + } + # None is a real expected value, not "unknown". + assert FailedCheck("x", expected=None).as_record() == {"name": "x", "expected": None} + + +def test_the_note_lists_each_failing_run_with_its_exit_and_failing_checks_only() -> None: + failed_runs = [ + { + "run_index": 2, + "exit_reason": "success", + "failed_checks": failed_check_records( + [FailedCheck("filters_correct", label="date", expected=[], actual="MONTH -11..0")] + ), + }, + {"run_index": 3, "exit_reason": "budget_exhausted", "failed_checks": [{"name": "metric_correct"}]}, + ] + + note = failing_runs_note(failed_runs, 3) + + assert note == ( + "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual MONTH -11..0]; " + "run 3/3 FAIL (loop exit: budget_exhausted): metric_correct" + ) + + +def test_the_head_starts_with_the_gate_note() -> None: + gate_note = "Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure." + failed_runs = [{"run_index": 2, "exit_reason": "success", "failed_checks": [{"name": "metric_correct"}]}] + + head = failure_head(gate_note, failed_runs, 3) + + assert head.startswith(gate_note) + assert head.endswith("Failing runs: run 2/3 FAIL (loop exit: success): metric_correct.") + + +def test_the_head_is_the_gate_note_alone_without_failing_runs() -> None: + assert failure_head("Gate pass@2 failed: 0/2 runs passed.", [], 2) == "Gate pass@2 failed: 0/2 runs passed." + + +def test_a_run_without_recorded_checks_says_so_and_a_kind_without_the_key_says_nothing() -> None: + assert failing_runs_note([{"run_index": 1, "exit_reason": None, "failed_checks": []}], 1) == ( + "run 1/1 FAIL: no failing check recorded" + ) + assert failing_runs_note([{"run_index": 1, "exit_reason": "agent_silent"}], 1) == ( + "run 1/1 FAIL (loop exit: agent_silent)" + ) + + +def test_an_errored_run_is_labelled_with_its_source() -> None: + record = { + "run_index": 1, + "exit_reason": "simulated_user_failed", + "outcome": {"status": "error", "source": "harness", "reason": "simulated user failed"}, + "failed_checks": [], + } + assert failing_runs_note([record], 3) == ( + "run 1/3 ERROR harness (loop exit: simulated_user_failed): no failing check recorded" + ) + + +def test_a_long_value_is_cut_to_one_short_line() -> None: + record = {"run_index": 1, "failed_checks": [{"name": "maql_correct", "actual": "SELECT\n" + "x" * 500}]} + note = failing_runs_note([record], 1) + assert "\n" not in note + assert note.endswith("...]") + assert len(note) < 250 + + +# ── the records ────────────────────────────────────────────────────────────── + + +@dataclass +class _Run: + ok: bool + exit_reason: LoopExit = LoopExit.SUCCESS + conversation_id: str = "c" + tool_call_events: list = field(default_factory=list) + + +def test_every_failing_record_carries_its_exit_reason_and_failed_checks() -> None: + runs = [_Run(True), _Run(False, LoopExit.BUDGET_EXHAUSTED)] + + records = build_failed_runs( + runs, + passed=lambda r: r.ok, + detail=lambda r: {}, + failed_checks=lambda r: [{"name": "metric_correct"}], + ) + + assert len(records) == 1 + assert records[0]["run_index"] == 2 + assert records[0]["exit_reason"] == "budget_exhausted" + assert records[0]["failed_checks"] == [{"name": "metric_correct"}] + + +def test_a_kind_that_names_no_checks_writes_no_failed_checks_key() -> None: + records = build_failed_runs([object()], passed=lambda r: False, detail=lambda r: {}) + assert records[0]["exit_reason"] is None + assert "failed_checks" not in records[0] + + +@dataclass +class _StrRun: + exit_reason: str = "chat_error" + + +def test_a_string_exit_reason_gets_the_same_outcome_as_its_enum(monkeypatch) -> None: + # exit_reason and outcome must agree whether a kind records the member or its value. + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + record = build_failed_runs([_StrRun()], passed=lambda r: False, detail=lambda r: {})[0] + + assert record["exit_reason"] == "chat_error" + assert record["outcome"]["status"] == "error" + assert record["outcome"]["source"] == "backend" + + +@pytest.mark.parametrize( + "exit_reason", + [LoopExit.CHAT_ERROR, "chat_error", LoopExit.SIMULATED_USER_FAILED], + ids=["chat_error_member", "chat_error_value", "simulated_user_failed"], +) +def test_off_an_error_exit_is_recorded_and_rendered_as_the_failure_the_gate_counted(monkeypatch, exit_reason) -> None: + monkeypatch.delenv(ERROR_OUTCOME_ENV_VAR, raising=False) + records = build_failed_runs( + [_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + value = LoopExit(exit_reason).value + assert records[0]["exit_reason"] == value + assert records[0]["outcome"] == {"status": "fail", "source": None, "reason": f"loop exit: {value}"} + assert failing_runs_note(records, 3) == f"run 1/3 FAIL (loop exit: {value}): no failing check recorded" + + +def test_on_an_error_exit_is_recorded_and_rendered_as_an_error(monkeypatch) -> None: + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + records = build_failed_runs([_StrRun()], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: []) + + assert records[0]["outcome"]["status"] == "error" + assert failing_runs_note(records, 3) == "run 1/3 ERROR backend (loop exit: chat_error): no failing check recorded" + + +def test_a_failed_check_record_keeps_short_values_and_caps_long_ones() -> None: + long_actual = {"filters": ["x" * 400]} + records = build_failed_runs( + [_StrRun("budget_exhausted")], + passed=lambda r: False, + detail=lambda r: {"actual": long_actual}, + failed_checks=lambda r: [ + {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]}, + {"name": "filters_correct", "label": "attribute", "expected": [], "actual": long_actual}, + ], + ) + + short, long_check = records[0]["failed_checks"] + assert short == {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]} + assert long_check["expected"] == [] + assert isinstance(long_check["actual"], str) + assert len(long_check["actual"]) == 160 + assert long_check["actual"].endswith("...") + # The run's detail keeps the whole value. + assert records[0]["detail"]["actual"] == long_actual + + +def test_a_fail_run_names_its_loop_exit_in_its_reason() -> None: + assert outcome_from_loop_exit(LoopExit.SUCCESS, passed=False).reason == "loop exit: success" + + +def test_a_metric_layout_failure_names_the_layout_problem_not_two_equal_sets() -> None: + ev = MagicMock( + scored_without_chart=False, + visualization_created=True, + cross_ref_valid=True, + metrics_correct=False, + expected_metric_uris={"metric/revenue"}, + actual_metric_uris={"metric/revenue"}, + metric_layout_problems=["metric/revenue is on the secondary axis"], + dimensions_correct=True, + filters_correct=True, + sorts_correct=True, + viz_type_hard=True, + requires_execution=False, + buckets_correct=None, + answer_correct=None, + format_correct=None, + part_present=None, + ) + run = MagicMock(eval_result=ev) + + assert agentic_visualization._failed_checks(run) == [ + {"name": "metrics_correct", "label": "layout", "actual": ["metric/revenue is on the secondary axis"]} + ] + + +# ── visualization: a partial gate failure names the run that failed ────────── + + +def _viz(filter_by: dict | None = None) -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": filter_by or {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn(filter_by: dict | None = None) -> ChatResult: + return ChatResult.model_validate( + {"createdVisualizations": {"objects": [_viz(filter_by)], "reasoning": ""}, "toolCallEvents": []} + ) + + +_LAST_YEAR = {"date": {"type": "date_filter", "using": "dataset/date", "granularity": "MONTH", "from": -11, "to": 0}} + + +def test_a_partial_visualization_failure_lists_the_failing_run_and_keeps_the_best_run_summary() -> None: + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [_viz_turn(), _viz_turn(_LAST_YEAR), _viz_turn()] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=3, + gate="power", + ) + + message = str(exc_info.value) + lines = message.splitlines() + gate_line = next(line for line in lines if line.startswith("Gate pass^3 failed")) + assert gate_line.startswith("Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure. Failing runs: ") + assert "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual " in gate_line + # Only the failing checks of the failing run, and no passing run. + assert "metrics_correct" not in gate_line + assert "run 1/3" not in gate_line + assert "run 3/3" not in gate_line + # The best-run summary that followed the gate note is still there, unchanged. + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + assert "Strict Check Summary:" in message + assert " Filters Correct : True\n" in message + + (record,) = exc_info.value.failed_runs + assert record["run_index"] == 2 + assert record["exit_reason"] == "success" + assert record["detail"]["exit_reason"] == "success" + assert [c["name"] for c in record["failed_checks"]] == ["filters_correct"] + assert record["failed_checks"][0]["label"] == "date" + assert record["failed_checks"][0]["expected"] == [] + + +# ── metric skill: same head, expected/actual MAQL of the failing run ───────── + + +def _metric_turn(maql: str) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": f'{{"data": {{"maql": "{maql}"}}}}', + } + ], + } + ) + + +def test_a_partial_metric_failure_lists_the_failing_run_with_its_maql() -> None: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = [_metric_turn("SELECT {metric/foo}"), _metric_turn("SELECT {metric/bar}")] + + with ( + patch.object(metric_skill_mod, "ChatClient", return_value=client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + gate="power", + ) + + message = str(exc_info.value) + assert message.startswith( + "Metric skill assertion failed. Gate pass^2 failed: 1/2 runs passed — unstable, not a clean failure. " + "Failing runs: run 2/2 FAIL (loop exit: success): " + 'maql_correct[expected ["SELECT {metric/foo}"], actual SELECT {metric/bar}].' + ) + # The best run's summary follows, as before. + assert "metric_created=True, maql_correct=True. Expected MAQL (candidates): 'SELECT {metric/foo}'." in message + (record,) = exc_info.value.failed_runs + assert record["exit_reason"] == "success" + assert record["failed_checks"] == [ + {"name": "maql_correct", "expected": ["SELECT {metric/foo}"], "actual": "SELECT {metric/bar}"} + ] + + +@pytest.mark.parametrize("module_name", ["visualization", "alert_skill", "metric_skill", "dashboard_skill"]) +def test_each_checked_kind_heads_its_message_through_the_shared_renderer(module_name: str) -> None: + source = inspect.getsource(importlib.import_module(f"gooddata_eval.core.agentic.{module_name}")) + assert "gate_head(gate, runs_verdict, failed_runs)" in source + assert "failed_checks=_failed_checks" in source diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 6a1690ebb..8f13a4093 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -16,8 +16,8 @@ evaluate_agentic_kda_skill, run_agentic_kda_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import ChatResult, LoopExit _EXPECTED = {"Measure": {"type": "metric", "id": "revenue"}} @@ -1296,3 +1296,29 @@ def _capture(_submit, _identity, **kwargs): assert scores["turns"] == 1 assert scores["steps"] == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("gen-ai returned 503"), LoopExit.CHAT_ERROR), + ], +) +def test_run_agentic_kda_skill_names_why_the_chat_call_failed(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.best.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 5efcbb0bb..50a907e4f 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -22,7 +22,8 @@ generate_simulated_response, run_agentic_metric_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.config import ERROR_OUTCOME_ENV_VAR from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent from gooddata_eval.core.timing import TIMERS_ENV_VAR @@ -1098,3 +1099,91 @@ def call(mid: str, created_new: bool) -> ToolCallEvent: ) assert _extract_created_metric_ids([call("new", True), call("old", False)]) == ["new"] + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=3, + ) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + + +@pytest.mark.parametrize("flag", [True, False], ids=["flag_on", "flag_off"]) +def test_an_errored_run_gets_eval_error_only_with_the_flag_on(monkeypatch, flag): + if flag: + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + else: + monkeypatch.delenv(ERROR_OUTCOME_ENV_VAR, raising=False) + mock_client = _client() + mock_client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + } + ), + ] + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + _patched(mock_client), + patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture), + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + run_ids = [MagicMock(name="run-1"), MagicMock(name="run-2")] + ctx.observe.return_value.__enter__.side_effect = run_ids + captured["write_scores"](ctx) + by_run = {id(t): [c.kwargs["name"] for c in ctx.score.call_args_list if c.args[0] is t] for t in run_ids} + + first, second = by_run[id(run_ids[0])], by_run[id(run_ids[1])] + if flag: + assert first == ["eval_error"] + (eval_error,) = [c for c in ctx.score.call_args_list if c.kwargs["name"] == "eval_error"] + assert eval_error.kwargs["value"] == 1 + assert eval_error.kwargs["comment"] == "backend: the chat call failed after its retries" + else: + assert "eval_error" not in first + assert "gate_passed" in first + assert "eval_error" not in second + assert "gate_passed" in second diff --git a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py index 10bae7aab..a74250d7c 100644 --- a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py +++ b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py @@ -345,3 +345,15 @@ def test_the_run_context_forwards_its_window_and_item_input_to_observe(): "item_input": "Show revenue by month", "output": {"passed": True}, } + + +def test_the_run_context_forwards_a_score_comment_only_when_given(): + lf = MagicMock() + ctx = RunTraceContext({}, lf, "client", "item-1", "base", True, {}, _WINDOW, "q") + + ctx.score("tid", name="gate_passed", value=True, data_type="BOOLEAN") + ctx.score("tid", name="eval_error", value=1, data_type="NUMERIC", comment="backend: the chat call failed") + + plain, commented = lf.score_safe.call_args_list + assert plain.kwargs == {"name": "gate_passed", "value": True, "data_type": "BOOLEAN"} + assert commented.kwargs["comment"] == "backend: the chat call failed" diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index 431c3f329..c50d2fddb 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -18,7 +18,7 @@ evaluate_agentic_visualization, run_agentic_visualization, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.evaluators.visualization import ExpectationOptions from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit @@ -822,3 +822,35 @@ def test_a_failed_request_with_a_cut_off_answer_fails_an_item_without_a_chart() assert result.exit_reason is LoopExit.CHAT_ERROR assert result.eval_result.strict_pass is False assert result.eval_result.answer_problems == ["the chat request failed before the answer ended"] + + +_AGENT_CHAT_FAILURES = [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), +] + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_the_opening_request(exc, loop_exit): + client = MagicMock() + client.send_message.side_effect = exc + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit + assert result.eval_result.visualization_created is False + # Like a chat error, the loop does not run after the opening failure. + assert client.send_message.call_count == 1 + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkeypatch, exc, loop_exit): + monkeypatch.setattr( + "gooddata_eval.core.agentic.visualization.generate_simulated_response", lambda *a, **k: "the revenue one" + ) + client = MagicMock() + client.send_message.side_effect = [_chat_clarification(), exc] + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 38b04ebf3..516337229 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -13,6 +13,7 @@ evaluate_agentic_what_if, run_agentic_what_if, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.what_if" @@ -499,3 +500,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 2a53f5cdc..5b53e8b6b 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -1,7 +1,7 @@ # (C) 2026 GoodData Corporation import io from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timezone +from datetime import date, datetime, timezone from unittest.mock import MagicMock import httpx @@ -9,13 +9,22 @@ import pytest from gooddata_eval.cli import main as cli_main from gooddata_eval.cli.main import _parse_model_arg -from gooddata_eval.core.config import JUDGE_MODEL_ENV_VAR, RunConfig, judge_model +from gooddata_eval.core.config import ( + ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR, + ERROR_OUTCOME_ENV_VAR, + JUDGE_MODEL_ENV_VAR, + RunConfig, + accept_context_date_filter, + error_outcome_enabled, + judge_model, +) from gooddata_eval.core.connection import ( ConnectionError_, # noqa: F401 - used in test_cli_operational_error_exits_nonzero ) from gooddata_eval.core.dataset.langfuse_source import _item_from_raw -from gooddata_eval.core.models import DatasetItem +from gooddata_eval.core.models import CreatedVisualization, DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport +from gooddata_eval.core.scoring import FilterScores, check_filters from gooddata_eval.core.timing import TIMERS_ENV_VAR, timers_enabled from gooddata_eval.core.workspace import ActiveLlmProvider, ResolvedModel from rich.console import Console @@ -1197,3 +1206,61 @@ def test_routing_backend_closes_every_client() -> None: for client in (chat, summary, takeaway): client.close.assert_called_once_with() + + +def test_error_outcome_flag_is_off_by_default_and_publishes_the_env_var(monkeypatch): + base = ["run", "--host", "h", "--workspace", "w", "--dataset", "d"] + assert cli_main.parse_args(base).error_outcome is False + assert cli_main.parse_args([*base, "--error-outcome"]).error_outcome is True + + # Set, not deleted, so monkeypatch restores what the flag writes. + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "0") + cli_main._apply_error_outcome_flag(False) + assert error_outcome_enabled() is False + cli_main._apply_error_outcome_flag(True) + assert error_outcome_enabled() is True + + +def test_accept_context_date_filter_flag_is_off_by_default_and_publishes_the_env_var(monkeypatch): + base = ["run", "--host", "h", "--workspace", "w", "--dataset", "d"] + assert cli_main.parse_args(base).accept_context_date_filter is False + assert cli_main.parse_args([*base, "--accept-context-date-filter"]).accept_context_date_filter is True + + # Set, not deleted, so monkeypatch restores what the flag writes. + monkeypatch.setenv(ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR, "0") + cli_main._apply_accept_context_date_filter_flag(False) + assert accept_context_date_filter() is False + cli_main._apply_accept_context_date_filter_flag(True) + assert accept_context_date_filter() is True + + +def test_accept_context_date_filter_flag_never_clears_an_exported_env_var(monkeypatch): + monkeypatch.setenv(ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR, "1") + cli_main._apply_accept_context_date_filter_flag(False) + assert accept_context_date_filter() is True + + +def test_accept_context_date_filter_flag_reaches_check_filters(monkeypatch): + def viz(filter_by: dict) -> CreatedVisualization: + return CreatedVisualization.model_validate( + { + "id": "v", + "type": "headline_chart", + "query": {"fields": {"m": {"using": "metric/m"}}, "filter_by": filter_by}, + "metrics": ["m"], + } + ) + + inherited = {"type": "date_filter", "using": "dataset/d", "granularity": "MONTH", "from": -11, "to": 0} + user_context = { + "view": {"dashboard": {"filters": [{"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}]}} + } + + def score() -> FilterScores: + return check_filters(viz({}), viz({"f": inherited}), date(2026, 11, 12), user_context=user_context) + + monkeypatch.setenv(ACCEPT_CONTEXT_DATE_FILTER_ENV_VAR, "0") + assert score().date_ok is False # default: strict date comparison + cli_main._apply_accept_context_date_filter_flag(True) + assert score().date_ok is True + assert score().date_inherited is True diff --git a/packages/gooddata-eval/tests/test_context_date_filter.py b/packages/gooddata-eval/tests/test_context_date_filter.py new file mode 100644 index 000000000..7f9df3753 --- /dev/null +++ b/packages/gooddata-eval/tests/test_context_date_filter.py @@ -0,0 +1,400 @@ +# (C) 2026 GoodData Corporation +"""A date filter inherited from the dashboard in the item's `user_context`. + +The rule is opt-in (`GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER`, or `accept_context_date_filter=`). +With it off, every case here must score exactly as the strict comparison does. +""" + +import json +from datetime import date +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic.visualization import ( + VisualizationAssertionError, + evaluate_agentic_visualization, + run_agentic_visualization, +) +from gooddata_eval.core.evaluators import get_evaluator +from gooddata_eval.core.evaluators.visualization import _evaluate_against_candidates, evaluation_result_detail +from gooddata_eval.core.models import ChatResult, CreatedVisualization, DatasetItem +from gooddata_eval.core.scoring import check_filters, dashboard_date_filters + +_ENV = "GD_EVAL_ACCEPT_CONTEXT_DATE_FILTER" +_TODAY = date(2026, 11, 12) + +# The real item from the ticket: globalmart_visualization_charts_ctx_2, run 40229016049. +_USER_CONTEXT = { + "view": { + "dashboard": { + "id": "dashboard_033", + "title": "Order Lines & Margin", + "filters": [{"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}], + } + } +} +_INHERITED = { + "type": "date_filter", + "using": "dataset/transaction_date", + "granularity": "MONTH", + "from": -11, + "to": 0, +} + + +def _headline(filter_by: dict | None = None, metric: str = "metric/gross_margin_percent") -> dict: + return { + "id": "v", + "type": "headline_chart", + "query": {"fields": {"m": {"using": metric}}, "filter_by": filter_by or {}}, + "metrics": ["m"], + } + + +def _viz(filter_by: dict | None = None) -> CreatedVisualization: + return CreatedVisualization.model_validate(_headline(filter_by)) + + +def _date(frm: int, to: int, granularity: str = "MONTH", using: str = "dataset/transaction_date") -> dict: + return {"type": "date_filter", "using": using, "granularity": granularity, "from": frm, "to": to} + + +# ── the five acceptance-criteria cases (flag on) ───────────────────────────── + + +def test_inherited_dashboard_range_passes_and_is_marked_inherited(): + scores = check_filters( + _viz(), + _viz({"f_transaction_date": _INHERITED}), + _TODAY, + user_context=_USER_CONTEXT, + accept_context_date_filter=True, + ) + assert scores.date_ok is True + assert scores.date_inherited is True + assert scores.date_inherited_dataset == "dataset/transaction_date" + assert scores.all_ok is True + + +def test_no_actual_date_filter_still_passes_without_being_inherited(): + scores = check_filters(_viz(), _viz(), _TODAY, user_context=_USER_CONTEXT, accept_context_date_filter=True) + assert scores.date_ok is True + assert scores.date_inherited is False + + +def test_a_range_other_than_the_dashboards_fails(): + scores = check_filters( + _viz(), _viz({"f": _date(-5, 0)}), _TODAY, user_context=_USER_CONTEXT, accept_context_date_filter=True + ) + assert scores.date_ok is False + assert scores.date_inherited is False + + +@pytest.mark.parametrize( + ("expected_filter", "actual_filter", "date_ok"), + [ + # The question names a period: the dashboard's range is not good enough... + (_date(-1, -1, "YEAR"), _INHERITED, False), + # ...and the expected one still matches exactly, as before. + (_date(-1, -1, "YEAR"), _date(-1, -1, "YEAR"), True), + # Even an expected filter equal to the dashboard's is a strict match, not "inherited". + (_INHERITED, _INHERITED, True), + ], +) +def test_an_expected_date_filter_keeps_the_strict_comparison(expected_filter, actual_filter, date_ok): + expected, actual = _viz({"f": expected_filter}), _viz({"f": actual_filter}) + scores = check_filters(expected, actual, _TODAY, user_context=_USER_CONTEXT, accept_context_date_filter=True) + assert scores.date_ok is date_ok + assert scores.date_inherited is False + assert scores == check_filters(expected, actual, _TODAY) + + +@pytest.mark.parametrize("user_context", [None, {}, {"view": {}}, {"view": {"dashboard": {"id": "d"}}}]) +def test_without_a_dashboard_date_filter_in_context_the_comparison_is_strict(user_context): + scores = check_filters( + _viz(), _viz({"f": _INHERITED}), _TODAY, user_context=user_context, accept_context_date_filter=True + ) + assert scores.date_ok is False + assert scores.date_inherited is False + + +# ── further edges of the rule ──────────────────────────────────────────────── + + +def test_any_date_dataset_is_accepted_and_recorded(): + scores = check_filters( + _viz(), + _viz({"f": _date(-11, 0, using="dataset/order_date")}), + _TODAY, + user_context=_USER_CONTEXT, + accept_context_date_filter=True, + ) + assert scores.date_ok is True + assert scores.date_inherited_dataset == "dataset/order_date" + + +@pytest.mark.parametrize(("actual_dataset", "date_ok"), [("dataset/order_date", True), ("dataset/ship_date", False)]) +def test_a_dashboard_filter_naming_a_dataset_accepts_only_that_one(actual_dataset, date_ok): + context = { + "view": { + "dashboard": { + "filters": [ + {"type": "date_filter", "using": "dataset/order_date", "granularity": "MONTH", "from": -11, "to": 0} + ] + } + } + } + scores = check_filters( + _viz(), + _viz({"f": _date(-11, 0, using=actual_dataset)}), + _TODAY, + user_context=context, + accept_context_date_filter=True, + ) + assert scores.date_ok is date_ok + assert scores.date_inherited is date_ok + + +def test_the_same_span_written_absolutely_is_accepted(): + # MONTH -11..0 on 2026-11-12 is 2025-12-01..2026-11-30. + absolute = {"type": "date_filter", "using": "dataset/transaction_date", "from": "2025-12-01", "to": "2026-11-30"} + scores = check_filters( + _viz(), _viz({"f": absolute}), _TODAY, user_context=_USER_CONTEXT, accept_context_date_filter=True + ) + assert scores.date_ok is True + assert scores.date_inherited is True + + +def test_two_actual_date_filters_fail_even_if_one_is_the_dashboards(): + actual = _viz({"a": _INHERITED, "b": _date(-1, -1, "YEAR", using="dataset/order_date")}) + scores = check_filters(_viz(), actual, _TODAY, user_context=_USER_CONTEXT, accept_context_date_filter=True) + assert scores.date_ok is False + + +def test_any_of_several_dashboard_date_filters_matches(): + context = { + "view": { + "dashboard": { + "filters": [ + {"type": "attribute_filter", "using": "label/region"}, + {"type": "date_filter", "granularity": "YEAR", "from": 0, "to": 0}, + {"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}, + ] + } + } + } + assert len(dashboard_date_filters(context)) == 2 + scores = check_filters( + _viz(), _viz({"f": _INHERITED}), _TODAY, user_context=context, accept_context_date_filter=True + ) + assert scores.date_ok is True + + +def test_ranking_and_attribute_filters_are_not_relaxed(): + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + scores = check_filters( + _viz(), + _viz({"f": _INHERITED, "r": attribute}), + _TODAY, + user_context=_USER_CONTEXT, + accept_context_date_filter=True, + ) + assert scores.date_ok is True + assert scores.attribute_ok is False + assert scores.all_ok is False + + +@pytest.mark.parametrize("malformed", ["text", {"view": "x"}, {"view": {"dashboard": {"filters": "x"}}}]) +def test_a_malformed_context_is_ignored_not_raised(malformed): + assert dashboard_date_filters(malformed) == [] + + +# ── the flag ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("value", [None, "", "0", "false", "no", "off"]) +def test_flag_off_scores_strictly(monkeypatch, value): + if value is not None: + monkeypatch.setenv(_ENV, value) + expected, actual = _viz(), _viz({"f": _INHERITED}) + scores = check_filters(expected, actual, _TODAY, user_context=_USER_CONTEXT) + assert scores.date_ok is False + assert scores == check_filters(expected, actual, _TODAY) + + +@pytest.mark.parametrize("value", ["1", "true", "yes", "on"]) +def test_flag_on_from_the_environment(monkeypatch, value): + monkeypatch.setenv(_ENV, value) + scores = check_filters(_viz(), _viz({"f": _INHERITED}), _TODAY, user_context=_USER_CONTEXT) + assert scores.date_ok is True + assert scores.date_inherited is True + + +def test_the_keyword_overrides_the_environment(monkeypatch): + monkeypatch.setenv(_ENV, "1") + scores = check_filters( + _viz(), _viz({"f": _INHERITED}), _TODAY, user_context=_USER_CONTEXT, accept_context_date_filter=False + ) + assert scores.date_ok is False + + +@pytest.mark.parametrize( + "actual_filter_by", + [{}, {"f": _INHERITED}, {"f": _date(-5, 0)}, {"a": _INHERITED, "b": _date(0, 0, "YEAR")}], +) +@pytest.mark.parametrize("expected_filter_by", [{}, {"f": _date(-1, -1, "YEAR")}]) +def test_flag_off_detail_is_byte_identical_with_and_without_user_context(expected_filter_by, actual_filter_by): + expected, actual = [_viz(expected_filter_by)], _viz(actual_filter_by) + before, _ = _evaluate_against_candidates(expected, actual, True, _TODAY) + after, _ = _evaluate_against_candidates(expected, actual, True, _TODAY, user_context=_USER_CONTEXT) + assert after == before + dump = json.dumps(evaluation_result_detail(before), sort_keys=True, default=sorted) + assert json.dumps(evaluation_result_detail(after), sort_keys=True, default=sorted) == dump + assert "filter_date_context" not in dump + + +# ── the real payload from the ticket, through both evaluation paths ────────── + + +def _ticket_item() -> DatasetItem: + return DatasetItem( + id="globalmart_visualization_charts_ctx_2", + dataset_name="globalmart_visualization_charts_ctx", + test_kind="visualization", + question="Show Gross Margin Percent as a headline.", + expected_output={"visualization": _headline()}, + user_context=_USER_CONTEXT, + ) + + +def _ticket_chat(metric: str = "metric/gross_margin_percent") -> ChatResult: + return ChatResult.model_validate( + { + "createdVisualizations": { + "objects": [_headline({"f_transaction_date": _INHERITED}, metric=metric)], + "reasoning": "", + }, + "toolCallEvents": [], + } + ) + + +def test_ticket_item_single_shot_fails_with_the_flag_off(): + result = get_evaluator("visualization").evaluate(_ticket_item(), _ticket_chat()) + assert result.passed is False + assert result.detail["filter_date_score"] is False + assert "filter_date_context" not in result.detail + + +def test_ticket_item_single_shot_passes_with_the_flag_on(monkeypatch): + monkeypatch.setenv(_ENV, "1") + result = get_evaluator("visualization").evaluate(_ticket_item(), _ticket_chat()) + assert result.passed is True + assert result.detail["filter_date_score"] is True + assert result.detail["filter_date_context"] == {"inherited": True, "dataset": "dataset/transaction_date"} + + +def _agentic_client(chat: ChatResult) -> MagicMock: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = chat + return client + + +@pytest.mark.parametrize(("flag", "passes"), [(None, False), ("1", True)]) +def test_ticket_item_agentic_run(monkeypatch, flag, passes): + if flag is not None: + monkeypatch.setenv(_ENV, flag) + with patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(_ticket_chat())): + summary = run_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=_USER_CONTEXT, + ) + ev = summary.best.eval_result + assert ev.strict_pass is passes + assert ev.filter_date_score is passes + assert ev.filter_date_inherited is passes + + +def test_strict_check_summary_shows_the_inherited_date(monkeypatch): + monkeypatch.setenv(_ENV, "1") + # A wrong metric keeps the item failing, so the summary is printed. + chat = _ticket_chat(metric="metric/revenue") + with ( + patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(chat)), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=_USER_CONTEXT, + ) + message = str(exc_info.value) + assert " date : True (inherited from dashboard context)\n" in message + assert "Metrics Correct : False" in message + + +def test_strict_check_summary_is_unchanged_with_the_flag_off(): + with ( + patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(_ticket_chat())), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=_USER_CONTEXT, + ) + message = str(exc_info.value) + assert " date : False\n" in message + assert "inherited" not in message + + +def test_failed_checks_leave_out_an_accepted_inherited_date(monkeypatch): + monkeypatch.setenv(_ENV, "1") + # An extra attribute filter fails the filters; the inherited date filter must not be blamed. + chat = ChatResult.model_validate( + { + "createdVisualizations": { + "objects": [ + _headline( + { + "f_transaction_date": _INHERITED, + "a": {"type": "attribute_filter", "using": "label/region", "state": {"include": ["x"]}}, + } + ) + ], + "reasoning": "", + }, + "toolCallEvents": [], + } + ) + with patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(chat)): + summary = run_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=_USER_CONTEXT, + ) + run = summary.best + assert run.eval_result.filter_date_inherited is True + assert run.eval_result.filters_correct is False + labels = [c.get("label") for c in agentic_visualization._failed_checks(run) if c.get("name") == "filters_correct"] + assert "date" not in labels + assert "attribute" in labels diff --git a/packages/gooddata-eval/tests/test_eval_error_scores.py b/packages/gooddata-eval/tests/test_eval_error_scores.py new file mode 100644 index 000000000..7d2cb0985 --- /dev/null +++ b/packages/gooddata-eval/tests/test_eval_error_scores.py @@ -0,0 +1,154 @@ +# (C) 2026 GoodData Corporation +"""Every agentic kind that gates through outcomes writes its Langfuse scores the same way. + +With GD_EVAL_ERROR_OUTCOME on, a run left out of the gate gets ``eval_error`` and nothing that +reads as a verdict; a scored run keeps its scores. With the flag off, every run is scored as +before and none gets ``eval_error``. The visualization and metric kinds have their own tests. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import alert_skill, anomaly_detection, dashboard_skill, kda_skill, what_if +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.config import ERROR_OUTCOME_ENV_VAR +from gooddata_eval.core.models import ChatResult + + +@dataclass(frozen=True) +class _Kind: + module: Any + evaluate: Callable[..., Any] + expected_output: dict + simulated_user: str | None + + +_KINDS = { + "alert_skill": _Kind( + alert_skill, + alert_skill.evaluate_agentic_alert_skill, + {"operator": "GREATER_THAN", "threshold": 100}, + "generate_simulated_alert_response", + ), + "kda_skill": _Kind( + kda_skill, + kda_skill.evaluate_agentic_kda_skill, + {"Measure": {"type": "metric", "id": "revenue"}}, + "generate_simulated_kda_response", + ), + "dashboard_skill": _Kind( + dashboard_skill, + dashboard_skill.evaluate_agentic_dashboard_skill, + { + "type": "dashboard", + "saved_dashboard_id": None, + "visualizations": [{"id": None, "title": "Revenue"}], + "date_range": None, + "min_new_visualizations": 1, + }, + None, + ), + "anomaly_detection": _Kind( + anomaly_detection, + anomaly_detection.evaluate_agentic_anomaly_detection, + {"metric": "metric/spend", "granularity": "MONTH"}, + "generate_simulated_anomaly_response", + ), + "what_if": _Kind( + what_if, + what_if.evaluate_agentic_what_if, + {"metric_id": "revenue"}, + "generate_simulated_what_if_response", + ), +} + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + def run_name(self, run_idx: int) -> str: + return f"run{run_idx}" + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(kind: _Kind) -> _ScoreCtx: + """Run 1's chat call fails after its retries; run 2 answers without doing the task.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate({"textResponse": "I cannot help with that."}), + ] + captured: dict[str, Any] = {} + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + captured["write_scores"] = kwargs["write_scores"] + + with ExitStack() as stack: + stack.enter_context(patch.object(kind.module, "ChatClient", return_value=client)) + stack.enter_context(patch.object(kind.module, "submit_trace_scoring", _capture)) + if kind.simulated_user is not None: + stack.enter_context(patch.object(kind.module, kind.simulated_user, return_value="please go on")) + try: + kind.evaluate( + "http://host/api/v1/actions/workspaces/ws1/ai", + "tok", + "ws1", + "Do the thing", + kind.expected_output, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + except AssertionError: + pass # the scored run fails; its scores are written before the raise + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_on_the_errored_run_gets_eval_error_only_and_the_scored_run_keeps_its_scores(monkeypatch, name) -> None: + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + ctx = _scores(_KINDS[name]) + + assert ctx.scores[0] == [("eval_error", 1, "backend: the chat call failed after its retries")] + scored = ctx.names(1) + assert "eval_error" not in scored + assert {"pass_at_k", "pass_power_k", "gate_passed"} <= set(scored) + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_off_every_run_is_scored_and_none_gets_eval_error(monkeypatch, name) -> None: + monkeypatch.delenv(ERROR_OUTCOME_ENV_VAR, raising=False) + ctx = _scores(_KINDS[name]) + + for run_idx in (0, 1): + names = ctx.names(run_idx) + assert "eval_error" not in names + assert {"pass_at_k", "pass_power_k", "gate_passed"} <= set(names) diff --git a/packages/gooddata-eval/tests/test_langfuse_sink.py b/packages/gooddata-eval/tests/test_langfuse_sink.py index 986f4ec4a..f4ea9351a 100644 --- a/packages/gooddata-eval/tests/test_langfuse_sink.py +++ b/packages/gooddata-eval/tests/test_langfuse_sink.py @@ -7,6 +7,7 @@ import httpx import pytest +from gooddata_eval.core.config import ERROR_OUTCOME_ENV_VAR from gooddata_eval.core.langfuse.sink import LangfuseSink, compute_scores from gooddata_eval.core.runner import ItemReport @@ -277,3 +278,66 @@ def test_langfuse_sink_appends_custom_tags_after_the_automatic_ones(monkeypatch) sink.log_item(_passing_report(), dataset_item_id="item-1") assert _tags_from(requests) == ["gd-eval", "visualization", "OPENAI", "acme", "nightly"] + + +def _scores_from(requests: list[httpx.Request]) -> list[dict]: + return [json.loads(r.content) for r in requests if r.url.path == "/api/public/scores"] + + +def test_on_an_errored_item_gets_its_error_eval_error_and_the_latency_score(monkeypatch): + # A pass_at_k of 0 would read as an agent failure. + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + requests: list[httpx.Request] = [] + sink = _make_sink(monkeypatch, _known_item_handler(requests)) + report = _passing_report() + report.pass_at_k = False + report.error = "RuntimeError: network down" + + sink.log_item(report, dataset_item_id="item-1") + + scores = _scores_from(requests) + assert [s["name"] for s in scores] == ["latency_s", "eval_error"] + assert scores[1]["value"] == 1 + assert scores[1]["comment"] == "RuntimeError: network down" + assert all("comment" not in s or s["name"] == "eval_error" for s in scores) + attrs = {a["key"]: a["value"] for a in _span_from(requests)["attributes"]} + assert attrs["langfuse.experiment.item.metadata.error"]["stringValue"] == "RuntimeError: network down" + + +def test_off_an_errored_item_passed_to_the_sink_gets_every_score_as_before(monkeypatch): + monkeypatch.delenv(ERROR_OUTCOME_ENV_VAR, raising=False) + requests: list[httpx.Request] = [] + sink = _make_sink(monkeypatch, _known_item_handler(requests)) + report = _passing_report() + report.pass_at_k = False + report.error = "RuntimeError: network down" + + sink.log_item(report, dataset_item_id="item-1") + + assert [s["name"] for s in _scores_from(requests)] == ["pass_at_k", "quality_score", "value_score", "latency_s"] + + +def test_a_report_without_an_error_has_no_error_metadata(monkeypatch): + requests: list[httpx.Request] = [] + sink = _make_sink(monkeypatch, _known_item_handler(requests)) + + sink.log_item(_passing_report(), dataset_item_id="item-1") + + attrs = {a["key"]: a["value"] for a in _span_from(requests)["attributes"]} + assert "langfuse.experiment.item.metadata.error" not in attrs + + +def test_on_an_item_that_errored_before_any_run_gets_no_latency_score(monkeypatch): + # runs == 0 makes avg_latency_s a 0.0 that measured nothing. + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + requests: list[httpx.Request] = [] + sink = _make_sink(monkeypatch, _known_item_handler(requests)) + report = _passing_report() + report.pass_at_k = False + report.runs = 0 + report.latency_s = 0.0 + report.error = "RuntimeError: network down" + + sink.log_item(report, dataset_item_id="item-1") + + assert [s["name"] for s in _scores_from(requests)] == ["eval_error"] diff --git a/packages/gooddata-eval/tests/test_outcome.py b/packages/gooddata-eval/tests/test_outcome.py new file mode 100644 index 000000000..2bbc1e409 --- /dev/null +++ b/packages/gooddata-eval/tests/test_outcome.py @@ -0,0 +1,825 @@ +# (C) 2026 GoodData Corporation +"""PASS / FAIL / ERROR per run, the gate over scored runs, and the GD_EVAL_ERROR_OUTCOME switch. + +Every verdict-changing test runs twice: with the flag on (the new behaviour) and off, where +the verdict and the existing message content must be exactly what they were before. +""" + +from collections import defaultdict +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import gooddata_eval +import httpx +import pytest +from gooddata_eval.cli.agentic_runner import run_agentic_items +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.agentic._outcome import chat_failure_exit, resolve_runs +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.config import ERROR_OUTCOME_ENV_VAR, error_outcome_enabled +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + CreatedVisualization, + DatasetItem, + LoopExit, +) +from gooddata_eval.core.outcome import ( + ErrorSource, + EvalHarnessError, + Outcome, + OutcomeStatus, + excluded_runs_note, + gate_outcomes, + outcome_from_loop_exit, +) +from gooddata_eval.core.reporting.json_report import build_json_report +from gooddata_eval.core.runner import run_items + +PASS = Outcome.passed() +FAIL = Outcome.failed() +HARNESS = Outcome.error(ErrorSource.HARNESS, "the simulated user failed to reply") +BACKEND = Outcome.error(ErrorSource.BACKEND, "the chat call failed after its retries") + + +@pytest.fixture +def flag_on(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, "1") + + +# ── the flag ───────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize(("value", "on"), [(None, False), ("", False), ("0", False), ("off", False), ("1", True)]) +def test_the_flag_is_off_unless_set_to_something_truthy(monkeypatch, value, on) -> None: + if value is None: + monkeypatch.delenv(ERROR_OUTCOME_ENV_VAR, raising=False) + else: + monkeypatch.setenv(ERROR_OUTCOME_ENV_VAR, value) + assert error_outcome_enabled() is on + + +# ── the public names ───────────────────────────────────────────────────────── + + +def test_the_error_is_public_and_not_an_assertion_error() -> None: + # Not an AssertionError, so a runner can tell it apart from an agent failure. + assert not issubclass(EvalHarnessError, AssertionError) + assert not issubclass(EvalHarnessError, AgenticAssertionError) + for name in ("EvalHarnessError", "Outcome", "OutcomeStatus", "ErrorSource", "outcome_from_loop_exit"): + assert name in gooddata_eval.__all__ + assert gooddata_eval.EvalHarnessError is EvalHarnessError + + +def test_existing_assertion_classes_keep_their_base() -> None: + assert issubclass(VisualizationAssertionError, AgenticAssertionError) + assert issubclass(AgenticAssertionError, AssertionError) + + +# ── LoopExit → Outcome ─────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("exit_reason", "status", "source"), + [ + (LoopExit.SUCCESS, OutcomeStatus.FAIL, None), + (LoopExit.BUDGET_EXHAUSTED, OutcomeStatus.FAIL, None), + (LoopExit.AGENT_SILENT, OutcomeStatus.FAIL, None), + (LoopExit.SIMULATED_USER_FAILED, OutcomeStatus.ERROR, ErrorSource.HARNESS), + (LoopExit.CHAT_ERROR, OutcomeStatus.ERROR, ErrorSource.BACKEND), + (LoopExit.NOT_RUN, OutcomeStatus.ERROR, ErrorSource.HARNESS), + (None, OutcomeStatus.FAIL, None), + ], +) +def test_each_loop_exit_maps_to_its_outcome_when_the_run_failed(exit_reason, status, source) -> None: + outcome = outcome_from_loop_exit(exit_reason, passed=False) + assert outcome.status is status + assert outcome.source is source + assert outcome.is_scored is (status is not OutcomeStatus.ERROR) + + +def test_every_loop_exit_is_mapped() -> None: + # A new LoopExit member must be classified on purpose: this lists what is scored today. + scored = {e for e in LoopExit if outcome_from_loop_exit(e, passed=False).is_scored} + assert scored == { + LoopExit.SUCCESS, + LoopExit.BUDGET_EXHAUSTED, + LoopExit.AGENT_SILENT, + LoopExit.AGENT_TIMEOUT, + LoopExit.TURN_INCOMPLETE, + } + + +@pytest.mark.parametrize("exit_reason", list(LoopExit)) +def test_a_run_that_passed_is_pass_whatever_its_exit(exit_reason) -> None: + assert outcome_from_loop_exit(exit_reason, passed=True) == PASS + + +def test_the_mapping_accepts_the_string_value() -> None: + assert outcome_from_loop_exit("chat_error", passed=False).source is ErrorSource.BACKEND + + +def test_an_outcome_record_is_json_ready() -> None: + assert BACKEND.as_record() == { + "status": "error", + "source": "backend", + "reason": "the chat call failed after its retries", + } + assert PASS.as_record() == {"status": "pass", "source": None, "reason": None} + + +# ── the gate over outcomes ─────────────────────────────────────────────────── + + +@pytest.mark.parametrize("gate", ["any", "power"]) +def test_error_pass_pass_passes_with_one_run_excluded(gate) -> None: + verdict = gate_outcomes([HARNESS, PASS, PASS], gate) + + assert verdict.outcome == PASS + assert (verdict.runs_scored, verdict.runs_passed, verdict.runs_total) == (2, 2, 3) + assert verdict.pass_power_k is True + note = excluded_runs_note(verdict) + assert note.startswith("1 run excluded (harness error)") + assert "run 1/3 the simulated user failed to reply" in note + + +@pytest.mark.parametrize("gate", ["any", "power"]) +def test_error_error_error_is_an_error(gate) -> None: + verdict = gate_outcomes([HARNESS, BACKEND, HARNESS], gate) + + assert verdict.outcome.status is OutcomeStatus.ERROR + assert verdict.outcome.source is ErrorSource.HARNESS + assert verdict.runs_scored == 0 + assert verdict.pass_at_k is False + assert verdict.pass_power_k is False + assert excluded_runs_note(verdict).startswith("3 runs excluded (1 backend error, 2 harness error)") + + +def test_the_gate_still_decides_over_the_scored_runs() -> None: + assert gate_outcomes([BACKEND, PASS, FAIL], "power").outcome == FAIL + assert gate_outcomes([BACKEND, PASS, FAIL], "any").outcome == PASS + assert gate_outcomes([FAIL, FAIL], "any").outcome == FAIL + assert excluded_runs_note(gate_outcomes([PASS, FAIL], "any")) == "" + + +# ── resolve_runs: off is today's arithmetic, whatever the runs say ─────────── + + +class _Run: + def __init__(self, ok: bool, exit_reason: LoopExit) -> None: + self.ok = ok + self.exit_reason = exit_reason + + +_MIXED = [ + _Run(False, LoopExit.SIMULATED_USER_FAILED), + _Run(True, LoopExit.SUCCESS), + _Run(True, LoopExit.SUCCESS), +] + + +def test_off_the_counts_and_verdict_inputs_are_the_kinds_own() -> None: + runs = resolve_runs(_MIXED, passed=lambda r: r.ok, gate="power", pass_at_k=True, pass_power_k=False) + + assert (runs.pass_at_k, runs.pass_power_k) == (True, False) + assert (runs.runs_passed, runs.runs_effective, runs.runs_total) == (2, 3, 3) + assert runs.errored is False + assert runs.excluded_note == "" + assert runs.detail() == {} + assert runs.outcome_record is None + assert runs.exception_cls(VisualizationAssertionError) is VisualizationAssertionError + + +def test_on_the_errored_run_leaves_k(flag_on) -> None: + runs = resolve_runs(_MIXED, passed=lambda r: r.ok, gate="power", pass_at_k=True, pass_power_k=False) + + assert (runs.pass_at_k, runs.pass_power_k) == (True, True) + # runs_effective stays the runs executed; runs_scored is what the gate counted. + assert (runs.runs_passed, runs.runs_scored, runs.runs_effective, runs.runs_total) == (2, 2, 3, 3) + assert runs.detail()["excluded_runs"][0]["exit_reason"] == "simulated_user_failed" + assert runs.excluded_indexes == {1} + assert runs.excluded_outcome(1) == HARNESS + assert runs.excluded_outcome(2) is None + + +# ── end to end through the visualization kind ──────────────────────────────── + + +def _viz() -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn() -> ChatResult: + return ChatResult.model_validate({"createdVisualizations": {"objects": [_viz()], "reasoning": ""}}) + + +def _question_turn() -> ChatResult: + return ChatResult.model_validate({"textResponse": "Which metric?"}) + + +def _evaluate(turns: list[Any], captured: dict[str, Any] | None = None) -> object: + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, len(turns) + 1)] + client.send_message.side_effect = turns + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + if captured is not None: + captured["write_scores"] = kwargs["write_scores"] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + patch.object(agentic_visualization, "generate_simulated_response", side_effect=RuntimeError("openai 401")), + patch.object(agentic_visualization, "submit_trace_scoring", _capture), + ): + return evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=len(turns), + gate="power", + langfuse=MagicMock() if captured is not None else None, + dataset_item_id="item-1" if captured is not None else None, + ) + + +def test_off_a_simulated_user_failure_fails_the_item_exactly_as_before() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _viz_turn(), _viz_turn()]) + + message = str(exc_info.value) + lines = message.splitlines() + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + assert lines[2].startswith(gate_failure_note("power", 2, 3)) + assert "excluded" not in message + assert "Strict Check Summary:" in message + assert exc_info.value.runs_passed == 2 + assert exc_info.value.runs_effective == 3 + assert "excluded_runs" not in exc_info.value.detail + assert not hasattr(exc_info.value, "outcome") + # The gate counted the run as a failure, so the message and the record say FAIL too. + assert "run 1/3 FAIL (loop exit: simulated_user_failed)" in message + assert "ERROR" not in message + (record,) = exc_info.value.failed_runs + assert record["outcome"]["status"] == "fail" + + +def test_on_error_pass_pass_passes_with_the_excluded_run_recorded(flag_on) -> None: + outcome = _evaluate([_question_turn(), _viz_turn(), _viz_turn()]) + + assert outcome.runs_passed == 2 + assert outcome.runs_effective == 3 + assert outcome.runs_scored == 2 + (excluded,) = outcome.detail["excluded_runs"] + assert excluded["run_index"] == 1 + assert excluded["outcome"]["status"] == "error" + assert excluded["outcome"]["source"] == "harness" + (record,) = outcome.failed_runs + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"]["status"] == "error" + + +def test_off_error_error_error_is_a_failure_as_before() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _question_turn(), _question_turn()]) + + assert gate_failure_note("power", 0, 3) in str(exc_info.value) + + +def test_on_error_error_error_raises_the_harness_error(flag_on) -> None: + with pytest.raises(EvalHarnessError) as exc_info: + _evaluate([_question_turn(), _question_turn(), _question_turn()]) + + exc = exc_info.value + assert not isinstance(exc, AssertionError) + message = str(exc) + assert "No scored run, item errored: 3 runs excluded (harness error)" in message + # No assertion about the agent was made, so the message does not claim one failed. + assert message.splitlines()[1] == "Agentic Visualization item could not be scored. (Critical Mode)" + assert "Assertion Failed" not in message + # Each excluded run is named once, in the excluded note, not again under Failing runs. + assert "run 1/3 the simulated user failed to reply" in message + assert "Failing runs" not in message + assert exc.runs_effective == 3 + assert exc.runs_scored == 0 + assert exc.outcome["status"] == "error" + assert exc.outcome["source"] == "harness" + assert exc.runs_passed == 0 + assert len(exc.failed_runs) == 3 + assert all(r["exit_reason"] == "simulated_user_failed" for r in exc.failed_runs) + + +# ── the agentic runner reports the harness error as an error, with its runs ── + + +def _alert_item() -> DatasetItem: + return DatasetItem( + id="item-1", + dataset_name="d", + test_kind="agentic_alert_skill", + question="Alert me when revenue drops below 100.", + expected_output={"operator": "LESS_THAN", "threshold": 100}, + ) + + +def test_the_agentic_runner_records_a_harness_error_as_an_error_with_its_runs() -> None: + exc = EvalHarnessError("No scored run") + exc.detail = {"excluded_runs": [{"run_index": 1}]} + exc.runs_passed = 0 + exc.runs_effective = 0 + exc.failed_runs = [{"run_index": 1, "exit_reason": "chat_error"}] + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items([_alert_item()], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01") + + item = report.items[0] + assert item.error == "EvalHarnessError: No scored run" + assert item.passed is False + assert item.failed_runs == [{"run_index": 1, "exit_reason": "chat_error"}] + assert item.best_detail == {"excluded_runs": [{"run_index": 1}]} + assert report.errored == 1 + + +# ── single-shot runner: never both passed and errored ──────────────────────── + + +def _single_shot_item() -> DatasetItem: + return DatasetItem( + id="i1", + dataset_name="d", + test_kind="visualization", + question="Show revenue by quarter", + expected_output={"visualization": _viz()}, + ) + + +class _PassThenBoom: + def __init__(self) -> None: + self.calls = 0 + + def ask(self, item: DatasetItem) -> ChatResult: + self.calls += 1 + if self.calls == 2: + raise RuntimeError("network down") + return _viz_turn() + + +def _run_single_shot() -> tuple[object, list[str]]: + logged: list[str] = [] + report = run_items( + [_single_shot_item()], + _PassThenBoom(), + runs=2, + on_langfuse_item_done=lambda _i, _t, r: logged.append(r.id), + ) + return report.items[0], logged + + +def test_off_a_pass_then_an_exception_is_both_passed_and_errored_as_before() -> None: + item, logged = _run_single_shot() + + assert item.pass_at_k is True + assert item.error == "RuntimeError: network down" + assert logged == [] + + +def test_on_a_pass_then_an_exception_is_an_error_only_and_still_logged(flag_on) -> None: + item, logged = _run_single_shot() + + assert item.error == "RuntimeError: network down" + assert item.pass_at_k is False + assert item.passed is False + assert item.runs_passed == 1 + assert logged == ["i1"] + + +# ── the agent's own timeouts and incomplete turns are scored, not errors ───── + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("503 after retries"), LoopExit.CHAT_ERROR), + (ChatError("SSE stream error"), LoopExit.CHAT_ERROR), + (httpx.ReadError("connection reset"), LoopExit.CHAT_ERROR), + ], +) +def test_a_chat_failure_maps_to_its_own_exit(exc: BaseException, loop_exit: LoopExit) -> None: + assert chat_failure_exit(exc) is loop_exit + + +@pytest.mark.parametrize("loop_exit", [LoopExit.AGENT_TIMEOUT, LoopExit.TURN_INCOMPLETE]) +def test_the_agents_own_chat_failures_are_scored_as_failures(loop_exit: LoopExit) -> None: + outcome = outcome_from_loop_exit(loop_exit, passed=False) + assert outcome.status is OutcomeStatus.FAIL + assert outcome.is_scored + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + ], +) +def test_on_an_agent_timeout_stays_in_the_gate_and_fails_the_item(flag_on, exc, exit_value) -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([exc, _viz_turn(), _viz_turn()]) + + assert exc_info.value.runs_passed == 2 + assert exc_info.value.runs_effective == 3 + assert "excluded" not in str(exc_info.value) + assert "excluded_runs" not in exc_info.value.detail + (record,) = exc_info.value.failed_runs + assert record["exit_reason"] == exit_value + assert record["outcome"]["status"] == "fail" + + +def test_on_only_agent_timeouts_is_a_failure_not_a_harness_error(flag_on) -> None: + timeout = TurnTimeoutError("agent exceeded the 60s turn budget") + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([timeout, timeout, timeout]) + + assert not isinstance(exc_info.value, EvalHarnessError) + assert gate_failure_note("power", 0, 3) in str(exc_info.value) + + +def test_off_an_agent_timeout_fails_the_item_exactly_as_a_chat_error_did() -> None: + timeout = TurnTimeoutError("agent exceeded the 60s turn budget") + with pytest.raises(VisualizationAssertionError) as timed_out: + _evaluate([timeout, _viz_turn(), _viz_turn()]) + with pytest.raises(VisualizationAssertionError) as chat_failed: + _evaluate([ChatError("SSE stream error"), _viz_turn(), _viz_turn()]) + + for exc_info in (timed_out, chat_failed): + assert exc_info.value.runs_passed == 2 + assert exc_info.value.runs_effective == 3 + assert str(exc_info.value).splitlines()[2].startswith(gate_failure_note("power", 2, 3)) + + +def test_on_a_backend_chat_error_is_still_excluded(flag_on) -> None: + outcome = _evaluate([ChatError("SSE stream error"), _viz_turn(), _viz_turn()]) + + assert outcome.runs_passed == 2 + (excluded,) = outcome.detail["excluded_runs"] + assert excluded["exit_reason"] == "chat_error" + assert excluded["outcome"]["source"] == "backend" + + +# ── a partly excluded item: K, latency and the message count the runs executed ── + + +def _silent_turn() -> ChatResult: + return ChatResult.model_validate({}) + + +def test_gate_failure_note_names_the_k_that_ran_and_how_many_were_excluded() -> None: + assert gate_failure_note("power", 1, 2, runs_excluded=1) == ( + "Gate pass^3 (1 excluded) failed: 1/2 scored runs passed — unstable, not a clean failure." + ) + assert gate_failure_note("any", 0, 1, runs_excluded=2) == "Gate pass@3 (2 excluded) failed: 0/1 scored runs passed." + # Nothing excluded: exactly the note as before. + assert gate_failure_note("power", 1, 3, runs_excluded=0) == gate_failure_note("power", 1, 3) + + +def test_on_an_excluded_run_is_listed_once_and_the_gate_keeps_its_k(flag_on) -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _silent_turn(), _viz_turn()]) + + exc = exc_info.value + gate_line = str(exc).splitlines()[2] + assert gate_line.startswith("Gate pass^3 (1 excluded) failed: 1/2 scored runs passed") + assert "1 run excluded (harness error): run 1/3 the simulated user failed to reply." in gate_line + failing = gate_line.split("Failing runs: ", 1)[1] + assert failing.startswith("run 2/3 FAIL (loop exit: agent_silent)") + assert "run 1/3" not in failing + assert (exc.runs_passed, exc.runs_scored, exc.runs_effective) == (1, 2, 3) + # The excluded run keeps its record: the message names it once, the triage data in full. + assert [r["run_index"] for r in exc.failed_runs] == [1, 2] + + +def test_off_the_same_item_is_reported_exactly_as_before() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _silent_turn(), _viz_turn()]) + + exc = exc_info.value + gate_line = str(exc).splitlines()[2] + assert gate_line.startswith(gate_failure_note("power", 1, 3) + " Failing runs: run 1/3 FAIL") + assert "excluded" not in gate_line + assert (exc.runs_passed, exc.runs_effective) == (1, 3) + assert not hasattr(exc, "runs_scored") + + +def _report_for(outcome: object) -> object: + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", return_value=outcome): + return run_agentic_items([_alert_item()], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01", k=3) + + +def test_on_an_item_report_counts_executed_runs_and_the_scored_ones_apart() -> None: + outcome = AgenticEvalOutcome(runs_passed=2, runs_effective=3, runs_scored=2) + report = _report_for(outcome) + + item = report.items[0] + assert item.runs_total == 3 + assert item.runs_scored == 2 + assert item.runs_gated == 2 + assert item.pass_power_k is True + assert item.avg_latency_s == pytest.approx(item.latency_s / 3) + entry = build_json_report(report)["items"]["item-1"] + assert (entry["runs"], entry["runs_scored"], entry["pass_power_k"]) == (3, 2, True) + + +def test_off_an_item_report_has_no_scored_count() -> None: + report = _report_for(AgenticEvalOutcome(runs_passed=2, runs_effective=3)) + + item = report.items[0] + assert item.runs_scored is None + assert item.runs_gated == item.runs_total == 3 + assert item.pass_power_k is False + assert "runs_scored" not in build_json_report(report)["items"]["item-1"] + + +# ── Langfuse: an ERROR run gets eval_error and no verdict score ────────────── + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(turns: list[Any]) -> _ScoreCtx: + captured: dict[str, Any] = {} + try: + _evaluate(turns, captured) + except (AssertionError, EvalHarnessError): + pass + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_VERDICT_SCORES = {"pass_at_k", "pass_power_k", "gate_passed", "quality_score", "pass_at_3", "pass_power_3"} + + +def test_on_an_error_run_gets_eval_error_only_and_the_scored_runs_keep_theirs(flag_on) -> None: + ctx = _scores([_question_turn(), _viz_turn(), _viz_turn()]) + + assert ctx.scores[0] == [("eval_error", 1, "harness: the simulated user failed to reply")] + for run_idx in (1, 2): + scores = {name: value for name, value, _ in ctx.scores[run_idx]} + assert "eval_error" not in scores + assert scores["gate_passed"] is True + assert scores["pass_power_k"] is True + assert "quality_score" in scores + assert ctx.run_metadata["eval_k"] == 3 + + +def test_on_an_item_whose_runs_all_errored_gets_no_gate_score(flag_on) -> None: + ctx = _scores([_question_turn(), _question_turn(), _question_turn()]) + + for run_idx in range(3): + assert ctx.names(run_idx) == ["eval_error"] + assert not any(_VERDICT_SCORES & set(ctx.names(run_idx)) for run_idx in range(3)) + + +def test_on_a_backend_error_run_names_its_source(flag_on) -> None: + ctx = _scores([ChatError("SSE stream error"), _viz_turn(), _viz_turn()]) + + assert ctx.scores[0] == [("eval_error", 1, "backend: the chat call failed after its retries")] + + +def test_off_every_run_is_scored_as_before_and_none_gets_eval_error() -> None: + ctx = _scores([_question_turn(), _viz_turn(), _viz_turn()]) + + for run_idx in range(3): + names = ctx.names(run_idx) + assert "eval_error" not in names + assert {"gate_passed", "pass_at_k", "pass_power_k", "quality_score"} <= set(names) + assert dict((n, v) for n, v, _ in ctx.scores[0])["gate_passed"] is False + + +def test_a_scored_item_keeps_its_assertion_lead_and_an_errored_one_says_it_could_not_be_scored(flag_on) -> None: + scored = resolve_runs(_MIXED, passed=lambda r: r.ok, gate="power", pass_at_k=True, pass_power_k=False) + assert scored.message_lead("Alert skill assertion failed.", "Alert skill") == "Alert skill assertion failed." + errored = resolve_runs( + [_MIXED[0], _MIXED[0]], passed=lambda r: r.ok, gate="power", pass_at_k=False, pass_power_k=False + ) + assert errored.errored is True + assert ( + errored.message_lead("Alert skill assertion failed.", "Alert skill") == "Alert skill item could not be scored." + ) + + +# ── single-shot: the agent's own timeout is a failed run with the flag on ──── + + +class _PassThenTimeout: + def __init__(self, exc: Exception, fail_on: set[int]) -> None: + self.exc = exc + self.fail_on = fail_on + self.calls = 0 + + def ask(self, item: DatasetItem) -> ChatResult: + self.calls += 1 + if self.calls in self.fail_on: + raise self.exc + return _viz_turn() + + +def _run_single_shot_with(backend: _PassThenTimeout, runs: int = 2) -> tuple[object, list[str]]: + logged: list[str] = [] + report = run_items( + [_single_shot_item()], + backend, + runs=runs, + on_langfuse_item_done=lambda _i, _t, r: logged.append(r.id), + ) + return report.items[0], logged + + +_AGENT_FAILURES = [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), +] + + +@pytest.mark.parametrize(("exc", "exit_value"), _AGENT_FAILURES) +def test_off_a_single_shot_timeout_errors_the_item_as_before(exc, exit_value) -> None: + item, logged = _run_single_shot_with(_PassThenTimeout(exc, {2})) + + assert item.error == f"{type(exc).__name__}: {exc}" + assert item.pass_at_k is True + assert item.runs == 1 + assert item.failed_runs == [] + assert logged == [] + + +@pytest.mark.parametrize(("exc", "exit_value"), _AGENT_FAILURES) +def test_on_a_single_shot_timeout_is_a_failed_run_and_the_item_is_scored(flag_on, exc, exit_value) -> None: + backend = _PassThenTimeout(exc, {2}) + item, logged = _run_single_shot_with(backend, runs=3) + + assert item.error is None + assert backend.calls == 3 + assert (item.runs, item.runs_passed, item.pass_at_k, item.pass_power_k) == (3, 2, True, False) + (record,) = item.failed_runs + assert record["run_index"] == 2 + assert record["exit_reason"] == exit_value + assert record["outcome"] == {"status": "fail", "source": None, "reason": f"loop exit: {exit_value}"} + assert len(item.run_latencies_s) == 3 + assert logged == ["i1"] + + +def test_on_a_single_shot_item_that_only_timed_out_fails_and_is_not_an_error(flag_on) -> None: + item, logged = _run_single_shot_with(_PassThenTimeout(TurnTimeoutError("too slow"), {1, 2})) + + assert item.error is None + assert item.pass_at_k is False + assert item.passed is False + assert item.runs == 2 + assert [r["exit_reason"] for r in item.failed_runs] == ["agent_timeout", "agent_timeout"] + assert logged == ["i1"] + + +def test_on_a_single_shot_backend_chat_error_still_errors_the_item(flag_on) -> None: + item, _logged = _run_single_shot_with(_PassThenTimeout(ChatError("SSE stream error"), {2})) + + assert item.error == "ChatError: SSE stream error" + assert item.pass_at_k is False + + +# ── flag off: an agentic message is the master text plus the always-on failing runs ── + +# The flag-off message of [simulated user failed, agent silent, pass] under pass^3, as the +# package produced it before error outcomes existed. Only the "Failing runs" part of the gate +# line is new, and it is always on. +_MASTER_FLAG_OFF_MESSAGE = ( + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + "Agentic Visualization Assertion Failed! (Critical Mode)\n" + "Gate pass^3 failed: 1/3 runs passed — unstable, not a clean failure.\n" + "------------------------------------------\n" + "Question:\n" + "Show revenue by quarter\n" + "------------------------------------------\n" + "Expected Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Actual Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Strict Check Summary:\n" + " Visualization Created : True\n" + " Cross-Ref Valid : True\n" + " Metrics Correct : True\n" + " expected : ['metric/revenue']\n" + " actual : ['metric/revenue']\n" + " Dimensions Correct : True\n" + " expected : ['label/date.quarter']\n" + " actual : ['label/date.quarter']\n" + " Filters Correct : True\n" + " date : True\n" + " ranking : True\n" + " attribute : True\n" + " Viz Type Hard : True\n" + " Executed : False\n" + " Stated Value Matches : None (reported only)\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" +) + + +def test_off_the_agentic_message_is_the_master_text_plus_the_failing_runs() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _silent_turn(), _viz_turn()]) + + message = str(exc_info.value) + gate_line = message.splitlines()[2] + head, failing = gate_line.split(" Failing runs: ", 1) + assert failing.startswith("run 1/3 FAIL (loop exit: simulated_user_failed): ") + assert "; run 2/3 FAIL (loop exit: agent_silent): " in failing + assert message.replace(gate_line, head, 1) == _MASTER_FLAG_OFF_MESSAGE diff --git a/packages/gooddata-eval/tests/test_reporting.py b/packages/gooddata-eval/tests/test_reporting.py index d579f0376..aa0600c33 100644 --- a/packages/gooddata-eval/tests/test_reporting.py +++ b/packages/gooddata-eval/tests/test_reporting.py @@ -647,3 +647,29 @@ def test_comparison_table_sets_single_shot_trace_median_and_tail_side_by_side(): assert "19.00s" in text # gpt-5.2 P90 assert "10.00s" in text # gpt-4o median of 8 and 12 assert "single-shot chat-turn response time" in text + + +def _gate_failed_item(*, runs_scored: int | None) -> ItemReport: + r = _item("gated", runs=3, runs_passed=1, passed=False) + r.gate_passed = False + r.runs_scored = runs_scored + return r + + +def test_console_gate_note_names_the_excluded_runs_when_some_were_left_out(): + report = EvalReport(model="m", gate="power") + report.items = [_gate_failed_item(runs_scored=2)] + + row = next(line for line in _rendered(report).splitlines() if "gated" in line) + + assert "pass^3 (1 excluded) failed: 1/2 scored runs passed" in row + + +def test_console_gate_note_without_excluded_runs_is_as_before(): + report = EvalReport(model="m", gate="power") + report.items = [_gate_failed_item(runs_scored=None)] + + row = next(line for line in _rendered(report).splitlines() if "gated" in line) + + assert "pass^3 failed: 1/3 runs passed" in row + assert "excluded" not in row