diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index e8feb992e..66bcf45c8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -16,6 +16,7 @@ from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation from gooddata_eval.core.agentic.dashboard_skill import evaluate_agentic_dashboard_skill +from gooddata_eval.core.agentic.dashboard_summary import evaluate_agentic_dashboard_summary from gooddata_eval.core.agentic.document_skill import evaluate_agentic_document_skill from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question from gooddata_eval.core.agentic.guardrail import evaluate_agentic_guardrail @@ -56,6 +57,7 @@ class _LfKw(TypedDict, total=False): "agentic_guardrail", "agentic_conversation", "agentic_kda_skill", + "agentic_dashboard_summary", "agentic_anomaly_detection", "agentic_what_if", } @@ -303,6 +305,28 @@ def _dispatch_agentic( user_context=item.user_context, **lf_kw, ) + elif kind == "agentic_dashboard_summary": + summary_input = item.summary_input + if summary_input is None: + raise ValueError(f"agentic_dashboard_summary item '{item.id}' is missing required 'summary_input'.") + return evaluate_agentic_dashboard_summary( + host=host, + token=token, + workspace_id=workspace_id, + dashboard_id=summary_input.dashboard_id, + expected_output=eo, + # The fixture's own wording is the prompt under test -- a localized fixture is + # only meaningful if its own phrasing is what reaches the agent. + question=item.question, + # Each widget costs an execution, so a fixture asserting on a handful of charts + # can name them instead of paying for the whole dashboard. Same field and meaning + # the headless /summary endpoint gives it; None summarizes everything. + only_visualizations=summary_input.visualizations, + k=k, + gate=gate, + agent_id=agent_id, + **lf_kw, + ) elif kind == "agentic_what_if": return evaluate_agentic_what_if( host=host, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index fdb797c09..1c7aaf771 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -3,6 +3,7 @@ from __future__ import annotations +import re import time from dataclasses import dataclass, field from typing import Any @@ -273,6 +274,65 @@ def _check_visualizations(widgets: list[dict], new_ids: set[str], expected: list return failures, title_mismatches +def _check_forbidden(widgets: list[dict], forbidden: list[dict]) -> list[str]: + """Report charts the expectation says must NOT be on the dashboard. + + The counterpart to ``_check_visualizations``, which allows extras. Extras are the right + default -- a user asking for three charts is not wronged by a fourth -- but it leaves the + cases that are about what the agent must *refrain* from unassertable: a chart the agent was + told it could not use, and a substitute authored in place of one it could not resolve. + + An entry with an ``id`` is matched on that id alone; a ``title`` beside it is documentation + for whoever reads the fixture, not a second matcher. Titles are not unique in a real + workspace -- the catalog this was written against has 102 visualizations whose title + contains "approval rate" -- so widening an id entry to also match its title would fail the + case on a chart that merely shares a name with the forbidden one. + + An entry with only a ``title`` matches on the title, which is what catches a substitution + whose id is not known in advance. + """ + failures: list[str] = [] + for entry in forbidden: + exp_id = entry.get("id") + exp_title = str(entry.get("title") or "") + if exp_id is not None: + if any(w.get("visualization") == exp_id for w in widgets): + failures.append(f"chart id={exp_id!r} title={exp_title!r} is on the dashboard and must not be") + else: + hits = [w for w in widgets if _norm(str(w.get("title") or "")) == _norm(exp_title)] + if hits: + ids = ", ".join(repr(w.get("visualization")) for w in hits) + failures.append(f"a chart titled {exp_title!r} is on the dashboard ({ids}) and must not be") + return failures + + +def _check_answer(answer: str, must_include: list[str], must_not_match: list[str]) -> list[str]: + """Score the text the user actually reads. + + Nothing else in this evaluator looks at the answer: every other check reads the drafted + document, so a run that builds the right dashboard and describes it wrongly scores as a + clean pass. The two cases that need this are the ones where the text *is* the deliverable + -- an agent that cannot build what was asked has to say so, and one that can has to say it + without leaking the raw ``{visualization/}`` tokens the renderer was meant to resolve. + + ``must_include`` is substring, case- and whitespace-insensitive, because it asserts that a + point was made rather than that a sentence was phrased a particular way. ``must_not_match`` + is a regular expression, because what it rules out is usually a shape -- a token, an id, a + claim pattern -- and not a fixed string. + """ + normalized = _norm(answer) + failures: list[str] = [ + f"the answer does not mention {phrase!r}" for phrase in must_include if _norm(str(phrase)) not in normalized + ] + for pattern in must_not_match: + # Compiled at validation time too, so a bad pattern fails the fixture rather than the + # run; compiling again here keeps this function usable on its own. + found = re.search(str(pattern), answer, re.IGNORECASE) + if found: + failures.append(f"the answer matches {pattern!r}, which it must not (matched {found.group(0)!r})") + return failures + + def _check_references(widgets: list[dict], known_ids: set[str]) -> list[str]: """Report widgets whose id the response's references never carried. @@ -1062,6 +1122,76 @@ def _min_new_visualizations(expected_output: dict) -> int: return value +def _max_new_visualizations(expected_output: dict) -> int | None: + """Upper bound on the number of charts the agent may author, or ``None`` for no bound. + + The bound that was missing. ``min_new_visualizations`` alone cannot express "use what is + already there", which is the central requirement wherever a curated catalog exists: with + only a floor, ``0`` is satisfied by any number of authored charts, so a case meant to + assert that the agent reused existing work passes however much it invented. Pairing it + with ``max_new_visualizations: 0`` is what turns that into an assertion. + + Absent means unbounded, which is the behaviour every fixture written before this had. + ``0`` is a real bound and must not be read as absence. + + Raises: + ValueError: the bound is not a number, is negative, or is below the floor — a ceiling + under the floor cannot be satisfied by any run, so the fixture is unscoreable + rather than strict. + """ + raw = expected_output.get("max_new_visualizations") + if raw is None: + return None + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"max_new_visualizations is not a number: {raw!r}") from exc + if value < 0: + raise ValueError(f"max_new_visualizations must not be negative, got {value}") + minimum = _min_new_visualizations(expected_output) + if value < minimum: + raise ValueError(f"max_new_visualizations ({value}) is below min_new_visualizations ({minimum})") + return value + + +def _forbidden_visualizations(expected_output: dict) -> list[dict]: + """The ``must_not_contain`` entries, as a list.""" + return expected_output.get("must_not_contain") or [] + + +def _answer_assertions(expected_output: dict) -> tuple[list[str], list[str]]: + """The answer-text assertions, as ``(must_include, must_not_match)``.""" + return ( + list(expected_output.get("answer_must_include") or []), + list(expected_output.get("answer_must_not_match") or []), + ) + + +def _compile_pattern(pattern: object) -> re.Pattern[str]: + """Compile one ``answer_must_not_match`` entry, naming the fixture key on failure. + + Raises: + ValueError: the pattern does not compile. + """ + try: + return re.compile(str(pattern)) + except re.error as exc: + raise ValueError(f"answer_must_not_match carries an invalid regular expression {pattern!r}: {exc}") from exc + + +def _expects_dashboard(expected_output: dict) -> bool: + """Whether the case expects a dashboard at all. + + ``false`` is the refusal shape: the agent was asked for something it cannot build, and the + right outcome is that it says so and drafts nothing. Those cases could not be authored + before, because an expectation with no ``visualizations`` is rejected as vacuous -- and + correctly so, since every chart check would pass against nothing. What makes a refusal + scoreable is not relaxing that rule but replacing the assertion: the answer carries the + whole of what the run is judged on, so a refusal fixture has to state it. + """ + return expected_output.get("expects_dashboard", True) is not False + + def _has_filters(expected_output: dict) -> bool: """Whether the expectation says anything about the attribute filters. @@ -1092,7 +1222,47 @@ def _validate_expectation(expected_output: dict) -> None: Raises: ValueError: the expectation is unusable. """ - if not expected_output.get("visualizations"): + for key in ("answer_must_include", "answer_must_not_match"): + value = expected_output.get(key) + if value is None: + continue + if not isinstance(value, list): + raise ValueError(f"{key} must be a list of strings, got {value!r}") + # isinstance, not str(entry): `- no` in YAML loads as the bool False, which str() + # turns into the assertion "false" -- a fixture that looks strict while checking + # text the question never mentions. Same for a bare number. + if any(not isinstance(entry, str) for entry in value): + raise ValueError(f"{key} must be a list of strings, got {value!r}") + if any(not entry.strip() for entry in value): + raise ValueError(f"{key} carries an empty entry, which asserts nothing: {value!r}") + must_include, must_not_match = _answer_assertions(expected_output) + # Compiled here so an unparseable pattern fails the fixture before the first API call, + # rather than raising mid-scoring with a run already spent. + for pattern in must_not_match: + _compile_pattern(pattern) + + forbidden = _forbidden_visualizations(expected_output) + if not isinstance(forbidden, list): + raise ValueError(f"must_not_contain must be a list of chart expectations, got {forbidden!r}") + for entry in forbidden: + if not isinstance(entry, dict): + raise ValueError(f"a must_not_contain entry must be an object, got {entry!r}") + if entry.get("id") is None and not str(entry.get("title") or "").strip(): + raise ValueError(f"a must_not_contain entry needs an id or a title, got {entry!r}") + + if not _expects_dashboard(expected_output): + # The refusal shape. Each of these would otherwise produce a fixture that looks strict + # and asserts nothing, which is the failure mode the vacuity check below exists to stop. + if expected_output.get("visualizations"): + raise ValueError("expects_dashboard is false, so the case must not list visualizations to find") + if _is_edit(expected_output): + raise ValueError("expects_dashboard is false is a creation shape; an edit always produces a patch") + if not must_include and not must_not_match: + raise ValueError( + "expects_dashboard is false needs an answer assertion; without one the case scores nothing " + "beyond the absence of a draft, which an agent that fell over satisfies too" + ) + elif not expected_output.get("visualizations"): raise ValueError("expected_output lists no visualizations; every chart check would pass vacuously") if _has_date_range(expected_output): date_range = expected_output.get("date_range") @@ -1146,6 +1316,7 @@ def _validate_expectation(expected_output: dict) -> None: elif saved_id is not None: raise ValueError(f"a creation expectation must not name a saved_dashboard_id, got {saved_id!r}") _min_new_visualizations(expected_output) + _max_new_visualizations(expected_output) def _validate_layout_expectations(expected_output: dict) -> None: @@ -1348,9 +1519,22 @@ class _Applies: chart_types: bool = False layout: bool = False filter_bar: bool = False + forbidden: bool = False + answer: bool = False + # Not a check of its own but a switch over the whole set: a refusal case is scored on the + # absence of a draft and on what the answer said, so publishing the document checks beside + # it would report six passes for a document that was never produced. + refusal: bool = False @classmethod - def of(cls, expected_output: dict) -> _Applies: + def of( + cls, + expected_output: dict, + *, + forbidden: bool = False, + answer: bool = False, + refusal: bool = False, + ) -> _Applies: return cls( patch=_is_edit(expected_output), date=_has_date_range(expected_output), @@ -1363,6 +1547,9 @@ def of(cls, expected_output: dict) -> _Applies: chart_types=_has_chart_types(expected_output), layout="layout" in expected_output, filter_bar="filter_bar" in expected_output, + forbidden=forbidden, + answer=answer, + refusal=refusal, ) @@ -1406,6 +1593,12 @@ class DashboardEvaluation: chart_types_correct: bool = False layout_correct: bool = False filter_bar_correct: bool = False + forbidden_absent: bool = True + answer_matched: bool = True + # Whether EITHER dashboard-producing tool was called at all, successful or not. Only a + # refusal case reads it: everywhere else a failed call is a retry the agent recovered + # from, which is why `_extract_tool_result` skips it. + producing_tool_called: bool = False failures: list[str] = field(default_factory=list) notes: list[str] = field(default_factory=list) @@ -1415,6 +1608,25 @@ def strict_pass(self) -> bool: @property def strict_checks(self) -> dict[str, bool]: + if self.applies.refusal: + # Named for what has to be true, not negated at read time: `dashboard_not_drafted` + # false means a dashboard came back from a case that asked for none, which is the + # failure. A successful tool call with no part, or a part with no successful call, + # are each a dashboard the user would be shown. + # + # The third term is the one that is easy to miss: an agent that CALLED the tool and + # had the call rejected leaves no result and no part, so the first two would read it + # as a clean refusal. It is not one -- the agent tried to build and the tool stopped + # it, and scoring that as correct credits the model for a guardrail. + checks = { + "dashboard_not_drafted": ( + not self.drafted and not self.part_present and not self.producing_tool_called + ), + "dashboard_skill_activated": self.skill_activated, + } + if self.applies.answer: + checks["answer_matched"] = self.answer_matched + return checks checks = { # Kept as-is through the editing work: every Langfuse view, saved filter and # combo-report field list already refers to it, and a rename would break them for @@ -1452,6 +1664,10 @@ def strict_checks(self) -> dict[str, bool]: checks["dashboard_layout_correct"] = self.layout_correct if self.applies.filter_bar: checks["dashboard_filter_bar_correct"] = self.filter_bar_correct + if self.applies.forbidden: + checks["forbidden_charts_absent"] = self.forbidden_absent + if self.applies.answer: + checks["answer_matched"] = self.answer_matched return checks @property @@ -1510,6 +1726,8 @@ def evaluate_dashboard_response( expected_output: dict, skill_activated: bool, patch_part: dict | None = None, + answer_text: str = "", + producing_tool_called: bool = False, ) -> DashboardEvaluation: """Score one dashboard response against its expectation. @@ -1521,8 +1739,56 @@ def evaluate_dashboard_response( without an agent. """ is_edit = _is_edit(expected_output) - applies = _Applies.of(expected_output) + forbidden = _forbidden_visualizations(expected_output) + must_include, must_not_match = _answer_assertions(expected_output) + expects_dashboard = _expects_dashboard(expected_output) + applies = _Applies.of( + expected_output, + forbidden=bool(forbidden), + answer=bool(must_include or must_not_match), + refusal=not expects_dashboard, + ) tool = _producing_tool(expected_output) + answer_failures = _check_answer(answer_text, must_include, must_not_match) + + if not expects_dashboard: + # The whole of a refusal case: nothing was drafted, and the answer said the right + # thing. Scored before the branches below so the absence of a tool result reads as the + # outcome asked for rather than as "the agent never produced a successful draft". + drafted = tool_result is not None + return DashboardEvaluation( + drafted=drafted, + part_present=dashboard_part is not None, + charts_matched=True, + date_range_correct=True, + filters_correct=True, + new_visualizations_met=True, + applies=applies, + skill_activated=skill_activated, + producing_tool_called=producing_tool_called, + answer_matched=not answer_failures, + failures=[ + *( + [f"the case expects no dashboard, but the agent produced a successful {tool} call"] + if drafted + else [] + ), + *( + ["the case expects no dashboard, but the response carries a 'dashboard' part"] + if dashboard_part is not None + else [] + ), + *( + [ + "the case expects no dashboard, but the agent called a dashboard-producing " + "tool (the call did not succeed)" + ] + if producing_tool_called and not drafted + else [] + ), + *answer_failures, + ], + ) if tool_result is None: return DashboardEvaluation( @@ -1536,15 +1802,25 @@ def evaluate_dashboard_response( skill_activated=skill_activated, saved_dashboard_id_correct=False, patch_applies=False, - failures=[f"the agent never produced a successful {tool} call"], + answer_matched=not answer_failures, + failures=[f"the agent never produced a successful {tool} call", *answer_failures], ) min_new = _min_new_visualizations(expected_output) + max_new = _max_new_visualizations(expected_output) actual_new = tool_result.get("new_visualization_count") if isinstance(actual_new, int): - new_met = actual_new >= min_new + over = max_new is not None and actual_new > max_new + new_met = actual_new >= min_new and not over # Naming the shortfall, never "expected at least 0" -- see the envelope branch below. - new_failures = [] if new_met else [f"expected at least {min_new} authored chart(s), tool reported {actual_new}"] + # The ceiling is reported as its own sentence: "too few" and "too many" are opposite + # diagnoses and a single range message would make whoever reads the failure work out + # which end was missed. + new_failures: list[str] = [] + if actual_new < min_new: + new_failures.append(f"expected at least {min_new} authored chart(s), tool reported {actual_new}") + if over: + new_failures.append(f"expected at most {max_new} authored chart(s), tool reported {actual_new}") else: # Not a shortfall: the tool result no longer carries the key. Said plainly, because # "expected at least 0 authored chart(s), tool reported None" reads as a broken test @@ -1569,9 +1845,11 @@ def evaluate_dashboard_response( skill_activated=skill_activated, saved_dashboard_id_correct=False, patch_applies=False, + answer_matched=not answer_failures, failures=[ f"the response carries no {' and no '.join(repr(p) for p in missing_parts)} part", *new_failures, + *answer_failures, ], ) @@ -1624,7 +1902,8 @@ def evaluate_dashboard_response( skill_activated=skill_activated, saved_dashboard_id_correct=not saved_failures, patch_applies=False, - failures=[*patch_failures, *saved_failures, *new_failures], + answer_matched=not answer_failures, + failures=[*patch_failures, *saved_failures, *new_failures, *answer_failures], ) widgets = _widgets_of(document) @@ -1640,6 +1919,7 @@ def evaluate_dashboard_response( title_notes: list[str] = [] else: title_notes = title_mismatches + forbidden_failures = _check_forbidden(widgets, forbidden) reference_notes = _check_references(widgets, known_ids) date_failures = ( _check_date_range(document, expected_output.get("date_range"), applies.patch) @@ -1691,8 +1971,11 @@ def evaluate_dashboard_response( chart_types_correct=not chart_type_failures, layout_correct=not layout_failures, filter_bar_correct=not filter_bar_failures, + forbidden_absent=not forbidden_failures, + answer_matched=not answer_failures, failures=[ *chart_failures, + *forbidden_failures, *saved_failures, *date_failures, *filter_failures, @@ -1705,6 +1988,7 @@ def evaluate_dashboard_response( *layout_failures, *filter_bar_failures, *new_failures, + *answer_failures, ], notes=[*title_notes, *reference_notes, *chart_type_notes], ) @@ -1724,6 +2008,8 @@ def _execute_single_dashboard_run( """ tool = _producing_tool(expected_output) is_edit = _is_edit(expected_output) + expects_dashboard = _expects_dashboard(expected_output) + answer_text = "" tool_result: dict | None = None dashboard_part: dict | None = None patch_part: dict | None = None @@ -1758,6 +2044,13 @@ def _execute_single_dashboard_run( all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) steps += chat_result.reasoning_step_count + # Read every turn, not only the ones that answer without drafting: the turn that + # produces the draft carries the text describing it, and that text is exactly what the + # answer assertions are about. Kept as the last turn that said anything, so a silent + # turn after a spoken one does not erase what the user was shown. + response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result) + answer_text = response_text or answer_text + candidate = _extract_tool_result(chat_result.tool_call_events or [], tool) if candidate is not None: log_timer( @@ -1780,9 +2073,14 @@ def _execute_single_dashboard_run( patch_part = _extract_dashboard_part(chat_result, _PATCH_TYPE) break - response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result) if not response_text and not chat_result.tool_call_events: break + if not expects_dashboard: + # A refusal case is answered in one turn. The reply this loop would otherwise send + # restates the charts and the date range and asks again for the dashboard, which is + # pressure to build the very thing the case says must not be built -- so a correct + # agent would be talked out of the right answer by the harness. + break if iteration >= max_iterations - 1: break log_timer( @@ -1805,6 +2103,12 @@ def _execute_single_dashboard_run( expected_output, _skill_activated(all_tool_call_events, _required_skill(expected_output)), patch_part=patch_part, + answer_text=answer_text, + # Any call, not just a successful one -- see DashboardEvaluation. BOTH tools, + # not the one this case selected: a refusal is a creation shape, so `tool` is + # `draft_dashboard`, and an agent that routed to the editor and had a + # `patch_dashboard` call rejected would otherwise leave this false and pass. + producing_tool_called=any(tc.function_name in (_DRAFT_TOOL, _PATCH_TOOL) for tc in all_tool_call_events), ), tool_result=tool_result, dashboard_part=dashboard_part, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py new file mode 100644 index 000000000..073e858f0 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_summary.py @@ -0,0 +1,581 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Agentic dashboard-summary evaluation runner. + +Covers the path a user actually takes: the dashboard's "Summarize" menu item drops them +into the assistant with "Summarize this dashboard" pre-filled, so the summary is produced +by the conversational ``dashboard_summary`` skill, not by the dedicated +``POST /api/v1/ai/workspaces/{ws}/summary`` endpoint the ``dashboard_summary`` test kind +exercises. That endpoint sits behind its own feature flag (``ENABLE_GEN_AI_HEADLESS_SUMMARY``) +and serves API/embedding consumers; the two share an idea and nothing else. + +The skill reads the dashboard from ``userContext.view.dashboard`` and collects only the +widgets that carry a ``result_id`` -- a widget without one is dropped from the summarize +scope entirely, which is why sending a bare dashboard id gets "no dashboard charts were +provided" and sending descriptors without results gets "these visualizations need to be +reloaded". In the browser the ids exist because the client has already rendered the +widgets. Here we do the same thing deliberately: walk the dashboard's layout, execute each +insight, and hand back the result ids that execution produced. + +Scoring is unchanged from the single-shot kind -- ``DashboardSummaryEvaluator`` grades free +text against the fixture's ``must_include``/``must_not_include``/``rubric``, so it does not +care which transport produced the summary. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +import gooddata_sdk.table as table_module +import httpx +from gooddata_sdk import GoodDataSdk + +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError +from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.evaluators.summary import DashboardSummaryEvaluator +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + DatasetItem, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer + +_DEFAULT_K = 1 + +# What the "Summarize" menu item pre-fills, used when no question is supplied. A fixture's +# own question wins: it is the localized or paraphrased wording under test, and the dispatch +# passes it through for exactly that reason. This is the default for direct callers only. +_DEFAULT_PROMPT = "Summarize this dashboard" + + +@dataclass +class DashboardWidget: + """One insight widget of a dashboard, as the assistant needs to see it.""" + + widget_id: str + title: str + visualization_id: str + # None when the execution failed. Mirrors the browser, where a widget that did not + # render carries no result and is therefore left out of the summarize scope. + result_id: str | None = None + + +def _insight_widgets(content: dict) -> list[DashboardWidget]: + """Every insight widget in a dashboard's layout, in document order. + + Walks the whole document rather than a fixed path: dashboards nest widgets in + sections, and tab-based ones nest those again under ``tabs``, so the depth is not + known ahead of time. Widgets whose insight carries no identifier are skipped -- a + rich-text or unresolved widget has nothing to execute. + """ + widgets: list[DashboardWidget] = [] + seen: set[str] = set() + + def walk(node: Any) -> None: + if isinstance(node, dict): + insight = node.get("insight") + if isinstance(insight, dict): + identifier = insight.get("identifier") + viz_id = identifier.get("id") if isinstance(identifier, dict) else None + if viz_id: + widget_id = str(node.get("localIdentifier") or viz_id) + if widget_id not in seen: + seen.add(widget_id) + widgets.append( + DashboardWidget( + widget_id=widget_id, + title=str(node.get("title") or viz_id), + visualization_id=str(viz_id), + ) + ) + for value in node.values(): + walk(value) + elif isinstance(node, list): + for value in node: + walk(value) + + walk(content) + return widgets + + +def _execute_widget(sdk: GoodDataSdk, workspace_id: str, visualization_id: str) -> str: + """Execute one saved visualization and return the result id its execution produced. + + Reuses the table service's own pivot/non-pivot split so the execution matches what the + client would run for that visualization; only the result id is wanted, not the data. + + ``sdk.tables.for_visualization`` is the public equivalent, but it reads the whole result + into an ExecutionTable and returns that instead of the response -- which discards the one + field wanted here and pays for every row to do it. Two of the three helpers it delegates + to are private, so they are checked up front: ``gooddata-sdk`` is depended on as + ``~=1.76.0`` and a patch release may rename them, which would otherwise surface as an + AttributeError on the first widget of a run rather than as a dependency problem. + """ + missing = [ + name + for name in ("_vis_is_table", "_get_exec_for_pivot", "get_exec_for_non_pivot") + if not hasattr(table_module, name) + ] + if missing: + raise RuntimeError( + f"gooddata_sdk.table no longer provides {', '.join(missing)}, so a dashboard widget cannot be " + "executed for its result id. The installed gooddata-sdk has moved these helpers -- either pin it " + "back or port _execute_widget onto whatever replaced them." + ) + + visualization = sdk.visualizations.get_visualization(workspace_id, visualization_id) + is_pivot = table_module._vis_is_table(visualization) or visualization.has_bucket_of_type( + table_module.BucketType.ROWS + ) + exec_def = ( + table_module._get_exec_for_pivot(visualization) + if is_pivot + else table_module.get_exec_for_non_pivot(visualization) + ) + return sdk.compute.for_exec_def(workspace_id, exec_def).result_id + + +def _fetch_dashboard(host: str, token: str, workspace_id: str, dashboard_id: str) -> tuple[str | None, dict]: + """The dashboard's title and layout document. + + Read over plain HTTP rather than through ``entities_api``: the generated client + validates the entity's timestamp fields with a regex and raises + ``TypeError: expected string or bytes-like object, got 'datetime.datetime'`` on the + response, so the typed accessor cannot read a dashboard at all today. + """ + url = f"{host.rstrip('/')}/api/v1/entities/workspaces/{workspace_id}/analyticalDashboards/{dashboard_id}" + resp = httpx.get(url, headers={"Authorization": f"Bearer {token}"}, timeout=120.0) + resp.raise_for_status() + attributes = resp.json()["data"]["attributes"] + return attributes.get("title"), attributes.get("content") or {} + + +def build_dashboard_user_context( + sdk: GoodDataSdk, + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + *, + only_visualizations: list[str] | None = None, + max_widgets: int | None = None, +) -> tuple[dict, list[DashboardWidget]]: + """Assemble the ``userContext`` for one dashboard, executing its widgets to get result ids. + + Returns the context and the widgets it describes, each carrying the result id its + execution produced or None if that execution failed. A failed widget is still returned + (reporting needs to know the summary was partial) but is left out of the context, since + the skill would drop it anyway. + + A dashboard of thirty widgets costs thirty executions per item, so both arguments exist + to bound that -- at the price of summarizing less than the user would see. + ``only_visualizations`` keeps just the named visualizations, which is what a fixture sets + (via ``summary_input.visualizations``, the same field and meaning the headless endpoint + gives it) because naming the widgets a rubric asserts on stays stable as the dashboard + grows. ``max_widgets`` truncates in layout order and is a blunt cap for direct callers. + """ + title, content = _fetch_dashboard(host, token, workspace_id, dashboard_id) + + widgets = _insight_widgets(content) + if only_visualizations is not None: + wanted = set(only_visualizations) + widgets = [w for w in widgets if w.visualization_id in wanted or w.widget_id in wanted] + if max_widgets is not None: + widgets = widgets[:max_widgets] + + for widget in widgets: + try: + widget.result_id = _execute_widget(sdk, workspace_id, widget.visualization_id) + except Exception as exc: # noqa: BLE001, PERF203 -- one dead widget must not lose the whole summary + log_timer(f"[dashboard_summary] widget '{widget.widget_id}' failed to execute: {exc}") + + context = { + "view": { + "dashboard": { + "id": dashboard_id, + "title": title, + "widgets": [ + { + "widgetId": w.widget_id, + "title": w.title, + "widgetType": "insight", + "visualizationId": w.visualization_id, + "resultId": w.result_id, + } + for w in widgets + if w.result_id is not None + ], + } + } + } + return context, widgets + + +@dataclass +class DashboardSummaryRunResult: + """Outcome of one K-run conversation for a dashboard summary.""" + + conversation_id: str + actual_output: str + evaluation: ItemEvaluation + widgets_total: int + widgets_executed: int + reasoning_steps: list[str] = field(default_factory=list) + response_id: str | None = None + timings: PhaseTimings = field(default_factory=PhaseTimings) + # This run's own wall time, from just before its chat call to the end of its scoring. + run_latency_s: float = 0.0 + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # Set when the chat call itself failed. Such a run has no summary to grade, so it is + # recorded rather than raised -- raising would discard the K-runs already completed. + chat_error: str | None = None + + @property + def passed(self) -> bool: + return self.chat_error is None and self.evaluation.passed + + +@dataclass +class AgenticDashboardSummarySummary: + """Aggregated outcome of K runs for one dashboard-summary item.""" + + run_results: list[DashboardSummaryRunResult] + pass_at_k: bool + pass_power_k: bool + best: DashboardSummaryRunResult + + +def _failed_run( + conversation_id: str, message: str, widgets: list[DashboardWidget], agent_s: float +) -> DashboardSummaryRunResult: + """A run that never produced a summary, recorded so the completed runs survive.""" + return DashboardSummaryRunResult( + conversation_id=conversation_id, + actual_output="", + evaluation=ItemEvaluation(passed=False, rank_key=(-1, 0.0), detail={}, error=message), + widgets_total=len(widgets), + widgets_executed=sum(1 for w in widgets if w.result_id is not None), + timings=PhaseTimings(agent_s=agent_s), + run_latency_s=agent_s, + chat_error=message, + ) + + +def _run_single_dashboard_summary( + client: ChatClient, + evaluator: DashboardSummaryEvaluator, + conversation_id: str, + item: DatasetItem, + user_context: dict, + widgets: list[DashboardWidget], + prompt: str, +) -> DashboardSummaryRunResult: + widgets_total = len(widgets) + widgets_executed = sum(1 for w in widgets if w.result_id is not None) + + agent_started = time.monotonic() + try: + chat_result = client.send_message(conversation_id, prompt, user_context=user_context) + except ChatError as exc: + return _failed_run(conversation_id, f"chat failed: {exc}", widgets, time.monotonic() - agent_started) + agent_elapsed = time.monotonic() - agent_started + + judge_started = time.monotonic() + evaluation = evaluator.evaluate(item, chat_result) + judge_elapsed = time.monotonic() - judge_started + log_timer( + f"[timer] dashboard_summary {conversation_id} agent {agent_elapsed:.2f}s, judge {judge_elapsed:.2f}s " + f"({widgets_executed}/{widgets_total} widgets executed)" + ) + + return DashboardSummaryRunResult( + conversation_id=conversation_id, + actual_output=str(evaluation.detail.get("actual_output", "")), + evaluation=evaluation, + widgets_total=widgets_total, + widgets_executed=widgets_executed, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed), + run_latency_s=agent_elapsed + judge_elapsed, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), + ) + + +def run_agentic_dashboard_summary( + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + expected_output: Any, + question: str = _DEFAULT_PROMPT, + k: int = _DEFAULT_K, + max_iterations: int = 1, # noqa: ARG001 -- single-turn kind; accepted so the runner can dispatch uniformly + initial_conversation_id: str | None = None, + reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, + only_visualizations: list[str] | None = None, + max_widgets: int | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_summary", +) -> AgenticDashboardSummarySummary: + """Run the agentic dashboard-summary evaluation K times and return a summary. + + The widgets are executed ONCE and their result ids reused across all K runs: they + identify cached executions, so re-running them per K would multiply the most expensive + part of the item without changing what the assistant sees. + """ + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) + sdk = GoodDataSdk.create(host, token) + evaluator = DashboardSummaryEvaluator() + item = DatasetItem( + id=dataset_item_id or dashboard_id, + dataset_name=dataset_name, + test_kind="agentic_dashboard_summary", + question=question, + expected_output=expected_output, + ) + run_results: list[DashboardSummaryRunResult] = [] + + try: + context_started = time.monotonic() + user_context, widgets = build_dashboard_user_context( + sdk, + host, + token, + workspace_id, + dashboard_id, + only_visualizations=only_visualizations, + max_widgets=max_widgets, + ) + log_timer( + f"[timer] dashboard_summary {dashboard_id} context built in " + f"{time.monotonic() - context_started:.2f}s ({len(widgets)} widgets)" + ) + + conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() + try: + run_results.append( + _run_single_dashboard_summary(client, evaluator, conv_id_0, item, user_context, widgets, question) + ) + finally: + if initial_conversation_id is None: + client.delete_conversation(conv_id_0) + + for _ in range(1, k): + try: + conv_id = client.create_conversation() + except Exception as exc: # noqa: BLE001 -- a lost conversation must not discard completed runs + # Same contract as the ChatError path below: this line used to sit outside any + # handler, so a transient failure here on run 2 of 3 threw away run 1. + run_results.append(_failed_run("", f"conversation creation failed: {exc}", widgets, 0.0)) + continue + try: + run_results.append( + _run_single_dashboard_summary(client, evaluator, conv_id, item, user_context, widgets, question) + ) + finally: + client.delete_conversation(conv_id) + finally: + client.close() + + scored = [r for r in run_results if r.chat_error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = len(scored) == len(run_results) and bool(scored) and all(r.passed for r in scored) + best = max(scored or run_results, key=lambda r: r.evaluation.rank_key) + return AgenticDashboardSummarySummary( + run_results=run_results, + pass_at_k=pass_at_k, + pass_power_k=pass_power_k, + best=best, + ) + + +def _run_detail(run: DashboardSummaryRunResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one. + + Extracted so a failing run is described by exactly the same keys as the winning run -- + for this kind that means the rubric breakdown plus how much of the dashboard the run + actually managed to execute, which is the whole diagnosis. + """ + detail = dict(run.evaluation.detail) + # How much of the dashboard the summary actually covered. A summary graded against a + # rubric that mentions a widget which never executed fails for a reason that has + # nothing to do with the agent, so the ratio has to be visible in the report. + detail["widgets_total"] = run.widgets_total + detail["widgets_executed"] = run.widgets_executed + if run.chat_error is not None: + detail["chat_error"] = run.chat_error + detail["latency_breakdown"] = build_latency_breakdown(run.tool_call_events, run.reasoning_step_events) + return detail + + +def _detail(summary: AgenticDashboardSummarySummary) -> dict: + return _run_detail(summary.best) + + +class DashboardSummaryAssertionError(AgenticAssertionError): + """Raised when an agentic dashboard-summary evaluation fails.""" + + +def evaluate_agentic_dashboard_summary( + host: str, + token: str, + workspace_id: str, + dashboard_id: str, + expected_output: Any, + question: str = _DEFAULT_PROMPT, + k: int = _DEFAULT_K, + initial_conversation_id: str | None = None, + gate: EvalGate = DEFAULT_GATE, + agent_id: str | None = None, + langfuse: object | None = None, + dataset_item_id: str = "", + dataset_name: str = "dashboard_summary", + run_timestamp: str | None = None, + model_version_override: str | None = None, + run_metadata_extra: dict | None = None, + reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + only_visualizations: list[str] | None = None, + max_widgets: int | None = None, +) -> AgenticEvalOutcome: + """Run the evaluation, log to Langfuse, and raise DashboardSummaryAssertionError on failure.""" + langfuse, window_start = open_trace_window(langfuse) + summary = run_agentic_dashboard_summary( + host=host, + token=token, + workspace_id=workspace_id, + dashboard_id=dashboard_id, + expected_output=expected_output, + question=question, + k=k, + initial_conversation_id=initial_conversation_id, + reasoning_effort=reasoning_effort, + agent_id=agent_id, + only_visualizations=only_visualizations, + max_widgets=max_widgets, + dataset_item_id=dataset_item_id, + dataset_name=dataset_name, + ) + + if langfuse is not None and dataset_item_id: + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + + for run_idx, run in enumerate(summary.run_results): + if run.chat_error is not None: + # The chat never produced a summary, so there is no verdict: writing + # 0.0 would publish a content failure the judge never assessed. + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score( + tid, name="dashboard_summary_pass", value=float(run.evaluation.passed), data_type="BOOLEAN" + ) + ctx.quality( + tid, + strict_checks={"dashboard_summary_pass": run.evaluation.passed}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) + + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + # A run whose chat failed is skipped above, so polling for its trace would only + # spend the item's shared retry budget on scores that never get written. + conversation_ids=[r.conversation_id for r in summary.run_results if r.chat_error is None], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, + # The question this run answered, so a score is readable without resolving the + # conversation back to its item. + item_input=question, + ) + + best = summary.best + detail = _detail(summary) + timings = PhaseTimings() + for run in summary.run_results: + timings = timings + run.timings + runs_passed = sum(1 for r in summary.run_results if r.passed) + # Same predicate runs_passed is taken over, so an item's failed_runs and its counts + # cannot disagree about which runs failed. + failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.passed, detail=_run_detail) + + # The gate decides the item, not pass@K alone. `pass_power_k` was computed and never + # read, so `--gate power` passed any item with one good run -- the same defect fixed in + # forecasting, anomaly detection and what-if, and this was the last evaluator still + # carrying it. + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + error = DashboardSummaryAssertionError( + f"Dashboard summary failed for '{dashboard_id}'. {gate_note} " + f"({best.widgets_executed}/{best.widgets_total} widgets executed): " + f"{best.evaluation.error or 'criteria not satisfied'}" + ) + error.reasoning_steps = best.reasoning_steps + error.conversation_id = best.conversation_id + error.response_id = best.response_id + error.detail = detail + error.timings = timings + error.runs_passed = runs_passed + error.runs_effective = len(summary.run_results) + error.failed_runs = failed_runs + error.run_latencies_s = [r.run_latency_s for r in summary.run_results] + raise error + + return AgenticEvalOutcome( + reasoning_steps=best.reasoning_steps, + conversation_id=best.conversation_id, + response_id=best.response_id, + detail=detail, + timings=timings, + runs_passed=runs_passed, + runs_effective=len(summary.run_results), + failed_runs=failed_runs, + run_latencies_s=[r.run_latency_s for r in summary.run_results], + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py index 2a6e4cbfa..ccf84491c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/interaction.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/interaction.py @@ -28,6 +28,7 @@ "agentic_general_question", "agentic_guardrail", "agentic_search", + "agentic_dashboard_summary", } ) diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index 94a39a970..c29bfbbab 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -2452,3 +2452,399 @@ def _evaluate_with(client, expected_output): expected_output=expected_output, initial_conversation_id="conv-1", ) + + +# ── negative and answer-text assertions ───────────────────────────────────── +# The four capabilities a curated-catalog engagement needs and the document checks above +# cannot express: a ceiling on authored charts, charts that must not appear, a case whose +# right outcome is no dashboard at all, and the text the user actually reads. + +_SUBSTITUTE = "269d4650-2c1f-4bbf-8e30-5f2bba7f74b3" + + +class TestMaxNewVisualizations: + """`min` alone cannot say "use what is already there".""" + + def test_the_ceiling_is_unbounded_when_absent(self): + """Every fixture written before the ceiling existed carries no key, and must keep + passing when the agent authors charts it was not asked for.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_response( + _draft_result(new_visualization_count=5), part, _DC05_EXPECTED, skill_activated=True + ) + assert ev.new_visualizations_met + assert ev.strict_pass + + def test_authoring_beyond_the_ceiling_fails(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "max_new_visualizations": 0} + ev = evaluate_dashboard_response(_draft_result(new_visualization_count=1), part, expected, skill_activated=True) + assert not ev.new_visualizations_met + assert any("at most 0" in f for f in ev.failures) + + def test_a_zero_ceiling_is_a_bound_not_an_absence(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "max_new_visualizations": 0} + assert evaluate_dashboard_response( + _draft_result(new_visualization_count=0), part, expected, skill_activated=True + ).strict_pass + + def test_too_few_and_too_many_are_reported_as_different_diagnoses(self): + """One range message would make whoever reads the failure work out which end was + missed, and the two have opposite fixes.""" + part = _dashboard_part([_widget("Activity by Hour", _ACTIVITY_BY_HOUR)], date_filter=_ALL_TIME_FILTER) + under = evaluate_dashboard_response( + _draft_result(new_visualization_count=0), + part, + {**_DC03_EXPECTED, "max_new_visualizations": 2}, + skill_activated=True, + ) + assert any("at least 1" in f for f in under.failures) + assert not any("at most" in f for f in under.failures) + + def test_a_ceiling_below_the_floor_is_rejected_up_front(self): + client = MagicMock() + expected = {**_DC05_EXPECTED, "min_new_visualizations": 2, "max_new_visualizations": 1} + with pytest.raises(ValueError, match="below min_new_visualizations"): + _run_with(client, expected) + client.send_message.assert_not_called() + + def test_an_unparseable_ceiling_is_rejected_up_front(self): + client = MagicMock() + with pytest.raises(ValueError, match="max_new_visualizations is not a number"): + _run_with(client, {**_DC05_EXPECTED, "max_new_visualizations": "none"}) + client.send_message.assert_not_called() + + +class TestMustNotContain: + def test_a_forbidden_chart_id_on_the_dashboard_fails(self): + part = _dashboard_part( + [ + _widget("Total Customers", _TOTAL_CUSTOMERS), + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("Approval Rate", _SUBSTITUTE), + ], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "must_not_contain": [{"id": _SUBSTITUTE, "title": "Approval Rate"}]} + ev = evaluate_dashboard_response(_draft_result(), part, expected, skill_activated=True) + assert not ev.forbidden_absent + assert not ev.strict_pass + assert any(_SUBSTITUTE in f for f in ev.failures) + + def test_the_check_is_published_only_when_the_case_carries_it(self): + """A check that could not fail is not evidence, and publishing it as passed lifts + `quality_score` above what the run earned.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + plain = evaluate_dashboard_response(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert "forbidden_charts_absent" not in plain.strict_checks + expected = {**_DC05_EXPECTED, "must_not_contain": [{"id": _SUBSTITUTE}]} + guarded = evaluate_dashboard_response(_draft_result(), part, expected, skill_activated=True) + assert guarded.strict_checks["forbidden_charts_absent"] is True + + def test_an_id_entry_does_not_also_forbid_its_title(self): + """Titles are not unique in a real workspace -- the catalog this was written against + holds 102 visualizations whose title contains "approval rate" -- so widening an id + entry to match its title too would fail the case on a different chart.""" + part = _dashboard_part( + [ + _widget("Total Customers", _TOTAL_CUSTOMERS), + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("Approval Rate", _RETURN_CUSTOMERS), + ], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "must_not_contain": [{"id": _SUBSTITUTE, "title": "Approval Rate"}]} + assert evaluate_dashboard_response(_draft_result(), part, expected, skill_activated=True).forbidden_absent + + def test_a_title_only_entry_catches_a_substitute_whose_id_is_not_known_up_front(self): + part = _dashboard_part( + [ + _widget("Total Customers", _TOTAL_CUSTOMERS), + _widget("Active Customers", _ACTIVE_CUSTOMERS), + _widget("approval RATE", _SUBSTITUTE), + ], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "must_not_contain": [{"title": "Approval Rate"}]} + ev = evaluate_dashboard_response(_draft_result(), part, expected, skill_activated=True) + assert not ev.forbidden_absent + assert any(_SUBSTITUTE in f for f in ev.failures) + + def test_an_entry_naming_neither_id_nor_title_is_rejected_up_front(self): + client = MagicMock() + with pytest.raises(ValueError, match="needs an id or a title"): + _run_with(client, {**_DC05_EXPECTED, "must_not_contain": [{"columns": 6}]}) + client.send_message.assert_not_called() + + +class TestAnswerAssertions: + def test_a_missing_phrase_fails_the_case(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "answer_must_include": ["could not be covered"]} + ev = evaluate_dashboard_response( + _draft_result(), part, expected, skill_activated=True, answer_text="Here is your dashboard." + ) + assert not ev.answer_matched + assert not ev.strict_pass + + def test_the_phrase_match_ignores_case_and_whitespace(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "answer_must_include": ["Could Not Be Covered"]} + ev = evaluate_dashboard_response( + _draft_result(), + part, + expected, + skill_activated=True, + answer_text="One widget\ncould not\nbe covered by the catalog.", + ) + assert ev.answer_matched + + def test_a_forbidden_pattern_in_the_answer_fails(self): + """The raw `{visualization/}` token the renderer was meant to resolve. A run + that leaks it builds the right dashboard and still shows the user markup.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + expected = {**_DC05_EXPECTED, "answer_must_not_match": [r"\{visualization/[0-9a-f-]+\}"]} + ev = evaluate_dashboard_response( + _draft_result(), + part, + expected, + skill_activated=True, + answer_text=f"I can use {{visualization/{_SUBSTITUTE}}} for approvals.", + ) + assert not ev.answer_matched + assert any("visualization/" in f for f in ev.failures) + + def test_a_clean_answer_passes_both_kinds_of_assertion(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + expected = { + **_DC05_EXPECTED, + "answer_must_include": ["Total Customers"], + "answer_must_not_match": [r"\{visualization/"], + } + ev = evaluate_dashboard_response( + _draft_result(), + part, + expected, + skill_activated=True, + answer_text="I put Total Customers and Active Customers on the draft.", + ) + assert ev.answer_matched + assert ev.strict_pass + + def test_the_check_is_published_only_when_the_case_carries_it(self): + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + ev = evaluate_dashboard_response(_draft_result(), part, _DC05_EXPECTED, skill_activated=True) + assert "answer_matched" not in ev.strict_checks + + def test_an_invalid_regular_expression_is_rejected_up_front(self): + client = MagicMock() + with pytest.raises(ValueError, match="invalid regular expression"): + _run_with(client, {**_DC05_EXPECTED, "answer_must_not_match": ["([unclosed"]}) + client.send_message.assert_not_called() + + def test_an_empty_phrase_is_rejected_up_front(self): + """`""` is a substring of everything, so it would read as an assertion and be none.""" + client = MagicMock() + with pytest.raises(ValueError, match="asserts nothing"): + _run_with(client, {**_DC05_EXPECTED, "answer_must_include": [" "]}) + client.send_message.assert_not_called() + + @pytest.mark.parametrize("entry", [False, 123, None]) + def test_a_non_string_entry_is_rejected_up_front(self, entry: object) -> None: + """An unquoted `no` in YAML loads as the bool False. Coerced with str() it would + become the assertion "false" -- a fixture that looks strict while checking text the + question never mentions, and passes or fails for reasons nobody wrote down.""" + client = MagicMock() + with pytest.raises(ValueError, match="must be a list of strings"): + _run_with(client, {**_DC05_EXPECTED, "answer_must_include": [entry]}) + client.send_message.assert_not_called() + + @pytest.mark.parametrize("key", ["answer_must_include", "answer_must_not_match"]) + @pytest.mark.parametrize("value", [5, True, "oops"]) + def test_a_non_list_value_is_rejected_up_front(self, key: str, value: object) -> None: + """A truthy non-iterable raises `TypeError` if coerced with `list(...)` before this + check runs, instead of the `ValueError` the docstring promises.""" + client = MagicMock() + with pytest.raises(ValueError, match="must be a list of strings"): + _run_with(client, {**_DC05_EXPECTED, key: value}) + client.send_message.assert_not_called() + + def test_the_text_of_the_turn_that_drafted_is_what_gets_scored(self): + """The draft arrives with the sentence describing it, and that sentence is what the + assertions are about -- so the loop must read every turn, not only the ones that + answer without drafting.""" + part = _dashboard_part( + [_widget("Total Customers", _TOTAL_CUSTOMERS), _widget("Active Customers", _ACTIVE_CUSTOMERS)], + date_filter=_THIS_YEAR_FILTER, + ) + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[_tool_call("draft_dashboard", _draft_result())], + parts=[part], + text="I left one widget out because the catalog has no fraud data.", + ) + expected = {**_DC05_EXPECTED, "answer_must_include": ["no fraud data"]} + summary = _run_with(client, expected) + assert summary.best.evaluation.answer_matched + assert summary.best.evaluation.strict_pass + + +class TestRefusalCases: + """`expects_dashboard: false` -- the agent was asked for something it cannot build.""" + + _NO_DATA = { + "type": "dashboard", + "saved_dashboard_id": None, + "expects_dashboard": False, + "visualizations": [], + "answer_must_include": ["no fraud data"], + } + + def test_an_answer_with_no_draft_passes(self): + ev = evaluate_dashboard_response( + None, + None, + self._NO_DATA, + skill_activated=True, + answer_text="The workspace has no fraud data, so I cannot build that dashboard.", + ) + assert ev.strict_pass + assert ev.failures == [] + + def test_the_document_checks_are_not_published_for_a_refusal(self): + """Reporting six passes for a document that was never produced would make a refusal + read as the best-scoring case in the suite.""" + ev = evaluate_dashboard_response( + None, None, self._NO_DATA, skill_activated=True, answer_text="There is no fraud data here." + ) + assert set(ev.strict_checks) == {"dashboard_not_drafted", "dashboard_skill_activated", "answer_matched"} + + def test_building_anyway_fails(self): + """The defect this shape exists to catch: an agent that proxies the request with + whatever it could find rather than saying it cannot be met.""" + part = _dashboard_part([_widget("Total Customers", _TOTAL_CUSTOMERS)], date_filter=_ALL_TIME_FILTER) + ev = evaluate_dashboard_response( + _draft_result(), part, self._NO_DATA, skill_activated=True, answer_text="There is no fraud data here." + ) + assert not ev.strict_checks["dashboard_not_drafted"] + assert not ev.strict_pass + + def test_falling_over_silently_is_not_a_refusal(self): + """Absence of a draft alone is satisfied by an agent that crashed, which is why the + answer assertion is mandatory for this shape.""" + ev = evaluate_dashboard_response(None, None, self._NO_DATA, skill_activated=True, answer_text="") + assert ev.strict_checks["dashboard_not_drafted"] + assert not ev.answer_matched + assert not ev.strict_pass + + def test_a_rejected_draft_attempt_is_not_a_refusal(self): + """The gap the first two checks leave: `_extract_tool_result` takes only SUCCESSFUL + calls, so an agent whose draft the tool rejected leaves no result and no part and + would read as a clean refusal. It is not one -- it tried to build and a guardrail + stopped it, and scoring that as correct credits the model for the guardrail.""" + ev = evaluate_dashboard_response( + None, + None, + self._NO_DATA, + skill_activated=True, + answer_text="There is no fraud data here.", + producing_tool_called=True, + ) + assert not ev.strict_checks["dashboard_not_drafted"] + assert not ev.strict_pass + assert any("did not succeed" in f for f in ev.failures) + + def test_a_failed_draft_attempt_is_read_off_the_turn_not_the_result(self): + """End to end, because the flag is computed in the run loop rather than passed in by + a fixture: the tool call is present on the turn, its result is an error, so nothing + reaches the scorer except the fact that the call happened.""" + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[_tool_call("draft_dashboard", {"status": "error", "message": "no such metric"})], + text="There is no fraud data in this workspace, so I cannot build that.", + ) + summary = _run_with(client, self._NO_DATA) + assert summary.best.evaluation.producing_tool_called + assert not summary.best.evaluation.strict_pass + + def test_a_rejected_patch_call_counts_too(self): + """A refusal is a creation shape, so the case's selected tool is `draft_dashboard`. + An agent that routed to the editor instead and had its `patch_dashboard` call + rejected leaves the same empty result, so matching only the selected tool would let + it pass.""" + client = MagicMock() + client.send_message.return_value = _chat_result( + tool_calls=[_tool_call("patch_dashboard", {"status": "error", "message": "no such dashboard"})], + text="There is no fraud data in this workspace, so I cannot build that.", + ) + summary = _run_with(client, self._NO_DATA) + assert summary.best.evaluation.producing_tool_called + assert not summary.best.evaluation.strict_pass + + def test_an_answer_alone_still_passes_when_nothing_was_called(self): + """The control for the two above: the flag must not fail a genuine refusal.""" + client = MagicMock() + client.send_message.return_value = _chat_result( + text="There is no fraud data in this workspace, so I cannot build that." + ) + summary = _run_with(client, self._NO_DATA) + assert not summary.best.evaluation.producing_tool_called + assert summary.best.evaluation.strict_pass + + def test_a_refusal_case_without_an_answer_assertion_is_rejected_up_front(self): + client = MagicMock() + expected = {"type": "dashboard", "expects_dashboard": False, "visualizations": []} + with pytest.raises(ValueError, match="needs an answer assertion"): + _run_with(client, expected) + client.send_message.assert_not_called() + + def test_a_refusal_case_listing_charts_is_rejected_up_front(self): + client = MagicMock() + expected = {**self._NO_DATA, "visualizations": [{"id": _TOTAL_CUSTOMERS, "title": "Total Customers"}]} + with pytest.raises(ValueError, match="must not list visualizations"): + _run_with(client, expected) + client.send_message.assert_not_called() + + def test_an_edit_cannot_be_a_refusal_case(self): + client = MagicMock() + expected = {**self._NO_DATA, "type": "dashboardPatch", "saved_dashboard_id": _OVERVIEW_DASHBOARD} + with pytest.raises(ValueError, match="creation shape"): + _run_with(client, expected) + client.send_message.assert_not_called() + + def test_the_simulated_user_never_argues_a_refusal_into_building(self): + """The reply this loop sends restates the charts and asks again, which is pressure to + build the very thing the case says must not be built.""" + client = MagicMock() + client.send_message.return_value = _chat_result(text="The workspace has no fraud data.") + summary = _run_with(client, self._NO_DATA) + assert client.send_message.call_count == 1 + assert summary.best.evaluation.strict_pass diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py new file mode 100644 index 000000000..8655a4fba --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_summary.py @@ -0,0 +1,368 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from unittest.mock import MagicMock, patch + +import gooddata_sdk.table as table_module +import pytest +from gooddata_eval.core.agentic import dashboard_summary as dashboard_summary_module +from gooddata_eval.core.agentic.dashboard_summary import ( + DashboardSummaryAssertionError, + DashboardWidget, + _execute_widget, + _insight_widgets, + build_dashboard_user_context, + evaluate_agentic_dashboard_summary, + run_agentic_dashboard_summary, +) +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.evaluators.base import ItemEvaluation +from gooddata_eval.core.models import ChatResult + +_MODULE = "gooddata_eval.core.agentic.dashboard_summary" + + +def _widget(local_id: str, viz_id: str, title: str = "A chart") -> dict: + return { + "localIdentifier": local_id, + "title": title, + "insight": {"identifier": {"id": viz_id, "type": "visualizationObject"}}, + } + + +def _summary_result(text: str = "## Executive Summary\n- Approvals trended up.") -> ChatResult: + return ChatResult.model_validate({"textResponse": text, "toolCallEvents": [], "reasoningSteps": []}) + + +# ── layout walking ────────────────────────────────────────────────────────── + + +def test_insight_widgets_walks_nested_sections_and_tabs(): + """Widget depth is not fixed: sections nest widgets, and a tab-based dashboard nests + those again, so the walk cannot assume a path.""" + content = { + "version": "3", + "tabs": [ + {"localIdentifier": "tab1", "sections": [{"items": [_widget("w1", "viz-1")]}]}, + {"localIdentifier": "tab2", "sections": [{"items": [_widget("w2", "viz-2"), _widget("w3", "viz-3")]}]}, + ], + } + assert [w.widget_id for w in _insight_widgets(content)] == ["w1", "w2", "w3"] + assert [w.visualization_id for w in _insight_widgets(content)] == ["viz-1", "viz-2", "viz-3"] + + +def test_insight_widgets_skips_widgets_with_nothing_to_execute(): + """Rich text and unresolved widgets carry no insight identifier, so there is nothing to + execute and nothing the summarize scope could use.""" + content = { + "sections": [ + {"items": [_widget("w1", "viz-1"), {"localIdentifier": "rt", "richText": {"content": "hello"}}]}, + {"items": [{"localIdentifier": "broken", "insight": {}}]}, + ] + } + assert [w.widget_id for w in _insight_widgets(content)] == ["w1"] + + +def test_insight_widgets_does_not_repeat_a_widget_reached_twice(): + """The walk descends into every value, so a layout that references one widget from two + places must still yield it once -- a duplicate would be executed twice.""" + shared = _widget("w1", "viz-1") + content = {"sections": [{"items": [shared]}], "alsoHere": {"items": [shared]}} + assert [w.widget_id for w in _insight_widgets(content)] == ["w1"] + + +def test_insight_widget_falls_back_to_the_visualization_id_when_unnamed(): + content = {"items": [{"insight": {"identifier": {"id": "viz-1"}}}]} + (widget,) = _insight_widgets(content) + assert widget.widget_id == "viz-1" + assert widget.title == "viz-1" + + +# ── context building ──────────────────────────────────────────────────────── + + +def _patched_dashboard(widgets: list[dict], title: str = "Sales"): + return patch(f"{_MODULE}._fetch_dashboard", return_value=(title, {"sections": [{"items": widgets}]})) + + +def test_build_user_context_carries_a_result_id_per_widget(): + sdk = MagicMock() + with ( + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", "res-2"]), + ): + context, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1") + + dashboard = context["view"]["dashboard"] + assert dashboard["id"] == "dash-1" + assert dashboard["title"] == "Sales" + assert [w["resultId"] for w in dashboard["widgets"]] == ["res-1", "res-2"] + assert [w["widgetType"] for w in dashboard["widgets"]] == ["insight", "insight"] + assert [w.result_id for w in widgets] == ["res-1", "res-2"] + + +def test_a_widget_that_fails_to_execute_is_reported_but_left_out_of_the_context(): + """The skill drops any widget without a result_id, so sending one would be a silent + no-op -- but the report still has to show the summary covered less than the dashboard.""" + sdk = MagicMock() + with ( + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", RuntimeError("execution blew up")]), + ): + context, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1") + + assert [w["widgetId"] for w in context["view"]["dashboard"]["widgets"]] == ["w1"] + assert len(widgets) == 2 + assert widgets[1].result_id is None + + +def test_max_widgets_caps_the_executions(): + """A thirty-widget dashboard costs thirty executions per item; the cap bounds that.""" + sdk = MagicMock() + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(5)] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + _, widgets = build_dashboard_user_context(sdk, "http://h", "tok", "ws1", "dash-1", max_widgets=2) + + assert len(widgets) == 2 + assert execute.call_count == 2 + + +# ── runs ──────────────────────────────────────────────────────────────────── + + +def _run(chat_side_effect, *, k=1, passed=True, widgets=2): + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, k + 2)] + client.send_message.side_effect = chat_side_effect + evaluation = ItemEvaluation(passed=passed, rank_key=(int(passed), 1.0), detail={"actual_output": "text"}) + evaluator = MagicMock() + evaluator.evaluate.return_value = evaluation + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(widgets)] + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + return run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=k, + ) + + +def test_a_passing_run_records_how_much_of_the_dashboard_it_covered(): + summary = _run([_summary_result()]) + assert summary.pass_at_k is True + assert summary.best.widgets_total == 2 + assert summary.best.widgets_executed == 2 + + +def test_widgets_are_executed_once_and_reused_across_k_runs(): + """Result ids identify cached executions, so re-running them per K would multiply the + most expensive part of the item without changing what the assistant sees.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation(passed=True, rank_key=(1, 1.0), detail={"actual_output": "t"}) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1")]), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + summary = run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=3, + ) + + assert len(summary.run_results) == 3 + assert execute.call_count == 1 + + +def test_a_chat_error_is_recorded_on_the_run_rather_than_raised(): + """Raising would discard the K-runs already completed and leave the item with no + verdict at all -- the same failure mode fixed for the other agentic kinds.""" + summary = _run([ChatError("gen-ai fell over")]) + + assert summary.pass_at_k is False + run = summary.run_results[0] + assert run.chat_error is not None + assert "gen-ai fell over" in run.chat_error + assert run.passed is False + assert run.evaluation.error is not None + + +def test_a_chat_error_on_a_later_run_does_not_discard_the_earlier_one(): + summary = _run([_summary_result(), ChatError("boom")], k=2) + + assert len(summary.run_results) == 2 + assert summary.run_results[0].chat_error is None + assert summary.run_results[1].chat_error is not None + assert summary.pass_at_k is True # run 0 still counts + # The chat-error run is unscored, so it cannot certify that every run passed. + assert summary.pass_power_k is False + + +def test_the_best_run_is_never_a_chat_error_when_a_graded_run_exists(): + summary = _run([ChatError("boom"), _summary_result()], k=2) + assert summary.best.chat_error is None + + +# ── evaluate_* wrapper ────────────────────────────────────────────────────── + + +def _evaluate(passed: bool): + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation( + passed=passed, rank_key=(int(passed), 1.0), detail={"actual_output": "text", "include_0": passed} + ) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1"), _widget("w2", "viz-2")]), + patch(f"{_MODULE}._execute_widget", side_effect=["res-1", RuntimeError("dead")]), + ): + return evaluate_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + ) + + +def test_detail_reports_coverage_and_the_latency_breakdown(): + outcome = _evaluate(passed=True) + # A rubric naming a widget that never executed fails for a reason that is not the + # agent's, so the ratio has to be visible without opening the trace. + assert outcome.detail["widgets_total"] == 2 + assert outcome.detail["widgets_executed"] == 1 + assert "latency_breakdown" in outcome.detail + assert outcome.detail["include_0"] is True + assert outcome.conversation_id == "conv-1" + + +def test_a_failing_item_raises_with_the_same_detail_attached(): + with pytest.raises(DashboardSummaryAssertionError) as exc_info: + _evaluate(passed=False) + + error = exc_info.value + assert "1/2 widgets executed" in str(error) + assert error.detail["widgets_executed"] == 1 + assert error.runs_passed == 0 + assert error.runs_effective == 1 + assert error.conversation_id == "conv-1" + + +def test_dashboard_widget_defaults_to_no_result(): + assert DashboardWidget(widget_id="w", title="t", visualization_id="v").result_id is None + + +def test_only_visualizations_keeps_just_the_named_charts(): + """A fixture asserting on a handful of charts should not pay to execute thirty. Names + are matched against the visualization id and the widget id, since a fixture author + reading a dashboard's AAC sees both.""" + sdk = MagicMock() + layout = [_widget("w0", "viz-0"), _widget("w1", "viz-1"), _widget("w2", "viz-2")] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res") as execute, + ): + _, widgets = build_dashboard_user_context( + sdk, "http://h", "tok", "ws1", "dash-1", only_visualizations=["viz-1", "w2"] + ) + + assert [w.widget_id for w in widgets] == ["w1", "w2"] + assert execute.call_count == 2 + + +def test_only_visualizations_and_max_widgets_compose(): + sdk = MagicMock() + layout = [_widget(f"w{i}", f"viz-{i}") for i in range(4)] + with ( + _patched_dashboard(layout), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + _, widgets = build_dashboard_user_context( + sdk, + "http://h", + "tok", + "ws1", + "dash-1", + only_visualizations=["viz-1", "viz-2", "viz-3"], + max_widgets=2, + ) + + assert [w.widget_id for w in widgets] == ["w1", "w2"] + + +def test_a_failed_conversation_creation_does_not_discard_completed_runs(): + """create_conversation sat outside any handler, so a transient failure on run 2 of 3 + threw away run 1 -- the same failure mode the ChatError path already guards.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", RuntimeError("no conversation for you"), "conv-3"] + client.send_message.return_value = _summary_result() + evaluator = MagicMock() + evaluator.evaluate.return_value = ItemEvaluation(passed=True, rank_key=(1, 1.0), detail={"actual_output": "t"}) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.GoodDataSdk"), + patch(f"{_MODULE}.DashboardSummaryEvaluator", return_value=evaluator), + _patched_dashboard([_widget("w1", "viz-1")]), + patch(f"{_MODULE}._execute_widget", return_value="res"), + ): + summary = run_agentic_dashboard_summary( + host="http://h", + token="tok", + workspace_id="ws1", + dashboard_id="dash-1", + expected_output={"must_include": ["x"]}, + k=3, + ) + + assert len(summary.run_results) == 3 + assert [r.chat_error is None for r in summary.run_results] == [True, False, True] + assert "conversation creation failed" in summary.run_results[1].chat_error + assert summary.pass_at_k is True + # The lost run has no conversation to score, so it must not certify an all-passed item. + assert summary.pass_power_k is False + + +def test_execute_widget_fails_loudly_when_the_sdk_moves_its_private_helpers(): + """gooddata-sdk is depended on as ~=1.76.0, so a patch release may rename the private + helpers this borrows. Without the check that surfaces as an AttributeError on the first + widget of a run rather than as the dependency problem it is.""" + stripped = MagicMock(spec=[]) # a gooddata_sdk.table exposing none of the three helpers + # Both bindings: `import a.b as c` reads the parent package's attribute, while + # sys.modules is what keeps a re-import from restoring the real one. + # The module-level binding is what _execute_widget reads now that the import is at the + # top of the file, so patching sys.modules alone would no longer reach it. + with ( + patch.object(dashboard_summary_module, "table_module", stripped), + pytest.raises(RuntimeError, match="no longer provides"), + ): + _execute_widget(MagicMock(), "ws1", "viz-1") + + +def test_execute_widget_guard_passes_against_the_installed_sdk(): + """The guard must not be a permanent tripwire: the helpers exist in the pinned version, + so a real call gets past it and fails (if at all) on the network, not on the check.""" + for name in ("_vis_is_table", "_get_exec_for_pivot", "get_exec_for_non_pivot"): + assert hasattr(table_module, name), f"gooddata_sdk.table.{name} is gone -- update _execute_widget" diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index f28c0fae6..8682c9bec 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -98,10 +98,19 @@ def test_dispatch_agentic_omits_agent_id_by_default(): ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), ("agentic_kda_skill", {"Measure": {"type": "metric", "id": "revenue"}}, "evaluate_agentic_kda_skill"), ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ( + "agentic_dashboard_summary", + {"must_include": ["States the overall trend."]}, + "evaluate_agentic_dashboard_summary", + ), ("agentic_anomaly_detection", {"metric": "metric/spend"}, "evaluate_agentic_anomaly_detection"), ("agentic_what_if", {"metric_id": "spend"}, "evaluate_agentic_what_if"), ] +# The one kind whose dispatch needs more than question/expected_output: the dashboard to +# summarize is named in summary_input, exactly as for the single-shot dashboard_summary. +_SUMMARY_INPUT_KINDS = {"agentic_dashboard_summary"} + def test_all_agentic_kind_cases_covers_every_registered_kind(): """Guards the two parametrized tests below against silently going stale: a kind added @@ -119,6 +128,7 @@ def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_ test_kind=kind, question="q", expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: _dispatch_agentic( @@ -260,6 +270,7 @@ def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_o test_kind=kind, question="q", expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) canned = AgenticEvalOutcome(reasoning_steps=["x"], conversation_id="c1", response_id="r1", detail={"k": "v"}) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned) as mock_eval: @@ -831,6 +842,7 @@ def test_an_errored_item_without_timings_keeps_its_zero_defaults(): ("agentic_what_if", "evaluate_agentic_what_if", {}), ("agentic_anomaly_detection", "evaluate_agentic_anomaly_detection", {}), ("agentic_document_skill", "evaluate_agentic_document_skill", {}), + ("agentic_dashboard_summary", "evaluate_agentic_dashboard_summary", {}), ("agentic_conversation", "evaluate_agentic_conversation", {"id": "c1", "expected_skills": [], "turns": []}), ] @@ -853,6 +865,7 @@ def test_dispatch_agentic_passes_user_context_through_to_every_kind( question="What does the visualization I attached show?", expected_output=expected_output, user_context=user_context, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, ) with patch(f"gooddata_eval.cli.agentic_runner.{evaluator}") as mock_eval: _dispatch_agentic( @@ -865,9 +878,18 @@ def test_dispatch_agentic_passes_user_context_through_to_every_kind( run_ts="2026-01-01", model_version_override=None, ) + if kind in _KINDS_WITHOUT_USER_CONTEXT: + assert "user_context" not in mock_eval.call_args.kwargs + return assert mock_eval.call_args.kwargs["user_context"] == user_context +# agentic_dashboard_summary builds its own context from the dashboard it summarises and +# takes its scope from summary_input, so relaying the item's context would override the +# thing under test rather than add to it. Excluded on purpose, not by omission. +_KINDS_WITHOUT_USER_CONTEXT = {"agentic_dashboard_summary"} + + class _Stop(Exception): """Raised by a patched collaborator to end a run once the call under test is captured.""" @@ -1004,7 +1026,14 @@ def test_failed_runs_reaches_the_report_for_every_kind(kind, expected_output, ta field empty on every item (confirmed live: 135 agentic_guardrail results, all with `failed_runs: []`, including 16 items that passed 1 of 3 runs). A kind whose evaluator stops attaching them fails here rather than quietly reporting nothing.""" - item = DatasetItem(id="q1", dataset_name="ds", test_kind=kind, question="q", expected_output=expected_output) + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + ) canned = AgenticEvalOutcome(reasoning_steps=["x"], detail={"k": "v"}, failed_runs=[_A_FAILED_RUN]) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned): report = run_agentic_items([item], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") @@ -1140,7 +1169,14 @@ def test_every_evaluator_hands_out_its_run_latencies(kind, expected_output, targ @pytest.mark.parametrize(("kind", "expected_output", "target"), _ALL_AGENTIC_KIND_CASES) def test_run_latencies_reach_the_report_for_every_kind(kind, expected_output, target): - item = DatasetItem(id="q1", dataset_name="ds", test_kind=kind, question="q", expected_output=expected_output) + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + ) canned = AgenticEvalOutcome(reasoning_steps=["x"], detail={"k": "v"}, run_latencies_s=[4.5, 6.0]) with patch(f"gooddata_eval.cli.agentic_runner.{target}", return_value=canned): report = run_agentic_items([item], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") @@ -1204,7 +1240,14 @@ def evaluate(*args: Any, **kwargs: Any) -> Any: def test_every_kind_gets_its_chat_turns_recorded_as_trace_latencies(kind, expected_output, target): """The recorder wraps the item, so a kind needs no code of its own to be covered -- which is the point: nothing a new evaluator forgets can drop it out of the per-trace percentiles.""" - item = DatasetItem(id="q1", dataset_name="ds", test_kind=kind, question="q", expected_output=expected_output) + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + summary_input={"dashboard_id": "dash-1"} if kind in _SUMMARY_INPUT_KINDS else None, + ) with patch(f"gooddata_eval.cli.agentic_runner.{target}", side_effect=_noting(1.5, 2.5, 0.5)): report = run_agentic_items([item], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") assert report.items[0].trace_latencies_s == [1.5, 2.5, 0.5] diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py index 7c7c0e0e1..5a230f3be 100644 --- a/packages/gooddata-eval/tests/test_trace_linker.py +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -120,6 +120,7 @@ def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): ("visualization", "evaluate_agentic_visualization"), ("kda_skill", "evaluate_agentic_kda_skill"), ("conversation", "evaluate_agentic_conversation"), + ("dashboard_summary", "evaluate_agentic_dashboard_summary"), ("anomaly_detection", "evaluate_agentic_anomaly_detection"), ("what_if", "evaluate_agentic_what_if"), ]