From 03b854d1c06b425abb920d238eaf4347d976dfac Mon Sep 17 00:00:00 2001 From: Dan Wolfson Date: Sun, 8 Nov 2026 19:56:01 +0000 Subject: [PATCH 1/2] feat(overview-metrics): totals count the whole repository (push-down) by default Owner decision O-cap-1 (2026-11-08): report-spec and analytic totals count with population="repository" (push_down=True) -- the whole repository, including elements the viewer may not be able to see -- as a deliberate interim choice until zone-aware push-down exists. Live: Asset 0.9 s instead of a 30 s timeout. - count_elements, count_elements_by_property, count_relationships, counts_by_type, people_counts, usage_context_counts, feedback_summary, business_value_signals and growth_series default to "repository"; "viewer" stays an explicit opt-in, and scope=None still reaches the server default. - A failed repository count is None with an unmeasured note, strict or not: it has no find fallback, and 0 would read as an empty catalog. - count_relationships uses ce itself as the native counter when it has the method (EgeriaTech does); semantic_grounding passes expert=mgr (R4). - One population per payload, and no ratio mixes populations: term_definition_completeness, context_readiness_funnel and business_value_signals now count every figure natively (count_elements_by_property gains operator="NEQ": description != ""), replacing capped scans whose totals were the page size (live: 500 terms instead of 1061). certifications_summary, orphan_glossary_terms, stale_assets, contextualised_coverage and solution_component_realisation keep their scan-backed payloads wholly viewer-scoped, explicitly. - Declarations: each measure's population says "repository", the population param defaults to it, and descriptions say the totals include elements the viewer may not be able to see. - The native-method cache no longer hands a cached method name to a client of the same class name that lacks it. Signed-off-by: Dan Wolfson --- pyegeria/view/analytic_demo_specs.py | 27 +- pyegeria/view/analytic_registry.py | 181 +++++----- pyegeria/view/overview_metrics.py | 315 +++++++++--------- .../test_capped_count_not_measured.py | 53 ++- .../test_issue_123_strict_counts.py | 27 +- tests/micro-tests/test_overview_metrics.py | 285 ++++++++++------ .../test_rs_w2_3_inputs_by_role.py | 7 +- 7 files changed, 526 insertions(+), 369 deletions(-) diff --git a/pyegeria/view/analytic_demo_specs.py b/pyegeria/view/analytic_demo_specs.py index c7fc8cc5..1cba4361 100644 --- a/pyegeria/view/analytic_demo_specs.py +++ b/pyegeria/view/analytic_demo_specs.py @@ -116,10 +116,10 @@ def _demo( "Analytic Demo - Business Value Signals": _demo( heading="Business Value Signals", what="Real signals behind the Overview dashboard's four Business Value tiles " - "(NEXT-9): confidentialCount and describedCount come from a single " - "Asset-hierarchy fetch (assetTotal is that fetch's size, assetCapped=true " - "if it hit DEFAULT_CAP); duplicateCount is a separate ConsolidatedDuplicate " - "classification count. Each is a documented proxy, not a direct measure -- " + "(NEXT-9): assetTotal, confidentialCount (Assets classified Confidentiality), " + "describedCount (Assets whose description is set and not empty) and duplicateCount " + "(ConsolidatedDuplicate elements) are native counts over the whole repository, " + "including elements the viewer may not be able to see. Each is a documented proxy, not a direct measure -- " "see business_value_signals()'s docstring for the causal-claim caveat that " "belongs with each field before treating it as more than that.", analytic_function_name="business_value_signals", @@ -181,7 +181,8 @@ def _demo( "Analytic Demo - Semantic Grounding": _demo( heading="Semantic Grounding Coverage", what="groundingLinks = count of SemanticAssignment relationships (term<->asset). " - "groundingPct = that count as a percent of the broad Asset supertype count " + "groundingPct = that count as a percent of the broad Asset supertype count; both are " + "native counts over the whole repository, including elements the viewer may not see " "(i.e. 'percent of Assets' as the denominator, capped at 100 -- an asset with " "multiple assignments can push the raw ratio over 100%, which is why it's capped, " "not because grounding coverage is literally bounded there). A per-asset detail " @@ -193,12 +194,13 @@ def _demo( ), "Analytic Demo - AI Context Readiness": _demo( heading="AI Context Readiness Funnel", - what="'cataloged' (Asset supertype count), 'documented' (Assets with a non-empty " - "description, capped at DEFAULT_CAP -- a floor, not exact, on a large catalog), " + what="'cataloged' (Asset supertype count), 'documented' (Assets whose description is " + "set and not empty), " "'classified' (elements matching ANY of the same governance classifications " "governed_coverage uses), and 'lineage' (count of DataFlow relationships -- design/ " - "business lineage, distinct from OpenLineage's operational lineage) are all computed " - "as of 2026-08-01. 'aiReady' stays None from this function ALWAYS, by design -- these " + "business lineage, distinct from OpenLineage's operational lineage) are all native counts " + "over the whole repository (including elements the viewer may not be able to see), " + "computed as of 2026-08-01. 'aiReady' stays None from this function ALWAYS, by design -- these " "four are independent counts, not a per-asset check, so they can't answer 'how many " "assets are ALL of these at once'. See 'Analytic Demo - AI-Ready Assets' (below) for " "that -- a genuinely different function, not a missing field on this one.", @@ -226,7 +228,8 @@ def _demo( "Analytic Demo - Catalog Growth Trend": _demo( heading="Catalog Growth Trend (6 Month)", what="Snapshot-based growth series -- assets/terms/governed/products, one point " - "per window step. Run with output_format SERIES for a Vega-Lite line chart.", + "per window step, each a native count over the whole repository (including elements " + "the viewer may not be able to see). Run with output_format SERIES for a Vega-Lite line chart.", analytic_function_name="growth_series", demo_params={"window": "6mo"}, result_shape="list[dict] (time series: {label, date, : count, ...})", @@ -234,7 +237,9 @@ def _demo( ), "Analytic Demo - Term Definition Completeness": _demo( heading="Term Definition Completeness", - what="Share of GlossaryTerms carrying a non-empty description. undefinedPct is the gap " + what="Share of GlossaryTerms whose description is set and not empty; total and defined are " + "native counts over the whole repository (including terms the viewer may not be able to " + "see). undefinedPct is the gap " "(not the coverage) -- a near-100%-defined glossary is the uninteresting case, and " "the gap is what's actionable.", analytic_function_name="term_definition_completeness", diff --git a/pyegeria/view/analytic_registry.py b/pyegeria/view/analytic_registry.py index d3e9e62a..a30bc780 100644 --- a/pyegeria/view/analytic_registry.py +++ b/pyegeria/view/analytic_registry.py @@ -204,8 +204,10 @@ class MeasureDeclaration(BaseModel): `as_of` argument (None = when it ran), for a routine whose result carries no time of its own. `completeness_field` names the result field that says the value may be incomplete (one of COMPLETENESS, or a boolean: true = only a lower bound). `capped_field` is its old name, read - with a DeprecationWarning. `population` is set when the value is always counted in one - population; empty when the caller chooses it or the server default applies.""" + with a DeprecationWarning. `population` is the population the value is counted in: always, + or by default when the routine takes a `population` parameter (the caller may pass the other + one). Empty when the server default applies. A `repository` value includes elements the viewer + may not be able to see.""" name: str unit: str # in words, plural where it counts things: "tables", "fraction of rows (0-1)" as_of: str # the result field saying when this was measured, e.g. "asOf"; or run.as_of @@ -764,9 +766,16 @@ def _strict() -> AnalyticParam: def _population() -> AnalyticParam: - return AnalyticParam(name="population", type="str", - description="viewer (what the caller may see) or repository (whole repository, " - "fast; native count only). Omit for the server default.") + return AnalyticParam(name="population", type="str", default="repository", + description="repository (the default: the whole repository, including elements " + "the caller may not be able to see -- fast, native count only) or " + "viewer (only what the caller may see; can take minutes for a large type).") + + +#: Said by every routine whose totals count the whole repository (owner decision 2026-11-08, an interim +#: choice until zone-aware push-down exists). +REPOSITORY_WORDS = (" Its totals count the whole repository (population=repository), including elements you may not be able to " + "see (a deliberate interim choice until zone-aware push-down exists).") # ── Declaration literals (RS-W2-1a). Short builders so each entry below reads as its declaration. ── @@ -842,7 +851,7 @@ def _always_applies(what: str) -> str: "Cost: one native count, sub-second in the repository population (0.2-0.8 s " "measured); a viewer-population count reads every element and can take minutes " "for a large type (Process: 120 s); a server with no native count falls back to " - "a capped find.", + "a capped find." + REPOSITORY_WORDS, returns="scalar (int)", generic=True, params=[ @@ -856,7 +865,7 @@ def _always_applies(what: str) -> str: ], declaration=AnalyticDeclaration( result_kind="measure", - measures=[_measure("elements", "elements", "value")], + measures=[_measure("elements", "elements", "value", population="repository")], absence=_absence( not_measured="The elements were not counted: the count query failed or timed out, or " "the server could not answer it in the population asked for. A fallback " @@ -876,7 +885,7 @@ def _always_applies(what: str) -> str: inputs=["egeria_client:MetadataExpert"], description="Per-type element counts for a set of (label, type_name) pairs -- a breakdown/" "composition metric, e.g. the assets-by-type chart. " - "Cost: one native count per type, sub-second each in the repository population; a viewer-population count can take minutes for a large type.", + "Cost: one native count per type, sub-second each in the repository population; a viewer-population count can take minutes for a large type." + REPOSITORY_WORDS, returns="list[dict] (breakdown: {label, type, count})", generic=True, params=[ @@ -888,7 +897,7 @@ def _always_applies(what: str) -> str: ], declaration=AnalyticDeclaration( result_kind="distribution", - measures=[_measure("count", "elements", "count")], + measures=[_measure("count", "elements", "count", population="repository")], # The caller picks the types; a supertype and its subtype may both be listed. categories=CategoriesDeclaration(exclusive=False), absence=_absence( @@ -988,7 +997,7 @@ def _always_applies(what: str) -> str: "Confidentiality-classified asset count, described-asset count, " "ConsolidatedDuplicate-flagged element count -- all proxies with a " "documented causal-claim caveat, not direct measures (NEXT-9). " - "Cost: one capped Asset find plus one count, seconds.", + "Cost: four native counts, sub-second each in the repository population." + REPOSITORY_WORDS, returns="dict (assetTotal, assetCapped, confidentialCount, describedCount, duplicateCount)", generic=False, binding_note="Fixed to the Asset type hierarchy and Confidentiality/ConsolidatedDuplicate " @@ -997,28 +1006,28 @@ def _always_applies(what: str) -> str: declaration=AnalyticDeclaration( result_kind="measure", measures=[ - _measure("assets_read", "Assets", "assetTotal", completeness_field="assetCapped", - population="viewer"), + _measure("assets", "Assets", "assetTotal", completeness_field="assetCapped", + population="repository"), _measure("confidential_assets", "Assets carrying a Confidentiality classification", - "confidentialCount", completeness_field="assetCapped", population="viewer", - basis=_basis("assetTotal", "Assets read")), - _measure("described_assets", "Assets with a description", "describedCount", - completeness_field="assetCapped", population="viewer", - basis=_basis("assetTotal", "Assets read")), + "confidentialCount", completeness_field="assetCapped", population="repository", + basis=_basis("assetTotal", "Assets")), + _measure("described_assets", "Assets with a description (set, not empty)", "describedCount", + completeness_field="assetCapped", population="repository", + basis=_basis("assetTotal", "Assets")), _measure("duplicate_elements", "elements of any type carrying ConsolidatedDuplicate", - "duplicateCount"), + "duplicateCount", population="repository"), ], absence=_absence( - not_measured="The business-value signals were not measured: the Asset find or the " - "duplicate count failed.", + not_measured="The business-value signals were not measured: their counts failed, or " + "the server could not answer them in the population asked for.", not_permitted="You are not permitted to read the Assets, so the business-value " "signals were not measured.", not_applicable=_always_applies("The business-value signals"), nothing_found="Counted: there are no active Assets and no elements marked as duplicates.", ), out=[ - _field("assetTotal", "Assets read", "int", unit="Assets"), - _field("assetCapped", "At least (page cap reached)", "bool"), + _field("assetTotal", "Assets", "int", unit="Assets"), + _field("assetCapped", "At least (always false: a native count is complete)", "bool"), _field("confidentialCount", "Confidential Assets", "int", unit="Assets"), _field("describedCount", "Described Assets", "int", unit="Assets"), _field("duplicateCount", "Duplicates", "int", unit="elements"), @@ -1065,7 +1074,7 @@ def _always_applies(what: str) -> str: function="pyegeria.view.overview_metrics.people_counts", inputs=["egeria_client:MetadataExpert"], description="Counts of Person / Team / Organization / ITProfile / Community actor profiles. " - "Cost: five native counts, sub-second each in the repository population.", + "Cost: five native counts, sub-second each in the repository population." + REPOSITORY_WORDS, returns="dict (persons, teams, organizations, itProfiles, communities)", generic=False, binding_note="Fixed to Person/Team/Organization/ITProfile/Community -- not a parameter.", @@ -1073,11 +1082,11 @@ def _always_applies(what: str) -> str: declaration=AnalyticDeclaration( result_kind="measure", measures=[ - _measure("persons", "Person profiles", "persons"), - _measure("teams", "Team profiles", "teams"), - _measure("organizations", "Organization profiles", "organizations"), - _measure("it_profiles", "ITProfile profiles", "itProfiles"), - _measure("communities", "Community profiles", "communities"), + _measure("persons", "Person profiles", "persons", population="repository"), + _measure("teams", "Team profiles", "teams", population="repository"), + _measure("organizations", "Organization profiles", "organizations", population="repository"), + _measure("it_profiles", "ITProfile profiles", "itProfiles", population="repository"), + _measure("communities", "Community profiles", "communities", population="repository"), ], absence=_absence( not_measured="The actor profiles were not counted: the count queries failed, or the " @@ -1102,7 +1111,8 @@ def _always_applies(what: str) -> str: inputs=["egeria_client:ClassificationExplorer"], description="Crowd-sourced feedback counts (ratings/comments/likes/tags/noteLogs) by type, " "via Collaboration OMAS relationships. " - "Cost: five relationship pages (native counts when an expert client is passed), seconds.", + "Cost: five native counts, sub-second each in the repository population; population=viewer " + "without a MetadataExpert reads five capped relationship pages instead." + REPOSITORY_WORDS, returns="dict (byType, total)", generic=False, binding_note="Fixed to Collaboration OMAS's feedback relationship types -- not a parameter.", @@ -1110,16 +1120,17 @@ def _always_applies(what: str) -> str: declaration=AnalyticDeclaration( result_kind="measure", measures=[ - _measure("feedback", "feedback relationships", "total"), - _measure("ratings", "AttachedRating relationships", "byType.ratings"), - _measure("comments", "AttachedComment relationships", "byType.comments"), - _measure("likes", "AttachedLike relationships", "byType.likes"), - _measure("tags", "AttachedTag relationships", "byType.tags"), - _measure("note_logs", "AttachedNoteLog relationships", "byType.noteLogs"), + _measure("feedback", "feedback relationships", "total", population="repository"), + _measure("ratings", "AttachedRating relationships", "byType.ratings", population="repository"), + _measure("comments", "AttachedComment relationships", "byType.comments", population="repository"), + _measure("likes", "AttachedLike relationships", "byType.likes", population="repository"), + _measure("tags", "AttachedTag relationships", "byType.tags", population="repository"), + _measure("note_logs", "AttachedNoteLog relationships", "byType.noteLogs", population="repository"), ], absence=_absence( not_measured="Feedback was not counted: a relationship query failed, or a repository " - "count was asked for without a native count to answer it.", + "count was asked for and no native count could answer it (the client " + "has none, or it failed or timed out).", not_permitted="You are not permitted to read feedback relationships.", not_applicable=_always_applies("A feedback summary"), nothing_found="Counted: nothing in the catalog has been rated, commented on, liked, " @@ -1138,7 +1149,7 @@ def _always_applies(what: str) -> str: description="InformationSupplyChain and SolutionBlueprint counts -- the two structures that " "put assets in a business usage context (supply chain membership, blueprint " "realization). " - "Cost: two native counts, sub-second each in the repository population.", + "Cost: two native counts, sub-second each in the repository population." + REPOSITORY_WORDS, returns="dict (informationSupplyChains, blueprints)", generic=False, binding_note="Fixed to InformationSupplyChain and SolutionBlueprint -- not a parameter.", @@ -1146,8 +1157,8 @@ def _always_applies(what: str) -> str: declaration=AnalyticDeclaration( result_kind="measure", measures=[ - _measure("supply_chains", "InformationSupplyChain elements", "informationSupplyChains"), - _measure("blueprints", "SolutionBlueprint elements", "blueprints"), + _measure("supply_chains", "InformationSupplyChain elements", "informationSupplyChains", population="repository"), + _measure("blueprints", "SolutionBlueprint elements", "blueprints", population="repository"), ], absence=_absence( not_measured="Supply chains and blueprints were not counted: the count queries failed, " @@ -1170,7 +1181,9 @@ def _always_applies(what: str) -> str: description="SemanticAssignment relationship count and coverage percent -- the meaning layer " "that grounds AI. Takes two leading clients (mgr, ce) -- the executor supplies the " "same EgeriaTech instance for both. " - "Cost: one count plus one capped relationship page, seconds.", + "Both counts are native, in the repository population, so the percentage never divides " + "one population by the other. " + "Cost: two native counts, sub-second each." + REPOSITORY_WORDS, returns="dict (groundingLinks, groundingPct)", generic=False, binding_note="Fixed to the SemanticAssignment relationship type -- not a parameter.", @@ -1179,15 +1192,15 @@ def _always_applies(what: str) -> str: result_kind="measure", measures=[ _measure("grounding_links", "SemanticAssignment relationships", "groundingLinks", - population="viewer"), + population="repository"), # links / Assets, not the share of Assets with a link: one Asset may carry several. _measure("grounding_percent", "percent: SemanticAssignment links per hundred Assets, " - "at most 100", "groundingPct"), + "at most 100", "groundingPct", population="repository"), ], absence=_absence( not_measured="Semantic grounding was not measured: the SemanticAssignment count or the " - "Asset count to divide by failed, timed out, or stopped at its page cap " - "(a lower bound, never a total).", + "Asset count to divide by failed or timed out, or the server could not " + "answer it in the repository population.", not_permitted="You are not permitted to read the semantic assignments.", not_applicable=_always_applies("Semantic grounding"), nothing_found="Counted: no glossary term is assigned to anything.", @@ -1208,24 +1221,26 @@ def _always_applies(what: str) -> str: "aiReady stays None always by design -- it's four independent counts, and aiReady " "needs a true cross-criteria intersection instead. Pair with ai_ready_assets " "(below) for that -- overview_handler.py calls both and merges the result. " - "Cost: two counts, one capped Asset find and one relationship page, seconds.", + "Every stage is a native count in the repository population. " + "Cost: four native counts, sub-second each." + REPOSITORY_WORDS, returns="dict (cataloged, documented, classified, lineage, aiReady)", generic=False, binding_note="Fixed 5-stage readiness definition baked into the function body -- not a parameter.", params=[_as_of()], declaration=AnalyticDeclaration( result_kind="measure", - # Four independent counts over different populations, not nested stages of one set. + # Four independent native counts in one population, not nested stages of one set. measures=[ - _measure("cataloged", "Assets", "cataloged"), - _measure("documented", "Assets with a description, among those one capped find returned", - "documented", population="viewer"), + _measure("cataloged", "Assets", "cataloged", population="repository"), + _measure("documented", "Assets with a description (set, not empty)", "documented", + population="repository"), _measure("classified", "elements of any type carrying a governance classification", - "classified"), - _measure("lineage", "DataFlow relationships", "lineage", population="viewer"), + "classified", population="repository"), + _measure("lineage", "DataFlow relationships", "lineage", population="repository"), ], absence=_absence( - not_measured="The readiness funnel was not measured: its counts or the Asset find failed.", + not_measured="The readiness funnel was not measured: its counts failed, or the server " + "could not answer them in the repository population.", not_permitted="You are not permitted to read the Assets, so the readiness funnel was " "not measured.", not_applicable=_always_applies("The readiness funnel"), @@ -1336,7 +1351,8 @@ def _always_applies(what: str) -> str: description="Snapshot-based growth series (assets/terms/governed/products by default) via " "asOfTime, one point per window step. The one shape SERIES output format " "(a Vega-Lite line chart) is proven against today. " - "Cost: one count per series per snapshot (four series by default), seconds to tens of seconds.", + "Cost: one native count per series per snapshot (four series by default), sub-second each " + "in the repository population." + REPOSITORY_WORDS, returns="list[dict] (time series: {label, date, : count, ...})", # Generic: what it counts is type_map. (Its binding_note used to repeat type_map's own # description; a generic routine fixes nothing, so it declares no note -- RS-W2-2.) @@ -1348,16 +1364,17 @@ def _always_applies(what: str) -> str: description="Override the number of snapshots (2-24)."), AnalyticParam(name="type_map", type="list[tuple[str, str, list[str] | None]]", description="Override which series to snapshot; defaults to assets/terms/governed/products."), + _population(), ], declaration=AnalyticDeclaration( result_kind="series", # One row per snapshot, oldest first; the measures are the default type_map's keys. measures=[ - _measure("assets", "Assets", "assets", as_of="date"), - _measure("terms", "GlossaryTerm elements", "terms", as_of="date"), + _measure("assets", "Assets", "assets", as_of="date", population="repository"), + _measure("terms", "GlossaryTerm elements", "terms", as_of="date", population="repository"), _measure("governed", "elements of any type carrying a governance classification", - "governed", as_of="date"), - _measure("products", "DigitalProduct elements", "products", as_of="date"), + "governed", as_of="date", population="repository"), + _measure("products", "DigitalProduct elements", "products", as_of="date", population="repository"), ], absence=_absence( not_measured="The growth series was not measured: the snapshot counts failed.", @@ -1381,33 +1398,35 @@ def _always_applies(what: str) -> str: name="term_definition_completeness", function="pyegeria.view.overview_metrics.term_definition_completeness", inputs=["egeria_client:MetadataExpert"], - description="Share of GlossaryTerms carrying a non-empty description -- a definitions-" + description="Share of GlossaryTerms whose description is set and not empty -- a definitions-" "coverage metric, distinct from semantic_grounding (term<->asset linkage, not " "whether the term itself is actually defined). " - "Cost: one capped find, seconds.", + "Both numbers are native counts in the repository population, so the share never mixes " + "populations; a whitespace-only description counts as defined. " + "Cost: two native counts, sub-second each." + REPOSITORY_WORDS, returns="dict (total, defined, undefinedPct)", generic=False, binding_note="Fixed to GlossaryTerm's description property -- not a parameter.", params=[_as_of()], declaration=AnalyticDeclaration( result_kind="measure", - # One capped find; the result carries no flag saying the page cap was reached. + # Two native counts in one population: the share never divides across populations. measures=[ - _measure("terms_read", "GlossaryTerm elements, among those one capped find returned", "total", - population="viewer"), - _measure("defined_terms", "GlossaryTerm elements with a description", "defined", - population="viewer", basis=_basis("total", "terms read")), - _measure("undefined_percent", "percent of terms read without a description (0-100)", - "undefinedPct"), + _measure("terms", "GlossaryTerm elements", "total", population="repository"), + _measure("defined_terms", "GlossaryTerm elements with a description (set, not empty)", "defined", + population="repository", basis=_basis("total", "terms")), + _measure("undefined_percent", "percent of terms without a description (0-100)", + "undefinedPct", population="repository"), ], absence=_absence( - not_measured="Term definitions were not measured: the find of glossary terms failed.", + not_measured="Term definitions were not measured: a count failed, or the server could not " + "answer it in the repository population.", not_permitted="You are not permitted to read the glossary terms.", not_applicable=_always_applies("Term definition completeness"), - nothing_found="Read: there are no active glossary terms.", + nothing_found="Counted: there are no active glossary terms.", ), out=[ - _field("total", "Terms read", "int", unit="elements"), + _field("total", "Terms", "int", unit="elements"), _field("defined", "Defined", "int", unit="elements"), _field("undefinedPct", "Undefined", "int", unit="percent"), ], @@ -1463,7 +1482,7 @@ def _always_applies(what: str) -> str: "small sum-reducer over its result; `function` above is the thin composite " "wrapper that runs both via run_analytic_action, so calling it directly (the " "old, single-function path) still works exactly like every other entry here. " - "Cost: as counts_by_type: one native count per type.", + "Cost: as counts_by_type: one native count per type." + REPOSITORY_WORDS, returns="dict (total, byType)", generic=True, action=AnalyticActionSpec( @@ -1482,7 +1501,7 @@ def _always_applies(what: str) -> str: ], declaration=AnalyticDeclaration( result_kind="measure", - measures=[_measure("total", "elements", "total")], + measures=[_measure("total", "elements", "total", population="repository")], absence=_absence( not_measured="The total was not counted: the per-type count queries failed.", not_permitted="You are not permitted to count elements of these types.", @@ -1503,7 +1522,7 @@ def _always_applies(what: str) -> str: "given value -- e.g. DigitalProduct elements with deploymentStatus=ACTIVE. Same " "native-count-with-fallback seam as count_elements, one cheap native COUNT call " "per distinct value, no element fetch. " - "Cost: one native count, sub-second in the repository population; a viewer-population count can take minutes for a large type.", + "Cost: one native count, sub-second in the repository population; a viewer-population count can take minutes for a large type." + REPOSITORY_WORDS, returns="scalar (int)", generic=True, params=[ @@ -1513,13 +1532,16 @@ def _always_applies(what: str) -> str: description="String property to match, e.g. deploymentStatus."), AnalyticParam(name="property_value", type="str", required=True, description="Value to match the property against, e.g. ACTIVE."), + AnalyticParam(name="operator", type="str", default="EQ", + description="EQ (equals the value) or NEQ (set, and not the value; " + "property_value='' counts elements with the property set and not empty)."), _as_of(), _strict(), _population(), ], declaration=AnalyticDeclaration( result_kind="measure", - measures=[_measure("elements", "elements", "value")], + measures=[_measure("elements", "elements", "value", population="repository")], absence=_absence( not_measured="The elements were not counted: the count query failed or timed out, or " "the server could not answer it in the population asked for. A fallback " @@ -1536,11 +1558,12 @@ def _always_applies(what: str) -> str: name="count_relationships", function="pyegeria.view.overview_metrics.count_relationships", inputs=["egeria_client:ClassificationExplorer"], - description="Count active relationships of one type. By default one capped " - "get_relationships page (the path the rest of the portal uses); a native count " - "when a MetadataExpert is passed as `expert` (opt-in: the two paths can disagree " - "for some types, PY-18). Takes ce as its leading client. Cost: one relationship " - "page, seconds; the native count is sub-second.", + description="Count active relationships of one type. By default a native count over the " + "whole repository, which needs a client with the native count (EgeriaTech has it; " + "a bare ClassificationExplorer does not, and gives not measured). population=viewer " + "without a MetadataExpert reads one capped get_relationships page instead (the two " + "paths can disagree for some types, PY-18). Takes ce as its leading client. " + "Cost: the native count is sub-second; a relationship page takes seconds." + REPOSITORY_WORDS, returns="scalar (int | None)", generic=True, params=[ @@ -1551,7 +1574,7 @@ def _always_applies(what: str) -> str: ], declaration=AnalyticDeclaration( result_kind="measure", - measures=[_measure("relationships", "relationships", "value")], + measures=[_measure("relationships", "relationships", "value", population="repository")], absence=_absence( not_measured="The relationships were not counted: the query failed, or a " "repository count was asked for and no native count could answer it.", diff --git a/pyegeria/view/overview_metrics.py b/pyegeria/view/overview_metrics.py index 23232763..84dc1960 100644 --- a/pyegeria/view/overview_metrics.py +++ b/pyegeria/view/overview_metrics.py @@ -203,7 +203,9 @@ def _native_count_method(client, candidates: Sequence[str], cache_key: str) -> O _count_caps[cache_key] = name logger.info(f"overview_metrics count seam [{cache_key}]: " + (f"native via {name}" if name else "fallback to len(find/get)")) - return _count_caps[cache_key] + name = _count_caps[cache_key] + # The cache is per class name; a client of another class with the same name may lack the method. + return name if name and callable(getattr(client, name, None)) else None def _as_count(res) -> Optional[int]: @@ -349,6 +351,11 @@ def _native_failure_words(exc: Optional[BaseException], tried: bool) -> str: def _count_label(body: dict) -> str: label = body.get("metadataElementTypeName") or "elements" + props = ((body.get("searchProperties") or {}).get("conditions")) or [] + words = [f"{c.get('property')} {'=' if c.get('operator') == 'EQ' else '!='} " + f"{((c.get('value') or {}).get('primitiveValue'))!r}" for c in props if isinstance(c, dict)] + if words: + label = f"{label} with {' and '.join(words)}" conditions = ((body.get("matchClassifications") or {}).get("conditions")) or [] names = [c.get("name") for c in conditions if isinstance(c, dict) and c.get("name")] return f"{label} classified {'/'.join(names)}" if names else label @@ -364,13 +371,14 @@ def _element_count(mgr, body: dict, as_of: Optional[str] = None, strict: bool = """Count elements matching a FindRequestBody -- native when available, else len(find_metadata_elements(...)). - scope="repository": only a native push-down count can answer. The - len(find) fallback is always viewer-scoped, so it is NOT used -- the result - is None (strict) or 0 (non-strict, the historical failure value) instead of - a viewer number masquerading as a repository one. + scope="repository" (the default of every public count helper): only a + native push-down count can answer. The len(find) fallback is always + viewer-scoped, so it is NOT used -- a failed native count is None (strict + or not) with a note, never 0 and never a viewer number masquerading as a + repository one. - strict=False: 0 when the fallback query fails (the historical behaviour). - strict=True: None when the fallback query fails. + scope="viewer" or None: strict=False gives 0 when the fallback query fails + (the historical behaviour); strict=True gives None. Either way: None when the fallback returned a full page (DEFAULT_CAP). That is only a lower bound, never a total, so it is never returned as one; the bound and the reason go to `recording_unmeasured()`'s notes. @@ -394,13 +402,10 @@ def _element_count(mgr, body: dict, as_of: Optional[str] = None, strict: bool = _disable_native(mgr, exc) native_words = _native_failure_words(native_exc, tried) if scope == SCOPE_REPOSITORY: - if strict: - _not_measured(what, f"{native_words}, and a repository-population count has no fallback " - f"(the find fallback is viewer-scoped)") - return None - logger.warning(f"overview_metrics: repository-scope count of {what!r} not measured -- " - f"native count unavailable or failed (the find fallback is viewer-scoped)") - return 0 + # Strict or not: a repository count has no fallback, and 0 here would read as an empty catalog. + _not_measured(what, f"{native_words}, and a repository-population count has no fallback " + f"(the find fallback only sees what the viewer may see)") + return None found = _find_or_none(mgr, b) if found is None: if not strict: @@ -420,7 +425,7 @@ def count_elements( as_of: Optional[str] = None, classifications: Optional[Sequence[str]] = None, strict: bool = False, - scope: Optional[str] = None, + scope: Optional[str] = SCOPE_REPOSITORY, ) -> Optional[int]: """ Count ACTIVE elements, optionally of a given type and/or matching ANY of @@ -442,16 +447,19 @@ def count_elements( single call's worth (some connectors mishandle 2+ ANY conditions) strict : when True, return None (not 0) if the query failed. A fallback count that hit DEFAULT_CAP is None in either mode - scope : None (server default), "viewer" (push_down=False: only what the - caller may see) or "repository" (push_down=True: whole-repository raw - count, fast, includes elements the caller cannot see). The len(find) + scope : "repository" (the default: push_down=True, a whole-repository raw + count -- fast, and it includes elements the caller may not be able to + see; a deliberate interim choice, until zone-aware push-down exists), + "viewer" (push_down=False: only what the caller may see) or None (omit + push_down; the server default, which is viewer-scoped). The len(find) fallback always answers in viewer scope, so with scope="repository" it - is never used: a failed native count is None (strict) or 0 (non-strict). + is never used: a failed native count is None, strict or not. Returns ------- - Optional[int] -- 0 on a failed query (never raises), None on a capped - fallback; with strict=True, None whenever the count is not measured + Optional[int] -- None whenever a repository count is not measured, and on + a capped fallback; in viewer scope a failed query is 0 unless strict=True + (never raises) """ body: Dict[str, Any] = {"class": "FindRequestBody", "limitResultsByStatus": ["ACTIVE"]} if type_name: @@ -472,7 +480,8 @@ def count_elements_by_property( property_value: str, as_of: Optional[str] = None, strict: bool = False, - scope: Optional[str] = None, + scope: Optional[str] = SCOPE_REPOSITORY, + operator: str = "EQ", ) -> Optional[int]: """ Count ACTIVE elements of a given type whose named string property equals @@ -491,12 +500,17 @@ def count_elements_by_property( property_value : the exact string value to match as_of : ISO-8601 asOfTime for a point-in-time count (None = now) strict : as for count_elements -- None instead of 0 when not measured - scope : as for count_elements (None | "viewer" | "repository") + scope : as for count_elements ("repository" by default | "viewer" | None) + operator : "EQ" (the default: the property equals `property_value`) or + "NEQ" (it is set and differs -- e.g. property_value="" counts elements + whose property is set to something other than the empty string) Returns ------- Optional[int] -- as for count_elements """ + if operator not in ("EQ", "NEQ"): + raise ValueError(f"count_elements_by_property: operator must be 'EQ' or 'NEQ'; got {operator!r}") body: Dict[str, Any] = { "class": "FindRequestBody", "metadataElementTypeName": type_name, @@ -506,7 +520,7 @@ def count_elements_by_property( "matchCriteria": "ALL", "conditions": [{ "property": property_name, - "operator": "EQ", + "operator": operator, "value": {"class": "PrimitiveTypePropertyValue", "typeName": "string", "primitiveValue": property_value}, }], }, @@ -516,7 +530,7 @@ def count_elements_by_property( @_population_keyword def count_relationships(ce, relationship_type: str, as_of: Optional[str] = None, expert=None, - scope: Optional[str] = None) -> Optional[int]: + scope: Optional[str] = SCOPE_REPOSITORY) -> Optional[int]: """ Count relationships of a type, optionally as of a past time. @@ -534,13 +548,17 @@ def count_relationships(ce, relationship_type: str, as_of: Optional[str] = None, ce : ClassificationExplorer (or compatible client) relationship_type : the relationship type name to count as_of : ISO-8601 asOfTime for a point-in-time count (None = now) - expert : optional MetadataExpert to enable the native fast-path - scope : None (default: today's behaviour), "viewer" (native push_down=False) - or "repository" (native push_down=True). Only the native path takes a - scope; the get_relationships path is always viewer-scoped, so with - scope="repository" it is never used -- a missing `expert`, an - unsupported server or a failed native call gives None, not a viewer - number. + expert : optional MetadataExpert to enable the native fast-path. With + scope="repository" and no `expert`, `ce` itself is used when it has the + native method (an EgeriaTech does; a bare ClassificationExplorer does not) + scope : "repository" (the default: native push_down=True, whole repository, + including relationships the caller may not be able to see), "viewer" + (native push_down=False) or None (the get_relationships path, unless an + `expert` is passed). Only the native path takes a scope; the + get_relationships path is always viewer-scoped, so with + scope="repository" it is never used -- no native method, an + unsupported server or a failed native call gives None with a note, not + a viewer number. Returns ------- @@ -551,6 +569,8 @@ def count_relationships(ce, relationship_type: str, as_of: Optional[str] = None, push_down = _push_down(scope) kw = {} if push_down is None else {"push_down": push_down} tried, native_exc = False, None + if expert is None and scope == SCOPE_REPOSITORY: + expert = ce # only a native count can answer for the repository; ce may be able to give one if expert is not None and not _native_disabled(expert): name = _native_count_method(expert, _REL_COUNT_CANDIDATES, "rel:" + type(expert).__name__) if name: @@ -567,8 +587,10 @@ def count_relationships(ce, relationship_type: str, as_of: Optional[str] = None, native_exc = exc _disable_native(expert, exc) if scope == SCOPE_REPOSITORY: - logger.warning(f"overview_metrics: repository-scope count of {relationship_type!r} not measured -- " - f"native count unavailable or failed (get_relationships is viewer-scoped)") + words = (_native_failure_words(native_exc, tried) if tried + else "no native relationship count is available (it needs a MetadataExpert client)") + _not_measured(relationship_type, f"{words}, and a repository-population count has no fallback " + f"(get_relationships only sees what the viewer may see)") return None try: body = {"class": "ResultsRequestBody", "asOfTime": as_of} if as_of else None @@ -588,7 +610,7 @@ def count_relationships(ce, relationship_type: str, as_of: Optional[str] = None, @_population_keyword def counts_by_type(mgr, type_map: Sequence[Tuple[str, str]], as_of: Optional[str] = None, - strict: bool = False, scope: Optional[str] = None) -> List[Dict[str, Any]]: + strict: bool = False, scope: Optional[str] = SCOPE_REPOSITORY) -> List[Dict[str, Any]]: """ Per-type counts for a set of (label, type_name) pairs -- e.g. the assets- by-type composition breakdown. Unknown/wrong types just yield 0 rather @@ -799,7 +821,8 @@ def ownership_coverage(mgr, as_of: Optional[str] = None) -> Dict[str, Any]: @_population_keyword -def business_value_signals(mgr, as_of: Optional[str] = None, scope: Optional[str] = None) -> Dict[str, Any]: +def business_value_signals(mgr, as_of: Optional[str] = None, + scope: Optional[str] = SCOPE_REPOSITORY) -> Dict[str, Any]: """ Real signals behind the Overview dashboard's four "Business Value" tiles (NEXT-9) -- replaces the four previously-synthetic narrative numbers @@ -810,68 +833,55 @@ def business_value_signals(mgr, as_of: Optional[str] = None, scope: Optional[str it (e.g. "more described assets" is a proxy for self-service findability, not a measure of actual query/access frequency). - A single Asset-hierarchy fetch answers two of the four fields - (confidentiality exposure and description coverage -- both are - per-element checks on the same fetched set, same "check several signals - from one fetch" shape `context_readiness_funnel` already uses). + Every field is a native count in one population, `scope` ("repository" by + default: the whole repository, including elements the viewer may not be + able to see; or "viewer"), so the asset-scoped shares never mix + populations: + + - assetTotal: Assets (the supertype, so every subtype). + - confidentialCount: Assets carrying Confidentiality. + - describedCount: Assets whose `description` is set and not the empty + string (`NEQ ""`; a whitespace-only description counts as described). + - duplicateCount: elements of any type carrying ConsolidatedDuplicate. IMPORTANT scoping distinction from `governed_coverage`'s own `byClassification["Confidentiality"]`: that number is NOT Asset-scoped (any element type carrying the classification counts, per its own - documented caveat), whereas `confidentialCount` here IS Asset-scoped by - construction (checked only on elements this function's own Asset- - hierarchy fetch returned). Confirmed live 2026-08-02 on a real dataset: - 5 total Confidentiality-classified elements exist, only 1 of which is - Asset-typed (a CSVFile; the other 4 are typed `Referenceable`, not under - `Asset`) -- the two numbers are both correct, just different + documented caveat), whereas `confidentialCount` here IS Asset-scoped + (the count names the Asset type). Confirmed live 2026-08-02 on a real + dataset: 5 total Confidentiality-classified elements exist, only 1 of + which is Asset-typed (a CSVFile; the other 4 are typed `Referenceable`, + not under `Asset`) -- the two numbers are both correct, just different populations, not a discrepancy to "fix." - Duplicate detection is a separate classification count. Data Products - (Trust & Adoption) is deliberately NOT duplicated here -- callers already - have a live `dataProducts` count from `count_elements(mgr, "DigitalProduct", as_of)` - elsewhere in the same request; no rating/adoption signal is wired (no - `AttachedRating` relationships exist against `DigitalProduct` in a typical - demo dataset, confirmed live -- so a rating average is honestly omitted - rather than faked). - - `scope` applies to the duplicate count only; the Asset-derived fields come - from a find and stay viewer-scoped. + Data Products (Trust & Adoption) is deliberately NOT duplicated here -- + callers already have a live `dataProducts` count from + `count_elements(mgr, "DigitalProduct", as_of)` elsewhere in the same + request; no rating/adoption signal is wired (no `AttachedRating` + relationships exist against `DigitalProduct` in a typical demo dataset, + confirmed live -- so a rating average is honestly omitted rather than + faked). - Any field whose query failed is None (unmeasured), never 0: assetTotal/ - assetCapped/confidentialCount/describedCount when the Asset find fails, - duplicateCount when its (strict) count fails. + These figures used to come from one capped Asset find, so on a catalog + over DEFAULT_CAP assets they described the first page only. A count that + was not measured is None, never 0. `assetCapped` is kept for callers that + read it: False when assetTotal was measured (a native count is + complete), None when it was not. Returns {"assetTotal": int|None, "assetCapped": bool|None, "confidentialCount": int|None, "describedCount": int|None, "duplicateCount": int|None}. """ - body = {"class": "FindRequestBody", "limitResultsByStatus": ["ACTIVE"], - "metadataElementTypeName": "Asset"} - if as_of: - body["asOfTime"] = as_of - assets = _find_or_none(mgr, body) - - confidential: Optional[int] = None - described: Optional[int] = None - if assets is not None: - confidential = 0 - described = 0 - for a in assets or []: - if "Confidentiality" in _classifications_of(a): - confidential += 1 - pvm = (a.get("elementProperties") or {}).get("propertyValueMap") or {} - desc = (pvm.get("description") or {}).get("primitiveValue") - if isinstance(desc, str) and desc.strip(): - described += 1 - - duplicate = count_elements(mgr, None, as_of, classifications=["ConsolidatedDuplicate"], strict=True, population=scope) - + total = count_elements(mgr, "Asset", as_of, strict=True, population=scope) return { - "assetTotal": len(assets) if assets is not None else None, - "assetCapped": (len(assets) >= DEFAULT_CAP) if assets is not None else None, - "confidentialCount": confidential, - "describedCount": described, - "duplicateCount": duplicate, + "assetTotal": total, + "assetCapped": None if total is None else False, + "confidentialCount": count_elements(mgr, "Asset", as_of, classifications=["Confidentiality"], + strict=True, population=scope), + "describedCount": count_elements_by_property(mgr, "Asset", "description", "", as_of, strict=True, + population=scope, operator="NEQ"), + "duplicateCount": count_elements(mgr, None, as_of, classifications=["ConsolidatedDuplicate"], + strict=True, population=scope), } @@ -935,8 +945,9 @@ def certifications_summary(ce, as_of: Optional[str] = None) -> Dict[str, Any]: except Exception as exc: # noqa: BLE001 logger.debug(f"overview_metrics certifications_summary: query failed: {exc}") - out["licenses"] = count_relationships(ce, "License", as_of) - out["exceptions"] = count_relationships(ce, "Exception", as_of) + # One population per payload: active/expiring come from a get_relationships page (viewer-scoped). + out["licenses"] = count_relationships(ce, "License", as_of, population=SCOPE_VIEWER) + out["exceptions"] = count_relationships(ce, "Exception", as_of, population=SCOPE_VIEWER) return out @@ -962,7 +973,10 @@ def orphan_glossary_terms(mgr, ce, as_of: Optional[str] = None) -> Dict[str, Opt "orphanCount": int|None}. Any field is None on total failure for its own step (never raises). """ - term_total = count_elements(mgr, "GlossaryTerm", as_of, strict=True) + # One population per payload: this payload's other figures come from a find, which only sees what the + # viewer may see, so its native total is counted in the viewer population too (never a repository + # total under a viewer numerator). + term_total = count_elements(mgr, "GlossaryTerm", as_of, strict=True, population=SCOPE_VIEWER) referenced_count = None try: @@ -1026,7 +1040,11 @@ def stale_assets(mgr, as_of: Optional[str] = None, days: int = 180) -> Dict[str, except Exception as exc: # noqa: BLE001 logger.debug(f"stale_assets: query failed: {exc}") - return {"staleCount": stale_count, "assetTotal": count_elements(mgr, "Asset", as_of, strict=True)} + # One population per payload: this payload's other figures come from a find, which only sees what the + # viewer may see, so its native total is counted in the viewer population too (never a repository + # total under a viewer numerator). + return {"staleCount": stale_count, + "assetTotal": count_elements(mgr, "Asset", as_of, strict=True, population=SCOPE_VIEWER)} def semantic_grounding(mgr, ce, as_of: Optional[str] = None) -> Dict[str, Any]: @@ -1035,10 +1053,15 @@ def semantic_grounding(mgr, ce, as_of: Optional[str] = None) -> Dict[str, Any]: asset), as a count and as a percentage of the broad `Asset` supertype count -- the "meaning layer" that grounds AI context. + Both counts are native and in the repository population (the whole + repository, including elements the viewer may not be able to see), so the + percentage never divides one population by the other. `mgr` is the + MetadataExpert the repository relationship count needs. + Returns {"groundingLinks": int|None, "groundingPct": int|None}. """ - cataloged = count_elements(mgr, "Asset", as_of, strict=True) - links = count_relationships(ce, "SemanticAssignment", as_of) + cataloged = count_elements(mgr, "Asset", as_of, strict=True, population=SCOPE_REPOSITORY) + links = count_relationships(ce, "SemanticAssignment", as_of, expert=mgr, population=SCOPE_REPOSITORY) pct = min(100, round(100 * links / cataloged)) if cataloged and links is not None else None return {"groundingLinks": links, "groundingPct": pct} @@ -1048,11 +1071,14 @@ def context_readiness_funnel(mgr, ce, as_of: Optional[str] = None) -> Dict[str, Context-readiness funnel stages: how much cataloged data becomes trustworthy AI context. - `documented` -- count of `Asset` elements carrying a non-empty - `description`, same field-value-check approach `term_definition_completeness` - uses for GlossaryTerm (fetches element bodies via `_find`, capped at - DEFAULT_CAP -- no native count exists for "has a description", same - tradeoff `governed_coverage`'s by-classification breakdown already accepts). + Every stage is a native count in the repository population (the whole + repository, including elements the viewer may not be able to see), so the + stages can be compared without mixing populations. + + `documented` -- count of `Asset` elements whose `description` is set and + not the empty string (a native `NEQ ""` property count, the same one + `term_definition_completeness` uses; a whitespace-only description counts + as documented). `lineage` -- count of `DataFlow` relationships (design/business lineage -- literal data-movement edges between assets/processes; distinct from @@ -1081,26 +1107,13 @@ def context_readiness_funnel(mgr, ce, as_of: Optional[str] = None) -> Dict[str, Returns {"cataloged": int|None, "documented": int|None, "classified": int|None, "lineage": int|None, "aiReady": None}. """ - cataloged = count_elements(mgr, "Asset", as_of, strict=True) - classified = count_elements(mgr, None, as_of, classifications=GOVERNANCE_CLASSIFICATIONS, strict=True) - - documented = None - try: - body: Dict[str, Any] = {"class": "FindRequestBody", "limitResultsByStatus": ["ACTIVE"], - "metadataElementTypeName": "Asset"} - if as_of: - body["asOfTime"] = as_of - assets = _find_or_none(mgr, body) # None when the find failed: documented stays None, not 0 - documented = None if assets is None else 0 - for a in assets or []: - pvm = (a.get("elementProperties") or {}).get("propertyValueMap") or {} - desc = (pvm.get("description") or {}).get("primitiveValue") - if isinstance(desc, str) and desc.strip(): - documented += 1 - except Exception as exc: # noqa: BLE001 - logger.debug(f"context_readiness_funnel: documented query failed: {exc}") - - lineage = count_relationships(ce, "DataFlow", as_of) + pop = SCOPE_REPOSITORY + cataloged = count_elements(mgr, "Asset", as_of, strict=True, population=pop) + classified = count_elements(mgr, None, as_of, classifications=GOVERNANCE_CLASSIFICATIONS, strict=True, + population=pop) + documented = count_elements_by_property(mgr, "Asset", "description", "", as_of, strict=True, + population=pop, operator="NEQ") + lineage = count_relationships(ce, "DataFlow", as_of, expert=mgr, population=pop) return { "cataloged": cataloged, @@ -1352,7 +1365,7 @@ def _update_time(el: dict): @_population_keyword -def people_counts(mgr, as_of: Optional[str] = None, strict: bool = False, scope: Optional[str] = None) -> Dict[str, Optional[int]]: +def people_counts(mgr, as_of: Optional[str] = None, strict: bool = False, scope: Optional[str] = SCOPE_REPOSITORY) -> Dict[str, Optional[int]]: """ Person / Team / Organization / ITProfile / Community counts via native element counts (sub-second where an older find-and-bucket approach would @@ -1384,7 +1397,7 @@ def people_counts(mgr, as_of: Optional[str] = None, strict: bool = False, scope: @_population_keyword -def feedback_summary(ce, as_of: Optional[str] = None, scope: Optional[str] = None, expert=None) -> Dict[str, Any]: +def feedback_summary(ce, as_of: Optional[str] = None, scope: Optional[str] = SCOPE_REPOSITORY, expert=None) -> Dict[str, Any]: """ Crowd-sourced feedback tally -- ratings/comments/likes/tags/note-logs (Collaboration OMAS relationship counts). Often sparse in demo data, but @@ -1406,7 +1419,7 @@ def feedback_summary(ce, as_of: Optional[str] = None, scope: Optional[str] = Non @_population_keyword -def usage_context_counts(mgr, as_of: Optional[str] = None, strict: bool = False, scope: Optional[str] = None) -> Dict[str, Optional[int]]: +def usage_context_counts(mgr, as_of: Optional[str] = None, strict: bool = False, scope: Optional[str] = SCOPE_REPOSITORY) -> Dict[str, Optional[int]]: """ InformationSupplyChain and SolutionBlueprint counts -- what puts assets in a *usage context* ("this store feeds the Clinical Trial supply @@ -1478,7 +1491,10 @@ def contextualised_coverage(mgr, ce, as_of: Optional[str] = None) -> Dict[str, O except Exception as exc: # noqa: BLE001 logger.debug(f"contextualised_coverage: ImplementedBy query failed: {exc}") - asset_total = count_elements(mgr, "Asset", as_of, strict=True) + # One population per payload: this payload's other figures come from a find, which only sees what the + # viewer may see, so its native total is counted in the viewer population too (never a repository + # total under a viewer numerator). + asset_total = count_elements(mgr, "Asset", as_of, strict=True, population=SCOPE_VIEWER) pct = None if contextualised_count is not None and asset_total: @@ -1523,7 +1539,10 @@ def solution_component_realisation(mgr, ce, as_of: Optional[str] = None) -> Dict int|None}. Either field is None on total failure for its own step (never raises). """ - total = count_elements(mgr, "SolutionComponent", as_of, strict=True) + # One population per payload: this payload's other figures come from a find, which only sees what the + # viewer may see, so its native total is counted in the viewer population too (never a repository + # total under a viewer numerator). + total = count_elements(mgr, "SolutionComponent", as_of, strict=True, population=SCOPE_VIEWER) realised_count = None try: @@ -1694,6 +1713,7 @@ def growth_series( window: str = "6mo", points: Optional[int] = None, type_map: Optional[Sequence[Tuple[str, Optional[str], Optional[Sequence[str]]]]] = None, + population: str = SCOPE_REPOSITORY, ) -> List[Dict[str, Any]]: """ A real growth series -- one snapshot per point, using Egeria's native @@ -1707,6 +1727,8 @@ def growth_series( type_map : what to snapshot at each point, as (series_key, type_name, classifications) triples. Defaults to the Overview dashboard's own four series: assets/terms/governed/products. + population : "repository" (the default: whole-repository native counts, + including elements the viewer may not be able to see) or "viewer" Returns a list of dicts, oldest first, each {"label": ..., "date": ..., : count, ...}. @@ -1721,6 +1743,7 @@ def growth_series( ("products", "DigitalProduct", None), ] + _push_down(population) # ValueError for an unknown population, before any count span_s, default_pts = WINDOWS.get(window, WINDOWS["6mo"]) n = points or default_pts @@ -1738,7 +1761,8 @@ def growth_series( point = {"label": growth_label(d, span_s), "date": d.strftime(date_fmt)} try: for key, type_name, classifications in type_map: - point[key] = count_elements(mgr, type_name, as_of, classifications=classifications, strict=True) + point[key] = count_elements(mgr, type_name, as_of, classifications=classifications, strict=True, + population=population) except Exception as exc: # noqa: BLE001 logger.debug(f"overview_metrics growth_series: snapshot {i} failed: {exc}") for key, _, _ in type_map: @@ -1840,40 +1864,29 @@ def term_definition_completeness(mgr, as_of: Optional[str] = None) -> Dict[str, `semantic_grounding` (which measures term<->asset linkage, not whether the term itself is actually defined). - Unlike `count_elements`'s classification-only filter, "has a - description" is a field-value check, so this fetches term bodies (via - `_find`, same `FindRequestBody`/`metadataElementTypeName` shape - `count_elements` uses) rather than using a native count -- no fast - native-count path exists for this, same tradeoff `governed_coverage` - accepts for its by-classification breakdown. `description` is read the - same way `_zone_names` reads `zoneMembership`: verified live against a - running server that `find_metadata_elements` returns raw - `elementProperties.propertyValueMap.description.primitiveValue`, not a - flat `properties.description` (that flatter shape belongs to the - output_format='JSON' relationship/report path other functions here use, - e.g. `certifications_summary`'s `end2`, not this raw element-query path). + Both numbers are native counts in the repository population (the whole + repository, including terms the viewer may not be able to see), so the + ratio never divides a viewer-scoped numerator by a repository total: + `total` counts GlossaryTerms, `defined` those whose `description` is set + and not the empty string (`NEQ ""`). A whitespace-only description counts + as defined -- Egeria's LIKE (regex) condition counted 0 for every pattern + tried live (2026-11-08), so "not blank" cannot be counted natively; live, + 9 of 1061 terms differ. This used to scan one page of terms, so on a + glossary over DEFAULT_CAP terms the total was the page size, not the total. Returns {"total": int|None, "defined": int|None, "undefinedPct": int|None} (`undefinedPct` mirrors `semantic_grounding.groundingPct`'s naming: the gap, not the coverage, since a near-100%-defined glossary is the - uninteresting case and the gap is what's actionable). A failed find makes - all three None -- unmeasured, not an empty glossary. + uninteresting case and the gap is what's actionable). A count that was not + measured is None, and so is the percentage -- unmeasured, not an empty + glossary. """ - body: Dict[str, Any] = {"class": "FindRequestBody", "limitResultsByStatus": ["ACTIVE"], - "metadataElementTypeName": "GlossaryTerm"} - if as_of: - body["asOfTime"] = as_of - terms = _find_or_none(mgr, body) - if terms is None: - return {"total": None, "defined": None, "undefinedPct": None} - total = len(terms) - defined = 0 - for t in terms: - pvm = (t.get("elementProperties") or {}).get("propertyValueMap") or {} - desc = (pvm.get("description") or {}).get("primitiveValue") - if isinstance(desc, str) and desc.strip(): - defined += 1 - undefined_pct = round(100 * (total - defined) / total) if total else None + pop = SCOPE_REPOSITORY + total = count_elements(mgr, "GlossaryTerm", as_of, strict=True, population=pop) + defined = count_elements_by_property(mgr, "GlossaryTerm", "description", "", as_of, strict=True, + population=pop, operator="NEQ") + undefined_pct = (round(100 * (total - defined) / total) + if total and defined is not None else None) return {"total": total, "defined": defined, "undefinedPct": undefined_pct} diff --git a/tests/micro-tests/test_capped_count_not_measured.py b/tests/micro-tests/test_capped_count_not_measured.py index 9c247d91..1d765a5d 100644 --- a/tests/micro-tests/test_capped_count_not_measured.py +++ b/tests/micro-tests/test_capped_count_not_measured.py @@ -8,6 +8,11 @@ at DEFAULT_CAP, and the Portal tile showed "Total Cataloged Assets = 500". A full page is a lower bound, never a total: the count is None ("not measured"), and the reason with the bound it saw reaches the output. Every branch is executed here, not read. + +Since 2026-11-08 the count helpers default to the repository population (push_down=True), which +has no find fallback at all: a failed repository count is None with its reason, never 0 and never a +viewer number. The fallback and its lower bound now only arise in the viewer population, so the +seam tests below ask for it explicitly. """ import json from unittest.mock import MagicMock @@ -19,6 +24,8 @@ from pyegeria.view.role_supply import Subject, Viewer CAP = om.DEFAULT_CAP +REPO_TIMEOUT = ("the native count timed out, and a repository-population count has no fallback " + "(the find fallback only sees what the viewer may see)") TIMEOUT = RuntimeError("TIMEOUT_ERROR_408 the request to .../by-search-conditions/count timed out") @@ -56,7 +63,7 @@ def test_a_timeout_whose_fallback_hits_the_cap_is_not_measured_with_its_lower_bo mgr = _mgr(native=MagicMock(side_effect=TIMEOUT), find_result=_page(CAP)) with om.recording_unmeasured() as notes: for strict in (True, False): - assert om.count_elements(mgr, "Asset", strict=strict) is None # never CAP + assert om.count_elements(mgr, "Asset", strict=strict, population="viewer") is None # never CAP assert notes == [{"count": "Asset", "at_least": CAP, "reason": f"the native count timed out, and the find fallback stopped at its " f"{CAP}-element page cap, so there are at least {CAP}"}] * 2 @@ -65,11 +72,20 @@ def test_a_timeout_whose_fallback_hits_the_cap_is_not_measured_with_its_lower_bo def test_a_timeout_whose_fallback_fails_is_not_measured_strict(): mgr = _mgr(native=MagicMock(side_effect=TIMEOUT), find_error=RuntimeError("boom")) with om.recording_unmeasured() as notes: - assert om.count_elements(mgr, "Process", strict=True) is None + assert om.count_elements(mgr, "Process", strict=True, population="viewer") is None assert notes == [{"count": "Process", "at_least": None, "reason": "the native count timed out, and the find fallback failed"}] +def test_a_timed_out_repository_count_is_not_measured_and_never_asks_the_find(): + mgr = _mgr(native=MagicMock(side_effect=TIMEOUT), find_result=_page(3)) + with om.recording_unmeasured() as notes: + for strict in (True, False): + assert om.count_elements(mgr, "Asset", strict=strict) is None # never 0, never a viewer 3 + mgr.find_metadata_elements.assert_not_called() + assert notes == [{"count": "Asset", "at_least": None, "reason": REPO_TIMEOUT}] * 2 + + def test_the_timeout_leaves_native_enabled_for_the_next_count(): native = MagicMock(side_effect=[TIMEOUT, {"count": 7}]) mgr = _mgr(native=native, find_result=_page(CAP)) @@ -80,8 +96,8 @@ def test_the_timeout_leaves_native_enabled_for_the_next_count(): def test_a_short_fallback_page_is_a_measured_count_and_records_nothing(): mgr = _mgr(native=MagicMock(side_effect=TIMEOUT), find_result=_page(CAP - 1)) with om.recording_unmeasured() as notes: - assert om.count_elements(mgr, "Asset") == CAP - 1 - assert om.count_elements(mgr, "Asset", strict=True) == CAP - 1 + assert om.count_elements(mgr, "Asset", population="viewer") == CAP - 1 + assert om.count_elements(mgr, "Asset", strict=True, population="viewer") == CAP - 1 assert notes == [] @@ -91,8 +107,8 @@ def test_a_native_count_above_the_cap_is_trusted(): assert notes == [] -def test_non_strict_failure_keeps_its_historical_zero(): - assert om.count_elements(_mgr(find_error=RuntimeError("boom")), "Asset") == 0 +def test_non_strict_viewer_failure_keeps_its_historical_zero(): + assert om.count_elements(_mgr(find_error=RuntimeError("boom")), "Asset", population="viewer") == 0 def test_a_classification_count_names_what_it_counted(): @@ -118,10 +134,10 @@ def test_a_full_relationship_page_is_not_a_relationship_count(): ce = MagicMock() ce.get_relationships.return_value = [{"guid": f"r{i}"} for i in range(CAP)] with om.recording_unmeasured() as notes: - assert om.count_relationships(ce, "SemanticAssignment") is None + assert om.count_relationships(ce, "SemanticAssignment", population="viewer") is None assert notes[0]["count"] == "SemanticAssignment" and notes[0]["at_least"] == CAP ce.get_relationships.return_value = [{"guid": "r1"}] - assert om.count_relationships(ce, "SemanticAssignment") == 1 + assert om.count_relationships(ce, "SemanticAssignment", population="viewer") == 1 def test_semantic_grounding_no_longer_reports_500_links_100_percent(): @@ -194,14 +210,26 @@ def _exec(label, fmt="DICT", **params): user="u", user_pass="p", supply=_Supply()) -def test_the_total_cataloged_assets_tile_says_not_measured_with_at_least_500(): +def test_the_total_cataloged_assets_tile_says_not_measured_and_why(): out = _exec("Analytic Demo - Element Count by Type", type_name="Asset") assert out["kind"] == "absent" and "data" not in out assert out["absence"]["state"] == "not_measured" + assert out["unmeasured"] == [{"count": "Asset", "at_least": None, "reason": REPO_TIMEOUT}] + assert "Asset: the native count timed out" in out["content"] + + +def test_a_non_strict_dashboard_count_is_never_a_confident_zero(): + out = _exec("Analytic Demo - Element Count by Type", type_name="Asset", strict=False) + assert out["kind"] == "absent" and "data" not in out + assert out["unmeasured"][0]["reason"] == REPO_TIMEOUT + + +def test_a_viewer_tile_still_says_at_least_500(): + out = _exec("Analytic Demo - Element Count by Type", type_name="Asset", population="viewer") + assert out["kind"] == "absent" assert out["unmeasured"] == [{"count": "Asset", "at_least": CAP, "reason": ( f"the native count timed out, and the find fallback stopped at its {CAP}-element page cap, " f"so there are at least {CAP}")}] - assert "Asset: the native count timed out" in out["content"] assert f"at least {CAP}" in out["content"] @@ -215,7 +243,7 @@ def test_the_breakdown_rows_keep_measured_counts_and_name_the_unmeasured_one(): assert out["kind"] == "json" assert out["data"] == [{"label": "Data Stores", "type": "DataStore", "count": 843}, {"label": "Processes", "type": "Process", "count": None}] - assert [(n["count"], n["at_least"]) for n in out["unmeasured"]] == [("Process", CAP)] + assert [(n["count"], n["at_least"]) for n in out["unmeasured"]] == [("Process", None)] assert CAP not in [r["count"] for r in out["data"]] @@ -232,7 +260,8 @@ def test_semantic_grounding_spec_reports_nulls_and_why(): out = _exec("Analytic Demo - Semantic Grounding") assert out["data"] == {"groundingLinks": None, "groundingPct": None} assert {n["count"] for n in out["unmeasured"]} == {"Asset", "SemanticAssignment"} - assert all(n["at_least"] == CAP for n in out["unmeasured"]) + rel = next(n for n in out["unmeasured"] if n["count"] == "SemanticAssignment") + assert rel["reason"].startswith("no native relationship count is available") def test_a_series_repeats_no_note(): diff --git a/tests/micro-tests/test_issue_123_strict_counts.py b/tests/micro-tests/test_issue_123_strict_counts.py index 5cd6bd19..ce8720dc 100644 --- a/tests/micro-tests/test_issue_123_strict_counts.py +++ b/tests/micro-tests/test_issue_123_strict_counts.py @@ -3,7 +3,10 @@ """ ISSUE-123: count_elements turns a failed query into 0, so an outage looks like an empty catalog. strict=True returns None whenever the count is not known; -the default keeps the old behaviour for existing callers. +the default keeps the old behaviour for existing callers -- in the viewer +population, the only one with a find fallback. A repository count (the +default population since 2026-11-08) has no fallback, so it is never 0 on +failure, strict or not. """ from unittest.mock import MagicMock @@ -37,8 +40,14 @@ def _reset_seam_caches(): om._native_server_ok.clear() -def test_default_still_returns_zero_on_failure(): - assert om.count_elements(_mgr(find_error=RuntimeError("401")), "Asset") == 0 +def test_default_still_returns_zero_on_failure_in_the_viewer_population(): + assert om.count_elements(_mgr(find_error=RuntimeError("401")), "Asset", population="viewer") == 0 + + +def test_a_failed_repository_count_is_none_never_zero(): + mgr = _mgr(native=MagicMock(side_effect=RuntimeError("401")), find_result=[]) + assert om.count_elements(mgr, "Asset") is None + assert om.count_elements(mgr, "Asset", strict=True) is None def test_strict_returns_none_when_the_query_fails(): @@ -52,8 +61,10 @@ def test_strict_returns_none_when_native_and_fallback_both_fail(): def test_strict_real_zero_is_zero(): # pyegeria answers a miss with its sentinel string, not an empty list. - assert om.count_elements(_mgr(find_result="No elements found"), "Asset", strict=True) == 0 - assert om.count_elements(_mgr(find_result=[]), "Asset", strict=True) == 0 + assert om.count_elements(_mgr(find_result="No elements found"), "Asset", strict=True, population="viewer") == 0 + assert om.count_elements(_mgr(find_result=[]), "Asset", strict=True, population="viewer") == 0 + om._count_caps.clear() # the seam caches native support per client class; these mocks share one + assert om.count_elements(_mgr(native=MagicMock(return_value={"count": 0})), "Asset", strict=True) == 0 def test_a_capped_fallback_is_none_in_either_mode_never_the_cap(): @@ -74,7 +85,8 @@ def test_strict_native_count_is_trusted_even_above_the_cap(): def test_strict_by_property(): mgr = _mgr(find_error=RuntimeError("500")) - assert om.count_elements_by_property(mgr, "DigitalProduct", "deploymentStatus", "ACTIVE") == 0 + assert om.count_elements_by_property(mgr, "DigitalProduct", "deploymentStatus", "ACTIVE", + population="viewer") == 0 assert om.count_elements_by_property(mgr, "DigitalProduct", "deploymentStatus", "ACTIVE", strict=True) is None @@ -84,7 +96,8 @@ def test_strict_propagates_through_the_composite_counts(): assert om.counts_by_type(mgr, [("Stores", "DataStore")], strict=True)[0]["count"] is None assert set(om.people_counts(mgr, strict=True).values()) == {None} assert set(om.usage_context_counts(mgr, strict=True).values()) == {None} - assert set(om.people_counts(mgr).values()) == {0} + assert set(om.people_counts(mgr, population="viewer").values()) == {0} + assert set(om.people_counts(mgr).values()) == {None} # repository: no fallback, never 0 @pytest.mark.parametrize("name", ["count_elements", "count_elements_by_property", "counts_by_type", diff --git a/tests/micro-tests/test_overview_metrics.py b/tests/micro-tests/test_overview_metrics.py index b754b5b8..8e7aee18 100644 --- a/tests/micro-tests/test_overview_metrics.py +++ b/tests/micro-tests/test_overview_metrics.py @@ -25,6 +25,21 @@ def _mgr(**attrs): return m +def _native(answer): + """A native count method fake: `answer(body)` gives the count for each request body.""" + return MagicMock(side_effect=lambda body, **kw: {"count": answer(body)}) + + +def _has_desc_neq(body): + conds = (body.get("searchProperties") or {}).get("conditions") or [] + return any(c.get("property") == "description" and c.get("operator") == "NEQ" for c in conds) + + +def _classified(body, name): + conds = (body.get("matchClassifications") or {}).get("conditions") or [] + return any(c.get("name") == name for c in conds) + + @pytest.fixture(autouse=True) def _reset_seam_caches(): """The count seam caches native-method detection per server; isolate tests.""" @@ -53,7 +68,7 @@ def test_count_elements_native_returns_plain_int(): def test_count_elements_falls_back_when_no_native_method(): mgr = _mgr() mgr.find_metadata_elements.return_value = [{"a": 1}, {"b": 2}, {"c": 3}] - n = om.count_elements(mgr, type_name="Asset") + n = om.count_elements(mgr, type_name="Asset", population="viewer") assert n == 3 mgr.find_metadata_elements.assert_called_once() @@ -61,11 +76,11 @@ def test_count_elements_falls_back_when_no_native_method(): def test_count_elements_falls_back_when_native_raises(): mgr = _mgr(count_metadata_elements=MagicMock(side_effect=RuntimeError("not supported"))) mgr.find_metadata_elements.return_value = [{"a": 1}] - n = om.count_elements(mgr, type_name="Asset") + n = om.count_elements(mgr, type_name="Asset", population="viewer") assert n == 1 # second call should skip native entirely (server marked unsupported) mgr.count_metadata_elements.reset_mock() - om.count_elements(mgr, type_name="Asset") + om.count_elements(mgr, type_name="Asset", population="viewer") mgr.count_metadata_elements.assert_not_called() @@ -73,7 +88,7 @@ def test_count_elements_passes_as_of_and_classifications(): mgr = _mgr() mgr.find_metadata_elements.return_value = [] om.count_elements(mgr, type_name="Asset", as_of="2026-01-01T00:00:00+00:00", - classifications=["ZoneMembership", "Confidentiality"]) + classifications=["ZoneMembership", "Confidentiality"], population="viewer") body = mgr.find_metadata_elements.call_args[0][0] assert body["asOfTime"] == "2026-01-01T00:00:00+00:00" assert body["matchClassifications"]["matchCriteria"] == "ANY" @@ -83,10 +98,10 @@ def test_count_elements_passes_as_of_and_classifications(): # ── count_relationships ────────────────────────────────────────────────────── -def test_count_relationships_default_path(): +def test_count_relationships_viewer_path_without_expert(): ce = MagicMock() ce.get_relationships.return_value = [{"a": 1}, {"b": 2}] - n = om.count_relationships(ce, "Certification") + n = om.count_relationships(ce, "Certification", population="viewer") assert n == 2 @@ -107,8 +122,7 @@ def test_count_relationships_native_fast_path_via_expert(): # ── counts_by_type ──────────────────────────────────────────────────────── def test_counts_by_type_composes_count_elements(): - mgr = _mgr() - mgr.find_metadata_elements.side_effect = [[{"a": 1}] * 3, [{"a": 1}] * 5] + mgr = _mgr(count_metadata_elements=MagicMock(side_effect=[3, 5])) rows = om.counts_by_type(mgr, [("Data Stores", "DataStore"), ("Data Sets", "DataSet")]) assert rows == [ {"label": "Data Stores", "type": "DataStore", "count": 3}, @@ -137,8 +151,7 @@ def test_sum_type_counts_runs_the_real_fetch_then_analytic_pipeline(): # End-to-end through run_analytic_action -- not a mocked shortcut -- using # a real MetadataExpert-shaped mock, same convention as the # counts_by_type test above. - mgr = _mgr() - mgr.find_metadata_elements.side_effect = [[{"a": 1}] * 3, [{"a": 1}] * 5] + mgr = _mgr(count_metadata_elements=MagicMock(side_effect=[3, 5])) result = om.sum_type_counts(mgr, [("Data Stores", "DataStore"), ("Data Sets", "DataSet")]) assert result == { "total": 8, @@ -220,13 +233,16 @@ def test_certifications_summary_degrades_on_failure(): # ── semantic_grounding ──────────────────────────────────────────────────── def test_semantic_grounding_computes_percent(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 100 + mgr = _mgr(count_metadata_elements=MagicMock(return_value=100), + count_relationships_between_elements=MagicMock(return_value=61)) ce = MagicMock() - ce.get_relationships.return_value = [{}] * 61 result = om.semantic_grounding(mgr, ce) assert result["groundingLinks"] == 61 assert result["groundingPct"] == 61 + # both counts in the repository population; the relationship count goes through mgr (R4) + assert mgr.count_metadata_elements.call_args.kwargs == {"push_down": True} + assert mgr.count_relationships_between_elements.call_args.kwargs == {"push_down": True} + ce.get_relationships.assert_not_called() def test_semantic_grounding_handles_zero_cataloged(): @@ -239,45 +255,48 @@ def test_semantic_grounding_handles_zero_cataloged(): def test_semantic_grounding_caps_at_100(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 10 + # more links than assets -- shouldn't happen, but don't overshoot 100% + mgr = _mgr(count_metadata_elements=MagicMock(return_value=10), + count_relationships_between_elements=MagicMock(return_value=50)) ce = MagicMock() - ce.get_relationships.return_value = [{}] * 50 # more links than assets -- shouldn't happen, but don't overshoot 100% result = om.semantic_grounding(mgr, ce) assert result["groundingPct"] == 100 # ── context_readiness_funnel / people_counts / usage_context_counts ────── +def _funnel_mgr(): + def answer(body): + if _has_desc_neq(body): + return 2 + if body.get("matchClassifications"): + return 4 + return 5 + return _mgr(count_metadata_elements=_native(answer), + count_relationships_between_elements=MagicMock(return_value=3)) + + def test_context_readiness_funnel_shape(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 5 + mgr = _funnel_mgr() ce = MagicMock() - ce.get_relationships.return_value = [{}] * 3 result = om.context_readiness_funnel(mgr, ce) assert set(result.keys()) == {"cataloged", "documented", "classified", "lineage", "aiReady"} - # documented/lineage are now computed (2026-08-01); aiReady still isn't -- - # it needs a true cross-criteria intersection, not another independent - # count (see NEXT-18, composite/derived analytic metrics). - assert result["documented"] == 0 # 5 elements, none carry a description - assert result["lineage"] == 3 # DataFlow relationship count - assert result["aiReady"] is None + # aiReady still isn't computed -- it needs a true cross-criteria + # intersection, not another independent count (see NEXT-18). + assert result == {"cataloged": 5, "documented": 2, "classified": 4, "lineage": 3, "aiReady": None} + ce.get_relationships.assert_not_called() + mgr.find_metadata_elements.assert_not_called() -def test_context_readiness_funnel_counts_nonempty_description(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [ - {"elementProperties": {"propertyValueMap": { - "description": {"primitiveValue": "A real description."}}}}, - {"elementProperties": {"propertyValueMap": { - "description": {"primitiveValue": " "}}}}, # blank -> not documented - {"elementProperties": {"propertyValueMap": {}}}, # missing -> not documented - ] - ce = MagicMock() - ce.get_relationships.return_value = [] - result = om.context_readiness_funnel(mgr, ce) - assert result["documented"] == 1 - assert result["lineage"] == 0 +def test_context_readiness_funnel_is_all_native_repository_counts(): + mgr = _funnel_mgr() + om.context_readiness_funnel(mgr, MagicMock()) + calls = mgr.count_metadata_elements.call_args_list + mgr.count_relationships_between_elements.call_args_list + assert len(calls) == 4 and all(c.kwargs == {"push_down": True} for c in calls) + documented = [c.args[0] for c in mgr.count_metadata_elements.call_args_list if _has_desc_neq(c.args[0])] + cond = documented[0]["searchProperties"]["conditions"][0] + assert documented[0]["metadataElementTypeName"] == "Asset" + assert (cond["operator"], cond["value"]["primitiveValue"]) == ("NEQ", "") # ── ai_ready_assets ────────────────────────────────────────────────────── @@ -417,65 +436,64 @@ def test_drl_readiness_gates_missing_update_time_not_counted_recent(): assert result["aiReadyRecentCount"] == 0 -def test_business_value_signals_counts_confidential_and_described_from_one_fetch(): - mgr = _mgr() - mgr.find_metadata_elements.side_effect = [ - [ - _asset("a1", "has a description", ["Confidentiality"]), - _asset("a2", None, ["Confidentiality"]), - _asset("a3", "has a description", []), - _asset("a4", None, []), - ], - [], # second find_metadata_elements call: count_elements' ConsolidatedDuplicate fetch - ] +def _bv_mgr(asset=4, confidential=2, described=2, duplicate=0): + def answer(body): + if _classified(body, "ConsolidatedDuplicate"): + return duplicate + if _classified(body, "Confidentiality"): + return confidential + if _has_desc_neq(body): + return described + return asset + return _mgr(count_metadata_elements=_native(answer)) + + +def test_business_value_signals_counts_each_signal_natively(): + mgr = _bv_mgr() result = om.business_value_signals(mgr) assert result == { "assetTotal": 4, "assetCapped": False, "confidentialCount": 2, "describedCount": 2, "duplicateCount": 0, } + mgr.find_metadata_elements.assert_not_called() + bodies = [c.args[0] for c in mgr.count_metadata_elements.call_args_list] + assert all(c.kwargs == {"push_down": True} for c in mgr.count_metadata_elements.call_args_list) + confidential = next(b for b in bodies if _classified(b, "Confidentiality")) + assert confidential["metadataElementTypeName"] == "Asset" # Asset-scoped, unlike governed_coverage -def test_business_value_signals_capped_flag(): - mgr = _mgr() - mgr.find_metadata_elements.side_effect = [ - [_asset(f"a{i}") for i in range(om.DEFAULT_CAP)], - [], - ] - result = om.business_value_signals(mgr) - assert result["assetCapped"] is True +def test_business_value_signals_is_never_capped_at_the_page_size(): + result = om.business_value_signals(_bv_mgr(asset=26994)) + assert result["assetTotal"] == 26994 and result["assetCapped"] is False def test_business_value_signals_counts_duplicates_separately(): - mgr = _mgr() - mgr.find_metadata_elements.side_effect = [ - [_asset("a1")], - [_asset("d1"), _asset("d2")], # ConsolidatedDuplicate-classified elements - ] - result = om.business_value_signals(mgr) - assert result["duplicateCount"] == 2 + assert om.business_value_signals(_bv_mgr(duplicate=2))["duplicateCount"] == 2 def test_business_value_signals_empty_catalog(): - mgr = _mgr() - mgr.find_metadata_elements.side_effect = [[], []] - result = om.business_value_signals(mgr) + result = om.business_value_signals(_bv_mgr(0, 0, 0, 0)) assert result == { "assetTotal": 0, "assetCapped": False, "confidentialCount": 0, "describedCount": 0, "duplicateCount": 0, } +def test_business_value_signals_viewer_population_on_request(): + mgr = _bv_mgr() + om.business_value_signals(mgr, population="viewer") + assert all(c.kwargs == {"push_down": False} for c in mgr.count_metadata_elements.call_args_list) + + def test_people_counts_shape(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 2 + mgr = _mgr(count_metadata_elements=MagicMock(return_value=2)) result = om.people_counts(mgr) assert set(result.keys()) == {"persons", "teams", "organizations", "itProfiles", "communities"} assert all(v == 2 for v in result.values()) def test_usage_context_counts_shape(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 4 + mgr = _mgr(count_metadata_elements=MagicMock(return_value=4)) result = om.usage_context_counts(mgr) assert result == {"informationSupplyChains": 4, "blueprints": 4} @@ -485,26 +503,43 @@ def test_usage_context_counts_shape(): def test_feedback_summary_sums_by_type(): ce = MagicMock() ce.get_relationships.side_effect = [[{}] * 3, [{}] * 1, [], [{}] * 2, []] - result = om.feedback_summary(ce) + result = om.feedback_summary(ce, population="viewer") assert result["byType"] == {"ratings": 3, "comments": 1, "likes": 0, "tags": 2, "noteLogs": 0} assert result["total"] == 6 +def test_feedback_summary_counts_the_repository_through_a_client_with_the_native_count(): + # PlainSupply hands an EgeriaTech for ce, which has the native relationship count. + ce = _mgr(count_relationships_between_elements=MagicMock(side_effect=[3, 1, 0, 2, 0]), get_relationships=MagicMock()) + result = om.feedback_summary(ce) + assert result["total"] == 6 + assert all(c.kwargs == {"push_down": True} for c in ce.count_relationships_between_elements.call_args_list) + ce.get_relationships.assert_not_called() + + # ── growth_series ───────────────────────────────────────────────────────── def test_growth_series_point_count_and_ordering(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 3 + mgr = _mgr(count_metadata_elements=MagicMock(return_value=3)) series = om.growth_series(mgr, window="7d", points=4) assert len(series) == 4 # last point (index -1) is "now" -- has a label/date but the loop counts down to i=0 assert all("label" in p and "date" in p for p in series) assert all(p["assets"] == 3 for p in series) + assert all(c.kwargs == {"push_down": True} for c in mgr.count_metadata_elements.call_args_list) + mgr.find_metadata_elements.assert_not_called() + + +def test_growth_series_viewer_population_on_request(): + mgr = _mgr(count_metadata_elements=MagicMock(return_value=3)) + om.growth_series(mgr, window="7d", points=2, population="viewer") + assert all(c.kwargs == {"push_down": False} for c in mgr.count_metadata_elements.call_args_list) + with pytest.raises(ValueError): + om.growth_series(mgr, window="7d", points=2, population="everyone") def test_growth_series_custom_type_map(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 9 + mgr = _mgr(count_metadata_elements=MagicMock(return_value=9)) series = om.growth_series(mgr, window="1d", points=2, type_map=[("widgets", "Widget", None)]) assert len(series) == 2 @@ -522,26 +557,34 @@ def test_growth_label_granularity(): # ── term_definition_completeness ────────────────────────────────────────── -def test_term_definition_completeness_counts_nonempty_description(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [ - {"elementProperties": {"propertyValueMap": { - "description": {"primitiveValue": "A real definition."}}}}, - {"elementProperties": {"propertyValueMap": { - "description": {"primitiveValue": " "}}}}, # blank -> not defined - {"elementProperties": {"propertyValueMap": {}}}, # missing -> not defined - ] +def test_term_definition_completeness_divides_two_repository_counts(): + mgr = _mgr(count_metadata_elements=_native(lambda b: 536 if _has_desc_neq(b) else 1061)) result = om.term_definition_completeness(mgr) - assert result == {"total": 3, "defined": 1, "undefinedPct": 67} + assert result == {"total": 1061, "defined": 536, "undefinedPct": 49} + calls = mgr.count_metadata_elements.call_args_list + assert len(calls) == 2 and all(c.kwargs == {"push_down": True} for c in calls) + assert all(c.args[0]["metadataElementTypeName"] == "GlossaryTerm" for c in calls) + mgr.find_metadata_elements.assert_not_called() # no capped scan: the total is never the page size def test_term_definition_completeness_empty_glossary(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [] + mgr = _mgr(count_metadata_elements=MagicMock(return_value=0)) result = om.term_definition_completeness(mgr) assert result == {"total": 0, "defined": 0, "undefinedPct": None} +def test_term_definition_completeness_unmeasured_numerator_is_no_percentage(): + def answer(body): + if _has_desc_neq(body): + raise RuntimeError("TIMEOUT_ERROR_408 timed out") + return 1061 + mgr = _mgr(count_metadata_elements=MagicMock(side_effect=lambda b, **kw: {"count": answer(b)})) + with om.recording_unmeasured() as notes: + result = om.term_definition_completeness(mgr) + assert result == {"total": 1061, "defined": None, "undefinedPct": None} + assert notes[0]["count"] == "GlossaryTerm with description != ''" + + # ── active_contributors ─────────────────────────────────────────────────── def test_active_contributors_dedupes_by_username_across_types(): @@ -576,8 +619,7 @@ def test_active_contributors_degrades_per_type_on_failure(): # ── metric_trend ────────────────────────────────────────────────────────── def test_metric_trend_calls_target_once_per_point_and_merges_dict_result(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [{}] * 3 + mgr = _mgr(count_metadata_elements=MagicMock(return_value=3)) series = om.metric_trend( mgr, MagicMock(), "people_counts", window="7d", points=3, @@ -637,10 +679,19 @@ def test_count_elements_by_property_uses_native_when_available(): def test_count_elements_by_property_falls_back_when_no_native_method(): mgr = _mgr() mgr.find_metadata_elements.return_value = [{}, {}, {}] - n = om.count_elements_by_property(mgr, "DigitalProduct", "deploymentStatus", "ACTIVE") + n = om.count_elements_by_property(mgr, "DigitalProduct", "deploymentStatus", "ACTIVE", population="viewer") assert n == 3 +def test_count_elements_by_property_neq_and_bad_operator(): + mgr = _mgr(count_metadata_elements=MagicMock(return_value={"count": 536})) + assert om.count_elements_by_property(mgr, "GlossaryTerm", "description", "", operator="NEQ") == 536 + cond = mgr.count_metadata_elements.call_args[0][0]["searchProperties"]["conditions"][0] + assert cond["operator"] == "NEQ" and cond["value"]["primitiveValue"] == "" + with pytest.raises(ValueError, match="operator"): + om.count_elements_by_property(mgr, "GlossaryTerm", "description", "", operator="LIKE") + + def test_count_elements_by_property_passes_as_of(): mgr = _mgr(count_metadata_elements=MagicMock(return_value={"count": 1})) om.count_elements_by_property(mgr, "DigitalProduct", "deploymentStatus", "ACTIVE", as_of="2026-01-01T00:00:00+00:00") @@ -716,22 +767,24 @@ def test_contextualised_coverage_zero_asset_total_yields_none_pct_not_zero_divis # ── unmeasured != zero (overview defects: egeria-workspaces#783) ───────── def test_funnel_keeps_measured_zero_and_none_on_failure(): - mgr = _mgr() - mgr.find_metadata_elements.return_value = [] + mgr = _mgr(count_metadata_elements=MagicMock(return_value=0), + count_relationships_between_elements=MagicMock(return_value=0)) ce = MagicMock() - ce.get_relationships.return_value = [] r = om.context_readiness_funnel(mgr, ce) - assert r["cataloged"] == 0 and r["classified"] == 0 + assert r["cataloged"] == 0 and r["classified"] == 0 and r["documented"] == 0 and r["lineage"] == 0 - mgr.find_metadata_elements.side_effect = RuntimeError("down") + mgr.count_metadata_elements.side_effect = RuntimeError("down") + mgr.count_relationships_between_elements.side_effect = RuntimeError("down") + mgr.find_metadata_elements.return_value = [] # a viewer answer must never stand in r = om.context_readiness_funnel(mgr, ce) assert r["cataloged"] is None and r["classified"] is None + assert r["documented"] is None and r["lineage"] is None def test_feedback_summary_total_none_unless_all_measured(): ce = MagicMock() ce.get_relationships.side_effect = [[{}] * 3, RuntimeError("x"), [], [], []] - r = om.feedback_summary(ce) + r = om.feedback_summary(ce, population="viewer") assert r["byType"]["comments"] is None and r["byType"]["ratings"] == 3 assert r["total"] is None @@ -752,7 +805,7 @@ def test_timeout_does_not_disable_native_counting(): from pyegeria.core._exceptions import PyegeriaTimeoutException mgr = _mgr(count_metadata_elements=MagicMock(side_effect=[PyegeriaTimeoutException(), 987])) mgr.find_metadata_elements.return_value = [{}] * 3 - assert om.count_elements(mgr, "Asset") == 3 # that one count falls back + assert om.count_elements(mgr, "Asset", population="viewer") == 3 # that one count falls back assert not om._native_disabled(mgr) assert om.count_elements(mgr, "GlossaryTerm", strict=True) == 987 # native still used @@ -773,9 +826,9 @@ def test_not_supported_error_disables_native(): om._native_server_ok.clear(); om._count_caps.clear() mgr = _mgr(count_metadata_elements=MagicMock(side_effect=exc)) mgr.find_metadata_elements.return_value = [{}] * 2 - assert om.count_elements(mgr, "Asset") == 2 + assert om.count_elements(mgr, "Asset", population="viewer") == 2 assert om._native_disabled(mgr) - om.count_elements(mgr, "Asset") + om.count_elements(mgr, "Asset", population="viewer") assert mgr.count_metadata_elements.call_count == 1 # not retried @@ -795,9 +848,15 @@ def test_scope_maps_to_push_down(scope, expected): assert mgr.count_metadata_elements.call_args.kwargs == {"push_down": expected} -def test_scope_none_omits_push_down(): +def test_default_population_is_repository_push_down(): mgr = _mgr(count_metadata_elements=MagicMock(return_value=11)) om.count_elements(mgr, "Asset") + assert mgr.count_metadata_elements.call_args.kwargs == {"push_down": True} + + +def test_scope_none_omits_push_down(): + mgr = _mgr(count_metadata_elements=MagicMock(return_value=11)) + om.count_elements(mgr, "Asset", scope=None) # the server default stays reachable assert mgr.count_metadata_elements.call_args.kwargs == {} @@ -809,9 +868,11 @@ def test_invalid_scope_raises(): def test_repository_scope_failure_strict_is_none_not_find_result(): mgr = _mgr(count_metadata_elements=MagicMock(side_effect=RuntimeError("boom"))) mgr.find_metadata_elements.return_value = [{}] * 43 # viewer-scope answer - assert om.count_elements(mgr, "Asset", strict=True, population="repository") is None - assert om.count_elements(mgr, "Asset", population="repository") == 0 + with om.recording_unmeasured() as notes: + assert om.count_elements(mgr, "Asset", strict=True, population="repository") is None + assert om.count_elements(mgr, "Asset", population="repository") is None # never 0, strict or not mgr.find_metadata_elements.assert_not_called() + assert len(notes) == 2 and all("has no fallback" in n["reason"] for n in notes) def test_repository_scope_without_native_method_is_none(): @@ -840,10 +901,18 @@ def test_count_relationships_scope(): ce.get_relationships.return_value = [{}] * 9 assert om.count_relationships(ce, "License", expert=expert, population="repository") == 4 assert expert.count_relationships_between_elements.call_args.kwargs == {"push_down": True} - # no expert: repository scope cannot be answered by the viewer-scoped path - assert om.count_relationships(ce, "License", population="repository") is None - assert om.count_relationships(ce, "License") == 9 - assert om.feedback_summary(ce, population="repository")["total"] is None + # no expert, and ce has no native count: repository scope cannot be answered by the viewer-scoped path + ce = MagicMock(spec=["get_relationships"]) + ce.get_relationships.return_value = [{}] * 9 + with om.recording_unmeasured() as notes: + assert om.count_relationships(ce, "License", population="repository") is None + assert om.count_relationships(ce, "License") is None # repository is the default + assert notes[0] == {"count": "License", "at_least": None, "reason": ( + "no native relationship count is available (it needs a MetadataExpert client), and a " + "repository-population count has no fallback (get_relationships only sees what the viewer may see)")} + assert om.count_relationships(ce, "License", population="viewer") == 9 + assert om.feedback_summary(ce)["total"] is None + assert om.feedback_summary(ce, population="viewer")["total"] == 45 # ── asset_subtype_tree / asset_composition (RS-W1-4) ───────────────────────── diff --git a/tests/micro-tests/test_rs_w2_3_inputs_by_role.py b/tests/micro-tests/test_rs_w2_3_inputs_by_role.py index 1713d9a4..fab2a7de 100644 --- a/tests/micro-tests/test_rs_w2_3_inputs_by_role.py +++ b/tests/micro-tests/test_rs_w2_3_inputs_by_role.py @@ -52,6 +52,10 @@ def count_metadata_elements(self, body, **kw): self.calls.append(("count_metadata_elements", body.get("metadataElementTypeName"))) return 7 + def count_relationships_between_elements(self, body, **kw): + self.calls.append(("count_relationships_between_elements", body.get("relationshipTypeName"))) + return 3 + def get_relationships(self, relationship_type=None, **kw): self.calls.append(("get_relationships", relationship_type)) return [{"relationshipHeader": {"guid": "r1"}, "end1": {"guid": "g1"}, "end2": {"guid": "g2"}}] @@ -263,7 +267,8 @@ def spy(mgr, ce, **kw): assert result["kind"] == "json" and set(result["data"]) == {"groundingLinks", "groundingPct"} assert len(fake_tech) == 1 and fake_tech[0].tokens == ["caller-token"] and fake_tech[0].minted == 0 assert seen["mgr"] is seen["ce"] is fake_tech[0] - assert {c[0] for c in fake_tech[0].calls} == {"count_metadata_elements", "get_relationships"} + # both counts native in the repository population; the relationship count goes through mgr (R4) + assert {c[0] for c in fake_tech[0].calls} == {"count_metadata_elements", "count_relationships_between_elements"} def test_end_to_end_an_environment_supply_is_asked_by_declared_class_in_order(fake_tech): From 4c64296d19b3edc39be783611e378244b77a12fa Mon Sep 17 00:00:00 2001 From: Dan Wolfson Date: Sun, 8 Nov 2026 20:13:36 +0000 Subject: [PATCH 2/2] fix(overview-metrics): repository relationship counts get a client that can answer them Read-back P1-1 on O-cap-1: feedback_summary and count_relationships declared egeria_client:ClassificationExplorer, which is what the Portal's supply builds, and a ClassificationExplorer has no native count_relationships_between_elements. With the repository default, every feedback count on the Portal was None ("needs a MetadataExpert client") where it used to be 61. - feedback_summary and count_relationships declare egeria_client:EgeriaTech (both get_relationships and the native count), as asset_composition does. - Sweep: metric_trend binds a target's ce slot to its EgeriaTech when the target declares EgeriaTech there (an input may be declared only once, so metric_trend keeps [EgeriaTech, ClassificationExplorer]). semantic_grounding and context_readiness_funnel count through mgr; certifications_summary is viewer-scoped. - Composites forward the server default (scope=None) explicitly. The keyword wrapper reads population=None as "not given", so it had become a repository count inside counts_by_type, people_counts, usage_context_counts, feedback_summary, business_value_signals and growth_series. - P3: count_elements / count_elements_by_property docstrings and descriptions say the capped-find fallback is viewer-only; assetCapped's label says it is empty when not measured. Tests run feedback_summary and count_relationships through a Portal-shaped supply that builds exactly the declared class, and sweep every demo spec through it for a missing native relationship count. Signed-off-by: Dan Wolfson --- pyegeria/view/analytic_registry.py | 29 ++-- pyegeria/view/overview_metrics.py | 79 ++++++---- tests/micro-tests/test_overview_metrics.py | 16 ++ .../test_portal_supply_relationship_counts.py | 145 ++++++++++++++++++ 4 files changed, 231 insertions(+), 38 deletions(-) create mode 100644 tests/micro-tests/test_portal_supply_relationship_counts.py diff --git a/pyegeria/view/analytic_registry.py b/pyegeria/view/analytic_registry.py index a30bc780..90d68e55 100644 --- a/pyegeria/view/analytic_registry.py +++ b/pyegeria/view/analytic_registry.py @@ -850,8 +850,9 @@ def _always_applies(what: str) -> str: description="Count active elements of a given type, optionally filtered by classification. " "Cost: one native count, sub-second in the repository population (0.2-0.8 s " "measured); a viewer-population count reads every element and can take minutes " - "for a large type (Process: 120 s); a server with no native count falls back to " - "a capped find." + REPOSITORY_WORDS, + "for a large type (Process: 120 s). Only in the viewer population does a server " + "with no native count fall back to a capped find; a repository count has no " + "fallback and is not measured instead." + REPOSITORY_WORDS, returns="scalar (int)", generic=True, params=[ @@ -1027,7 +1028,7 @@ def _always_applies(what: str) -> str: ), out=[ _field("assetTotal", "Assets", "int", unit="Assets"), - _field("assetCapped", "At least (always false: a native count is complete)", "bool"), + _field("assetCapped", "At least (false when counted: a native count is complete; empty when not measured)", "bool"), _field("confidentialCount", "Confidential Assets", "int", unit="Assets"), _field("describedCount", "Described Assets", "int", unit="Assets"), _field("duplicateCount", "Duplicates", "int", unit="elements"), @@ -1108,11 +1109,15 @@ def _always_applies(what: str) -> str: "feedback_summary": AnalyticFunctionSpec( name="feedback_summary", function="pyegeria.view.overview_metrics.feedback_summary", - inputs=["egeria_client:ClassificationExplorer"], + # EgeriaTech, not ClassificationExplorer: a repository count needs the native relationship count + # (MetadataExpert's), which a ClassificationExplorer lacks; EgeriaTech has both it and get_relationships. + inputs=["egeria_client:EgeriaTech"], description="Crowd-sourced feedback counts (ratings/comments/likes/tags/noteLogs) by type, " "via Collaboration OMAS relationships. " + "Takes an EgeriaTech: the repository count is MetadataExpert's native relationship count, " + "and the viewer path is ClassificationExplorer's get_relationships. " "Cost: five native counts, sub-second each in the repository population; population=viewer " - "without a MetadataExpert reads five capped relationship pages instead." + REPOSITORY_WORDS, + "reads five capped relationship pages instead." + REPOSITORY_WORDS, returns="dict (byType, total)", generic=False, binding_note="Fixed to Collaboration OMAS's feedback relationship types -- not a parameter.", @@ -1519,9 +1524,10 @@ def _always_applies(what: str) -> str: function="pyegeria.view.overview_metrics.count_elements_by_property", inputs=["egeria_client:MetadataExpert"], description="Count active elements of a given type whose named string property equals a " - "given value -- e.g. DigitalProduct elements with deploymentStatus=ACTIVE. Same " - "native-count-with-fallback seam as count_elements, one cheap native COUNT call " - "per distinct value, no element fetch. " + "given value (or, with operator NEQ, is set and differs) -- e.g. DigitalProduct elements with " + "deploymentStatus=ACTIVE. Same count seam as count_elements: one cheap native COUNT " + "call per distinct value, no element fetch; a capped-find fallback only in the " + "viewer population. " "Cost: one native count, sub-second in the repository population; a viewer-population count can take minutes for a large type." + REPOSITORY_WORDS, returns="scalar (int)", generic=True, @@ -1557,12 +1563,13 @@ def _always_applies(what: str) -> str: "count_relationships": AnalyticFunctionSpec( name="count_relationships", function="pyegeria.view.overview_metrics.count_relationships", - inputs=["egeria_client:ClassificationExplorer"], + # EgeriaTech, as for feedback_summary: the repository count needs MetadataExpert's native method. + inputs=["egeria_client:EgeriaTech"], description="Count active relationships of one type. By default a native count over the " "whole repository, which needs a client with the native count (EgeriaTech has it; " "a bare ClassificationExplorer does not, and gives not measured). population=viewer " "without a MetadataExpert reads one capped get_relationships page instead (the two " - "paths can disagree for some types, PY-18). Takes ce as its leading client. " + "paths can disagree for some types, PY-18). Takes an EgeriaTech as its leading client. " "Cost: the native count is sub-second; a relationship page takes seconds." + REPOSITORY_WORDS, returns="scalar (int | None)", generic=True, @@ -1821,6 +1828,8 @@ def _always_applies(what: str) -> str: "metric_trend": AnalyticFunctionSpec( name="metric_trend", function="pyegeria.view.overview_metrics.metric_trend", + # A target that declares EgeriaTech for a slot (feedback_summary, count_relationships: a repository + # relationship count needs MetadataExpert's native method) gets the EgeriaTech in that slot. inputs=["egeria_client:EgeriaTech", "egeria_client:ClassificationExplorer"], description="Turn any single-snapshot function in this module into a time series, by " "re-calling it once per asOfTime snapshot -- the same windowing growth_series " diff --git a/pyegeria/view/overview_metrics.py b/pyegeria/view/overview_metrics.py index 84dc1960..9ef72047 100644 --- a/pyegeria/view/overview_metrics.py +++ b/pyegeria/view/overview_metrics.py @@ -304,6 +304,13 @@ def _push_down(scope: Optional[str]) -> Optional[bool]: raise ValueError(f"scope must be None, {SCOPE_VIEWER!r} or {SCOPE_REPOSITORY!r}; got {scope!r}") +def _pop(scope: Optional[str]) -> Dict[str, Optional[str]]: + """Keyword arguments that forward a composite's population to a count helper. `population=None` + means "not given" to the keyword wrapper (the helper's repository default then applies), so the + server default, scope=None, is forwarded as `scope=None` instead -- explicitly, not lost.""" + return {"population": scope} if scope is not None else {"scope": None} + + # ── not-measured notes ───────────────────────────────────────────────────── # A count that comes back None is "not measured", never 0 and never the find # cap. The value alone cannot say why, or what lower bound the fallback did @@ -429,14 +436,17 @@ def count_elements( ) -> Optional[int]: """ Count ACTIVE elements, optionally of a given type and/or matching ANY of - the given classifications. Uses native `count_metadata_elements` when the - client + server support it, else `len(find_metadata_elements(...))`. + the given classifications, with native `count_metadata_elements`. - By default a failed query counts as 0, so an outage looks like an empty - catalog. Pass strict=True to get None instead whenever the number is not - known (ISSUE-123). Strict or not, a find fallback that stops at DEFAULT_CAP - gives None, never the cap: that page is a lower bound, not a total (its - bound and reason are recorded for `recording_unmeasured()`). + In the repository population (the default) only the native count can + answer: when it is unavailable, fails or times out the result is None, + strict or not, with its reason recorded for `recording_unmeasured()`. + + Only in the viewer population (or scope=None) does a missing or failed + native count fall back to `len(find_metadata_elements(...))`. There a + failed query counts as 0 unless strict=True (ISSUE-123), and a fallback + that stops at DEFAULT_CAP gives None, never the cap: that page is a lower + bound, not a total (its bound and reason are recorded too). Parameters ---------- @@ -487,10 +497,11 @@ def count_elements_by_property( Count ACTIVE elements of a given type whose named string property equals a given value -- e.g. counting DigitalProduct elements by deploymentStatus (Overview dashboard's Data Products tile, OVERVIEW_NEXT_STEPS.md - "Remaining app wiring"). Same native-count-with-fallback seam as - count_elements, just with a searchProperties EQ condition instead of a - bare type filter -- one cheap native COUNT call per distinct value, no - element fetch, same cost class as the plain count. + "Remaining app wiring"). Same count seam as count_elements -- native + only in the repository population, a capped-find fallback only in the + viewer population -- just with a searchProperties EQ (or NEQ) condition + instead of a bare type filter: one cheap native COUNT call per distinct + value, no element fetch, same cost class as the plain count. Parameters ---------- @@ -545,7 +556,9 @@ def count_relationships(ce, relationship_type: str, as_of: Optional[str] = None, Parameters ---------- - ce : ClassificationExplorer (or compatible client) + ce : EgeriaTech (or another client with both get_relationships and the native + count_relationships_between_elements; a bare ClassificationExplorer has + only get_relationships, so it can answer a viewer count only) relationship_type : the relationship type name to count as_of : ISO-8601 asOfTime for a point-in-time count (None = now) expert : optional MetadataExpert to enable the native fast-path. With @@ -621,7 +634,7 @@ def counts_by_type(mgr, type_map: Sequence[Tuple[str, str]], as_of: Optional[str Returns a list of {"label": ..., "type": ..., "count": ...} dicts, in the order given. """ - return [{"label": label, "type": type_name, "count": count_elements(mgr, type_name, as_of, strict=strict, population=scope)} + return [{"label": label, "type": type_name, "count": count_elements(mgr, type_name, as_of, strict=strict, **_pop(scope))} for label, type_name in type_map] @@ -872,16 +885,16 @@ def business_value_signals(mgr, as_of: Optional[str] = None, "confidentialCount": int|None, "describedCount": int|None, "duplicateCount": int|None}. """ - total = count_elements(mgr, "Asset", as_of, strict=True, population=scope) + total = count_elements(mgr, "Asset", as_of, strict=True, **_pop(scope)) return { "assetTotal": total, "assetCapped": None if total is None else False, "confidentialCount": count_elements(mgr, "Asset", as_of, classifications=["Confidentiality"], - strict=True, population=scope), + strict=True, **_pop(scope)), "describedCount": count_elements_by_property(mgr, "Asset", "description", "", as_of, strict=True, - population=scope, operator="NEQ"), + **_pop(scope), operator="NEQ"), "duplicateCount": count_elements(mgr, None, as_of, classifications=["ConsolidatedDuplicate"], - strict=True, population=scope), + strict=True, **_pop(scope)), } @@ -1379,11 +1392,11 @@ def people_counts(mgr, as_of: Optional[str] = None, strict: bool = False, scope: could not be measured is None (see count_elements). """ return { - "persons": count_elements(mgr, "Person", as_of, strict=strict, population=scope), - "teams": count_elements(mgr, "Team", as_of, strict=strict, population=scope), - "organizations": count_elements(mgr, "Organization", as_of, strict=strict, population=scope), - "itProfiles": count_elements(mgr, "ITProfile", as_of, strict=strict, population=scope), - "communities": count_elements(mgr, "Community", as_of, strict=strict, population=scope), + "persons": count_elements(mgr, "Person", as_of, strict=strict, **_pop(scope)), + "teams": count_elements(mgr, "Team", as_of, strict=strict, **_pop(scope)), + "organizations": count_elements(mgr, "Organization", as_of, strict=strict, **_pop(scope)), + "itProfiles": count_elements(mgr, "ITProfile", as_of, strict=strict, **_pop(scope)), + "communities": count_elements(mgr, "Community", as_of, strict=strict, **_pop(scope)), } @@ -1413,7 +1426,7 @@ def feedback_summary(ce, as_of: Optional[str] = None, scope: Optional[str] = SCO Returns {"byType": {ratings, comments, likes, tags, noteLogs: int|None}, "total": int|None}. """ - by_type = {key: count_relationships(ce, rel, as_of, expert=expert, population=scope) for rel, key in _FEEDBACK_RELATIONSHIP_TYPES} + by_type = {key: count_relationships(ce, rel, as_of, expert=expert, **_pop(scope)) for rel, key in _FEEDBACK_RELATIONSHIP_TYPES} measured = all(v is not None for v in by_type.values()) return {"byType": by_type, "total": sum(by_type.values()) if measured else None} @@ -1429,8 +1442,8 @@ def usage_context_counts(mgr, as_of: Optional[str] = None, strict: bool = False, strict=True, a count that could not be measured is None (see count_elements). """ return { - "informationSupplyChains": count_elements(mgr, "InformationSupplyChain", as_of, strict=strict, population=scope), - "blueprints": count_elements(mgr, "SolutionBlueprint", as_of, strict=strict, population=scope), + "informationSupplyChains": count_elements(mgr, "InformationSupplyChain", as_of, strict=strict, **_pop(scope)), + "blueprints": count_elements(mgr, "SolutionBlueprint", as_of, strict=strict, **_pop(scope)), } @@ -1762,7 +1775,7 @@ def growth_series( try: for key, type_name, classifications in type_map: point[key] = count_elements(mgr, type_name, as_of, classifications=classifications, strict=True, - population=population) + **_pop(population)) except Exception as exc: # noqa: BLE001 logger.debug(f"overview_metrics growth_series: snapshot {i} failed: {exc}") for key, _, _ in type_map: @@ -1804,6 +1817,11 @@ def metric_trend( Pass both `mgr` and `ce` regardless of what the target function actually needs -- only the client parameter names it declares (by leading positional parameter named "mgr"/"ce") are bound; the other is unused. + `mgr` is an EgeriaTech: a slot the target declares as + `egeria_client:EgeriaTech` gets `mgr`, even one named `ce` -- + feedback_summary and count_relationships count repository relationships + through `ce`, which needs MetadataExpert's native count, and a + ClassificationExplorer lacks it. `metric_params` are extra keyword arguments forwarded to the target function at every snapshot (e.g. `classifications` for `count_elements`); @@ -1821,12 +1839,17 @@ def metric_trend( func = resolve_report_spec_function(metric_path) clients = {"mgr": mgr, "ce": ce} + from pyegeria.view.analytic_registry import get_analytic_registry # lazy, as above + target = get_analytic_registry().get(metric_path) + declared = list(getattr(target, "inputs", None) or []) bound_args = [] for pname, param in inspect.signature(func).parameters.items(): if pname in _CLIENT_PARAM_NAMES and param.kind in ( inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD, ): - bound_args.append(clients[pname]) + i = len(bound_args) + wants_tech = i < len(declared) and declared[i] == "egeria_client:EgeriaTech" + bound_args.append(mgr if wants_tech else clients[pname]) else: break @@ -2112,7 +2135,7 @@ def _count_cell(mgr, type_name: str, as_of: Optional[str], scope: str, stamp: st reason = "" try: with recording_unmeasured() as notes: - value = count_elements(mgr, type_name, as_of, strict=True, population=scope) + value = count_elements(mgr, type_name, as_of, strict=True, **_pop(scope)) if value is None: reason = (f"the {scope} count of {type_name} could not be measured: " + ("; ".join(n["reason"] for n in notes) if notes diff --git a/tests/micro-tests/test_overview_metrics.py b/tests/micro-tests/test_overview_metrics.py index 8e7aee18..730a716b 100644 --- a/tests/micro-tests/test_overview_metrics.py +++ b/tests/micro-tests/test_overview_metrics.py @@ -1174,3 +1174,19 @@ def test_asset_composition_scopes_alias_warns_and_matches_populations(): assert old["populations"] == old["scopes"] == ["repository"] with pytest.raises(ValueError, match="disagree"): om.asset_composition(MagicMock(), _FakeVM(), depth=1, scopes=("viewer",), populations=("repository",)) + + +def test_scope_none_reaches_the_server_default_through_every_composite(): + # population=None means "not given" to the keyword wrapper, so a composite must forward the + # server default as scope=None explicitly, or its counts silently become repository counts. + mgr = _mgr(count_metadata_elements=MagicMock(return_value=2)) + om.counts_by_type(mgr, [("A", "Asset")], scope=None) + om.people_counts(mgr, scope=None) + om.usage_context_counts(mgr, scope=None) + om.business_value_signals(mgr, scope=None) + om.growth_series(mgr, window="7d", points=2, population=None) + calls = mgr.count_metadata_elements.call_args_list + assert len(calls) == 1 + 5 + 2 + 4 + 2 * 4 and all(c.kwargs == {} for c in calls) + ce = MagicMock() + ce.get_relationships.return_value = [{}] * 2 + assert om.feedback_summary(ce, scope=None)["total"] == 10 # the get_relationships path, not None diff --git a/tests/micro-tests/test_portal_supply_relationship_counts.py b/tests/micro-tests/test_portal_supply_relationship_counts.py new file mode 100644 index 00000000..d879541a --- /dev/null +++ b/tests/micro-tests/test_portal_supply_relationship_counts.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +""" +A repository relationship count needs MetadataExpert's native `count_relationships_between_elements`. +A ClassificationExplorer has only `get_relationships`, and the Portal's supply builds exactly the +class a routine declares. So a routine that counts repository relationships through its `ce` slot +must declare a client that has the native method (EgeriaTech), or on the Portal every count is None: +"no native relationship count is available" (O-cap-1 read-back P1-1, 2026-11-08). + +These tests run the routines through a RoleSupply that, like the Portal's, hands out a client shaped +like the class it is asked for, and check the declared class is the one used. +""" +import pytest + +import pyegeria.view.base_report_formats as brf +import pyegeria.view.overview_metrics as om +from pyegeria.view import format_set_executor as fse +from pyegeria.view._output_format_models import ActionParameter, Format, FormatSet +from pyegeria.view.analytic_demo_specs import get_analytic_demo_specs +from pyegeria.view.analytic_registry import get_analytic_registry +from pyegeria.view.role_supply import Subject, Viewer + +NO_NATIVE = "no native relationship count is available" + + +class _CE: + """ClassificationExplorer-shaped: get_relationships only (checked live: no native count).""" + + def __init__(self, calls): + self.calls = calls + + def get_relationships(self, relationship_type=None, **kw): + self.calls.append(("ClassificationExplorer.get_relationships", relationship_type)) + return [{"relationshipHeader": {"guid": "r1"}}] + + +class _Tech(_CE): + """EgeriaTech/MetadataExpert-shaped: the native counts as well as get_relationships.""" + + def count_relationships_between_elements(self, body, **kw): + self.calls.append(("count_relationships_between_elements", body["relationshipTypeName"], kw)) + return 12 + + def count_metadata_elements(self, body, **kw): + self.calls.append(("count_metadata_elements", body.get("metadataElementTypeName"), kw)) + return 100 + + def find_metadata_elements(self, body, *a, **kw): + return [] + + +class _PortalShapedSupply: + """Hands out a distinct client per declared class, shaped like that class, as the Portal does.""" + + def __init__(self): + self.asked, self.calls = [], [] + + def client(self, cls): + self.asked.append(cls.__name__) + return _CE(self.calls) if cls.__name__ == "ClassificationExplorer" else _Tech(self.calls) + + def viewer(self): + return Viewer(user_id="u", is_admin=False, display_name="U") + + def reader(self, source): + return None + + def subject(self): + return Subject(guid="g", type_name="Asset") + + def refusal_words(self, kind): + return f"cannot supply {kind}" + + +@pytest.fixture(autouse=True) +def _clean(): + om._count_caps.clear() + om._native_server_ok.clear() + yield + om._count_caps.clear() + om._native_server_ok.clear() + brf.clear_runtime_report_specs() + + +def _run(label, supply, **params): + return fse.exec_report_spec(label, output_format="DICT", params=params, view_server="vs", + view_url="https://x", user="u", user_pass="p", supply=supply) + + +def test_feedback_summary_on_a_portal_shaped_supply_counts_the_repository(): + supply = _PortalShapedSupply() + out = _run("Analytic Demo - Feedback Summary", supply) + assert supply.asked == ["EgeriaTech"] # the declared class is the one used + assert out["kind"] == "json" and "unmeasured" not in out + assert out["data"]["total"] == 5 * 12 + rel = [c for c in supply.calls if c[0] == "count_relationships_between_elements"] + assert len(rel) == 5 and all(c[2] == {"push_down": True} for c in rel) + + +def test_count_relationships_on_a_portal_shaped_supply_counts_the_repository(): + brf.register_report_specs({"P1-1 Count Relationships": FormatSet( + heading="P1-1", description="probe", target_type="Referenceable", + action=ActionParameter(analytic_function="count_relationships"), + formats=[Format(types=["DICT"], attributes=[])])}, source="test") + supply = _PortalShapedSupply() + out = _run("P1-1 Count Relationships", supply, relationship_type="SemanticAssignment") + assert supply.asked == ["EgeriaTech"] + assert out == {"kind": "json", "data": 12} + + +def test_a_classification_explorer_alone_cannot_count_the_repository(): + # Why the declaration matters: handed a ClassificationExplorer, the count is not measured. + with om.recording_unmeasured() as notes: + assert om.feedback_summary(_CE([]))["total"] is None + assert notes and all(n["reason"].startswith(NO_NATIVE) for n in notes) + + +@pytest.mark.parametrize("name", ["feedback_summary", "count_relationships"]) +def test_routines_counting_repository_relationships_through_ce_do_not_declare_a_classification_explorer(name): + assert get_analytic_registry()[name].inputs == ["egeria_client:EgeriaTech"] + + +def test_a_metric_trend_of_feedback_gives_its_egeria_tech_to_the_ce_slot(): + # metric_trend declares [EgeriaTech, ClassificationExplorer] (an input may be declared once only); + # a target that declares EgeriaTech for its ce slot gets metric_trend's EgeriaTech there. + calls = [] + series = om.metric_trend(_Tech(calls), _CE(calls), "feedback_summary", window="7d", points=2) + assert [p["total"] for p in series] == [60, 60] + assert not any(c[0] == "ClassificationExplorer.get_relationships" for c in calls) + + +def test_sweep_no_demo_spec_lacks_a_native_relationship_count_on_a_portal_shaped_supply(): + """Every demo spec, run through a supply that builds exactly the declared classes: none may come + back 'not measured' because its declared client lacks the native relationship count.""" + lacking = [] + for label in get_analytic_demo_specs(): + supply = _PortalShapedSupply() + try: + out = _run(label, supply) + except Exception: # noqa: BLE001 -- a spec that needs params it lacks is not this check's subject + continue + reasons = [n["reason"] for n in (out.get("unmeasured") or [])] if isinstance(out, dict) else [] + if any(r.startswith(NO_NATIVE) for r in reasons): + lacking.append(label) + assert lacking == []