From 587084d6966c31498cdf93cab30910cc0394a611 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 13:30:08 +0100 Subject: [PATCH 01/12] fix(search,datasets): read DAC2a bilateral composites within the bilateral destination population DAC2a aid type 206 is total ODA and stores multilateral agencies and DACAllRecipients as recipients, so an open destination under ONEDAC2aBilateral read agency rows and ONEDAC2aBilateralAndImputedMultilateral counted core contributions twice. A destination role can now be open within a declared population. code_index.RolePopulation declares the bilateral side (207 members, the named destinations and the DACDevelopingCountries total); composites._derived_identity binds it into each component identity and tags it on ComponentIdentity.populations. The mint reads it as dc_sdmx.Among, Search discovers role scope within it, the options door offers only its members, and an absent destination reads DACDevelopingCountries. Unclassified destinations in coverage refuse the composite. An identity binding these composites to an agency or DACAllRecipients refuses with 422. A fresh Search part that settles on a population composite is read again within the population through resolve_spec_resubmit, so fresh answers equal resubmits. The new bilateral_reading_unavailable warning reports when the bilateral default cannot apply. Scoped composite mints now apply the scope's entities and window to every component. The dfo-spec-kit pin moves to 9092fec, which registers bilateral_reading_unavailable. --- docs/conformance-search-api-v1.md | 10 +- packages/dfo-analysis-engine/pyproject.toml | 4 +- packages/dfo-data-tools/pyproject.toml | 10 +- .../assembler/pipeline/export_component.py | 50 +- .../src/code_index/composites.py | 528 ++++++- .../src/code_index/declarations.py | 23 +- .../dfo-data-tools/src/code_index/labels.py | 18 +- .../src/code_index/multilaterals.py | 78 + .../src/code_index/scope_conventions.py | 248 ++- .../dfo-data-tools/src/code_index/types.py | 14 +- .../dfo-data-tools/src/dc_sdmx/__init__.py | 2 + packages/dfo-data-tools/src/dc_sdmx/client.py | 21 +- .../src/entity_resolution/pack.py | 15 +- .../src/graph_release/declarations.py | 8 + .../src/graph_release/service.py | 26 +- .../dfo-data-tools/src/materialise/bundle.py | 55 +- .../src/materialise/bundle_docs.py | 63 +- .../src/materialise/bundle_sources.py | 120 +- .../src/materialise/composite.py | 17 +- .../src/materialise/dependencies.py | 32 +- .../dfo-data-tools/src/materialise/errors.py | 30 + .../dfo-data-tools/src/materialise/facets.py | 18 +- .../dfo-data-tools/src/materialise/finish.py | 14 +- .../src/materialise/generation.py | 6 +- .../src/materialise/observation_rows.py | 8 +- pyproject.toml | 7 +- src/datasets_api/adapter.py | 110 +- src/datasets_api/admission.py | 6 +- src/datasets_api/current.py | 15 +- src/datasets_api/derived.py | 15 +- src/datasets_api/doors/definition.py | 6 +- src/datasets_api/doors/mint_inventory.py | 3 + src/datasets_api/kinds/spec_id.py | 7 +- src/datasets_api/origin.py | 5 +- src/datasets_api/render.py | 71 +- src/datasets_api/serve.py | 25 +- src/datasets_api/warning_backfill.py | 13 + src/presentation_store/frame_store.py | 20 +- src/search/README.md | 20 +- src/search/catalogue/counts.py | 3 +- src/search/engine/app.py | 1 + .../engine/clients/materialise_sdmx_cache.py | 20 +- src/search/engine/clients/sdmx.py | 5 +- src/search/engine/codes.py | 3 + src/search/engine/concurrency.py | 14 +- src/search/engine/departures.py | 47 +- src/search/engine/entry/dataset_entries.py | 53 +- src/search/engine/entry/dataset_resubmit.py | 28 +- src/search/engine/identity_edit.py | 208 ++- src/search/engine/multilaterals.py | 44 +- .../engine/options/options_response_cache.py | 21 +- src/search/engine/options/resolve.py | 42 +- src/search/engine/options/surface.py | 20 +- .../engine/region/candidate_collapse.py | 23 +- src/search/engine/region/codes.py | 5 + src/search/engine/region/composite_answer.py | 708 ++++++++- src/search/engine/region/grounding.py | 89 +- src/search/engine/region/materialise_step.py | 7 +- .../region/observations/bulk_pair_reads.py | 8 +- src/search/engine/region/phase_answer.py | 14 +- .../engine/region/phase_place_materialise.py | 23 +- src/search/engine/region/phase_shared.py | 12 +- src/search/engine/region/pipeline.py | 126 +- src/search/engine/region/place_options.py | 13 +- src/search/engine/region/promote_finalize.py | 4 +- src/search/engine/region/promote_ground.py | 4 +- .../engine/region/promote_materialise.py | 2 + src/search/engine/region/resubmit.py | 10 +- src/search/engine/region/result.py | 101 +- src/search/engine/region/scope_decision.py | 126 +- src/search/engine/region/state.py | 15 + src/search/engine/region/whole_totals.py | 60 +- src/search/engine/response_cache_keys.py | 15 +- src/search/engine/retrieval/bound_values.py | 12 +- src/search/engine/retrieval/retrieve.py | 5 +- src/search/engine/retrieval/role_resolve.py | 9 +- src/search/engine/retrieval/role_scope.py | 13 +- src/search/engine/same_shape.py | 8 +- src/search/heading/departure_fact.py | 8 + src/search/heading/pins.py | 16 +- src/search/models/departure.py | 3 + src/search/models/envelope.py | 15 +- src/search/models/request.py | 18 + src/search/models/response.py | 3 +- src/search/models/scope.py | 12 +- src/search/models/spec.py | 8 +- src/search/models/vocab.py | 2 + src/search/render/departures.py | 29 + src/search/scope/composite.py | 282 +++- src/search/scope/edits.py | 82 +- src/search/scope/provider.py | 608 +++++-- tests/assembler/test_export_component.py | 66 + tests/code_index/test_composites.py | 28 +- .../code_index/test_destination_population.py | 319 ++++ tests/code_index/test_destination_side.py | 124 ++ tests/code_index/test_family_label.py | 27 + tests/datasets_api/test_composite_download.py | 2 +- tests/datasets_api/test_derived.py | 58 +- .../test_family_sum_matches_summable.py | 10 +- .../test_mint_dac2a_destinations.py | 439 ++++++ .../test_mint_error_classification.py | 14 + .../test_render_bundle_documents.py | 22 +- tests/datasets_api/test_sync_estimate.py | 20 +- tests/datasets_api/test_warning_backfill.py | 27 +- tests/engine/test_composite_answer.py | 437 +++++- .../test_dac2a_destination_populations.py | 1391 +++++++++++++++++ tests/engine/test_dac2a_population_reads.py | 415 +++++ .../test_grounding_identity_entities.py | 16 +- tests/engine/test_multilateral_donors.py | 2 +- tests/engine/test_promote_shape_sample.py | 4 +- tests/engine/test_request_warning_sites.py | 282 ++++ tests/engine/test_role_class_shapes.py | 5 +- tests/engine/test_whole_total_bind_e2e.py | 146 +- .../first_answer_place_scenarios.json | 166 +- tests/graph_release/test_service.py | 30 + tests/materialise/test_composite.py | 7 +- tests/materialise/test_dependencies.py | 75 +- tests/materialise/test_facets.py | 15 +- tests/presentation_store/test_frame_store.py | 26 + tests/render/test_render_departures.py | 44 + tests/scope/test_composite_preview.py | 2 +- tests/scope/test_provider.py | 875 ++++++++++- uv.lock | 30 +- 123 files changed, 9067 insertions(+), 805 deletions(-) create mode 100644 packages/dfo-data-tools/src/code_index/multilaterals.py create mode 100644 tests/code_index/test_destination_population.py create mode 100644 tests/code_index/test_destination_side.py create mode 100644 tests/datasets_api/test_mint_dac2a_destinations.py create mode 100644 tests/engine/test_dac2a_destination_populations.py create mode 100644 tests/engine/test_dac2a_population_reads.py create mode 100644 tests/engine/test_request_warning_sites.py diff --git a/docs/conformance-search-api-v1.md b/docs/conformance-search-api-v1.md index 2a0ad712d..ae9aa1ea2 100644 --- a/docs/conformance-search-api-v1.md +++ b/docs/conformance-search-api-v1.md @@ -157,6 +157,12 @@ Gap rows state what is needed to close the clause and name the open issue that t | C66a | slots[].expansion publishes the descendant codes the resolver admitted beyond a slot's own bound value(s), when the bound value's code system is declared hierarchical; null for every other slot, including one bound on a flat code system or a property the deployment's code index does not carry. Additive only -- never fed into spec identity, which hashes binding alone | implemented | | | C66b | slots[].summable is true when the slot's property is a code system the deployment's code index declares summable (the declaration the Datasets API's groupby collapses over and capabilities summable_axes publishes), whatever the slot's binding; false for every other slot. Additive only; spec identity does not hash it | implemented | | | C66c | resolved.stat_vars.sums_components is true when the definition binds a declared composite whose components sum (`code_index.composites.components_sum`, the rule the Datasets API's groupby adds a composite's components by, `datasets_api.groupby.sums_components`): a groupby whose `by` keeps no `stat_var` adds the members of its different components into one series per cell, and a cell holding one component's member serves that member. True for the stacked composites; false, and omitted from the wire, for the spliced one and for every answer binding no composite. It says nothing about slots; slots[].summable still governs a sum over each slot. Additive only; spec identity does not hash it | implemented | | +| C66d | A composite component that binds its destinations to one side of a donor's aid (`ComponentRecipe.destinations`, set on `provenance/ONEDAC2aBilateral` and on both components of `ONEDAC2aBilateralAndImputedMultilateral`) reads a role open within a declared population (`code_index.scope_conventions.RolePopulation`). `code_index.composites.destination_population` builds it once per code index from the parties the index publishes as destinations of the component's dataset, each placed by `code_index.scope_conventions.destination_side`: `members` holds the bilateral-side parties and residuals and no total, as an open read holds none; `named` holds every bilateral-side destination, totals included; `total` is the dataset's published total of that side, `DACDevelopingCountries` (`BilateralSource.rebound`). The population is bound where a component's identity derives (`code_index.composites._derived_identity`), so every reader of rows, every cache key and the mint read it from the identity without a rule of their own. An open destination binds `members`, and its `entities` key is listed in `ComponentIdentity.populations`. A named destination passes when it is in `named` (a ref the dataset does not publish as a destination passes and reads no rows) and otherwise raises `CompositeNotServable` at `/entities/destination`, with `refs` naming the destinations at fault. An identity that neither names nor opens a destination binds `[total]` and lists no population, so the answer states the DAC's total of developing countries as it states any bound destination. `table_identity` returns a `TableIdentity` (`identity`, `populations`) built from the composite's first component, so Search elects within the population the mint reads. That read holds the first component's rows alone, so a reader counting what the composite serves reads every component. Only a reader of the names a caller asked for reads `populations`. `composite_digest` takes the code index its caller reads and folds each bound population's dataset, side, members and total into the digest, so a stored frame re-mints when the population changes, and leaves out a `destinations` field that is `None`, so a composite declared without it keeps its digest | implemented | Pinned by `tests/code_index/test_destination_population.py` (`test_the_population_holds_bilateral_parties_and_residuals_and_names_its_totals`, `test_an_open_destination_binds_the_population_and_tags_its_key`, `test_a_named_destination_on_the_bilateral_side_passes_untagged`, `test_a_named_destination_off_the_bilateral_side_is_refused`, `test_an_absent_destination_binds_the_population_s_total_untagged`, `test_the_table_identity_carries_the_first_component_s_population`, `test_the_table_identity_of_an_identity_binding_no_composite_is_itself`, `test_the_composite_digest_moves_with_the_population`, `test_every_declared_component_binding_destinations_has_a_declared_source`), `tests/code_index/test_destination_side.py` (`test_a_dac2a_destination_is_placed_on_its_side`). | +| C66e | The Datasets mint reads a population through a typed filter. `datasets_api.adapter.to_materialisation_request` sends each role in `ComponentIdentity.populations` as `dc_sdmx.Among(members, side)`, which the SDMX client sends as the role's OR-list, as it sends a list of dcids. Each reader of an entity filter chooses between rows and names. Readers of rows use `members`: `dc_sdmx.client._entity_params`, the coverage and facet reads in `materialise.facets`, `datasets_api.admission.bound_roles`, and the cache keys `facets.entity_filter_key`, `materialise_sdmx_cache` and `bulk_pair_reads._bound_key`. Readers of the names a caller gave (`materialise.finish._plain_role_bindings`) skip it, as they skip `ContainedIn`, so the manifest states no population member as unserved (`coverage_below_scope`). A frame minted for an open destination holds no agency, `DACAllMultilateralRecipients` or `DACAllRecipients` row, the bilateral composite's sum is the bilateral-side rows alone, and the inclusive composite adds the imputed line (aid type 106) to it. The row ceiling, the admission estimate and the sync limits charge the population's rows as they charge a named list, and the bound read sends the whole population in one SDMX request within the client's URL budget. A scope's entities and window apply to a composite mint: its components derive from the scope-applied identity, for the CRS inclusive composite and both DAC2a composites, whether the definition is posted beside the mint or registered before it. A definition or scope that binds a population composite to a destination off its side is refused 422 `facet_not_available` at the mint and options doors, before any read and before the registry write, one issue per destination (`datasets_api.adapter._off_side_refusal`): `DACDevelopingCountries` on the destination role for `DACAllRecipients`, and the plain DAC2a table on the provenance axis for an agency or the DAC's total of them. The refusal's `facet_pointer` is `/entities/destination` when every such destination is `DACAllRecipients` and `/facets/provenance` otherwise | implemented | Pinned by `tests/datasets_api/test_mint_dac2a_destinations.py` (`test_an_open_destination_mints_no_agency_or_spanning_row`, `test_an_open_destination_sums_the_country_row_alone_and_inclusive_adds_106`, `test_the_bound_recipient_list_fits_one_request_under_the_url_budget`, `test_an_open_destination_states_no_population_member_as_unserved`, `test_the_mint_door_refuses_a_composite_bound_to_a_destination_off_its_side`, `test_the_definition_door_refuses_a_composite_bound_to_a_destination_off_its_side`, `test_the_off_side_refusal_points_where_its_fix_is`, `test_a_scope_binding_the_destination_reads_that_destination_alone`, `test_a_scope_binding_an_agency_refuses_before_any_read`, `test_a_scope_narrowing_the_window_reads_that_year_alone`), `tests/datasets_api/test_sync_estimate.py` (`test_a_population_bound_role_prices_only_its_members_rows`), `tests/materialise/test_facets.py` (`test_probe_facets_role_bearing_with_no_bound_role_covers_every_coverage_row`). | +| C66f | Search discovers an open role within its population, and a fresh answer reads the population as a resubmit of its definition does. At the resolver's entry a role in `ResubmitIdentityPayload.populations` reads as open `within` its members (`identity_edit.open_within`), and `retrieval.role_scope.discover_role_scope(within=...)` adds the members to the one bound every coverage read on that path takes, so `resolved.scope`, `role_scope_defaulted`, coverage, headline, plan and preview count the population alone. The resubmit door and the options door read the posted identity through `identity_edit.composite_table_identity`, which returns `TableRead(identity, populations, sides)`, and each component preview reads its component's populations (`composite_answer._resolve_component`). The part memo and the options cache key the bound list, so a population read and a caller naming the same parties do not collide. The options door adds the side's `named` destinations to its bound, so no agency, `DACAllMultilateralRecipients` or `DACAllRecipients` is offered on a destination axis a population composite reads, and its origin options list the donors holding rows at the population. A whole-total offer names `RolePopulation.total` in place of the side's whole totals (`whole_totals.whole_total_warnings`), so the comparable figure is the DAC's total of developing countries. `scope.composite._headline` and `_stat_vars` read each served component's own bound. A fresh answer that settles on a composite binding a population is read again within it before the pass judges the answer (`pipeline._within_population`, `composite_answer.population_table_answer`) and takes that read's scope, headline, coverage, plan and warnings, with the request's own words, the warnings raised before the read (`ResolveState.request_warnings`) and their departures (`composite_answer.with_population_read`). The fresh answer and a resubmit of its definition then agree on `resolved.scope`, headline, coverage and warnings. A composite whose identity names no destination is read at the population's total (`RolePopulation.total`, `DACDevelopingCountries`). The read supplies the answer's figures, members and warnings alone: the answer keeps the identity, spec_id and scope decided once on the request's own identity, so it names no destination and keeps its provider options. A composite that refuses the read answers `no_data` `facet_not_available` and logs the refusal at ERROR. A population holding no row in the window answers `no_data` under the request's words, and a candidate set drops a reading whose population holds none; a set left with one reading answers it definite, as a resubmit of that reading does. A candidate set's readings are read again concurrently, every part's reads together at most the request's concurrency limit (`concurrency.bounded_thread_permits`). An inclusive composite's scope, role axis facts and `role_scope_defaulted` count the recipients of both lines together, as its headline and the mint's frame do (`scope.composite._open_axes`, `result.recounted_scope`); where the coverage snapshot cannot list a component's parties it keeps its table read's counts and logs a WARNING naming the component. A same-shape merge of composites and each candidate restated as a composite read their table within the population. A posted identity that binds either DAC2a composite to an agency or to `DACAllRecipients` is refused 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `multilateral_destination` with the destinations in the cause's `entities`, at the identity's `/entities/destination`, with the plain DAC2a table as the alternative; no Search door settles it to another identity | implemented | Pinned by `tests/engine/test_dac2a_population_reads.py` (`test_a_resubmit_discovers_its_open_destination_within_the_population`, `test_a_resubmit_offers_the_population_total_in_place_of_the_spanning_one`, `test_a_composite_posted_with_a_destination_off_its_side_refuses`, `test_a_population_read_and_a_named_list_of_its_members_key_apart`, `test_each_component_preview_reads_its_destination_within_the_population`, `test_the_destination_options_offer_no_agency_or_spanning_total`, `test_the_origin_options_list_the_donors_holding_rows_at_the_population`, `test_a_composite_s_options_key_apart_from_its_table_identity_s`, `test_a_population_filter_keys_search_s_sdmx_reads_by_its_members`, `test_the_candidate_door_reads_each_composite_s_table_identity_once`), `tests/engine/test_dac2a_destination_populations.py` (`test_a_resubmit_counts_no_agency_or_spanning_series_for_an_open_destination`, `test_a_composite_preview_holds_the_country_row_and_inclusive_adds_106`, `test_a_fresh_answer_equals_a_resubmit_of_its_definition`, `test_a_fresh_composite_answer_reads_every_field_a_resubmit_reads`, `test_a_fresh_bilateral_reading_lists_no_agency_in_its_scope`, `test_a_merged_composite_reads_its_table_within_the_population`, `test_a_candidate_restated_as_the_bilateral_composite_lists_no_agency_in_its_scope`, `test_a_composite_whose_population_holds_no_row_answers_no_data_with_the_request_s_words`, `test_a_composite_answer_keeps_the_warnings_raised_before_its_read`, `test_a_candidate_whose_population_holds_no_row_is_dropped_not_restated`, `test_a_composite_answer_keeps_a_denominator_its_words_name_unverified`, `test_an_inclusive_answer_counts_the_recipients_of_both_lines`, `test_an_inclusive_answer_whose_component_parties_cannot_be_listed_says_so`, `test_a_composite_naming_no_destination_is_read_again_at_its_population_total`, `test_a_fresh_answer_naming_no_destination_states_the_scope_of_its_own_identity`, `test_a_population_read_the_composite_refuses_answers_no_data`, `test_a_candidate_set_left_with_one_reading_answers_as_its_resubmit`, `test_population_reads_take_the_request_s_limit_and_report_their_lost_reads`), `tests/engine/test_composite_answer.py` (`test_a_population_read_keeps_the_request_s_warnings_and_replaces_the_read_s`), `tests/engine/test_whole_total_bind_e2e.py` (`test_a_recipients_reading_under_total_oda_states_the_total_beside_its_recipients`, `test_ranked_recipients_under_total_oda_state_the_total_beside_them`). | +| C66g | DAC2a's plain aid type 206 table is total ODA, the contributions scope (`provider:contributions`, rule `contribution_recipient_total`), and counts the donor's core contributions at every multilateral destination. Its bilateral option binds `provenance/ONEDAC2aBilateral` (rule `bilateral_destinations`), aid type 206 over the bilateral population, and its inclusive option binds `ONEDAC2aBilateralAndImputedMultilateral`, total ODA plus DAC2a's imputed multilateral line (aid type 106), both lines over the population, so a core contribution is counted once as a flow or once as an imputed share and neither line holds a row at `DACAllRecipients`. Posted over a provider that holds the multilaterals' outflows, or over every donor, where the imputed line would count them twice, the inclusive composite serves the bilateral line. A fresh DAC2a total ODA reading settles on the bilateral composite (`provider.bilateral_default_reading`, `provider.with_bilateral_default`), with or without a provider axis, whatever donors it names, where it names no destination or only bilateral-side ones, so a ranking over all donors lists no agency. It keeps total ODA where the part's first provider entry, stated or carried, is `provider:contributions`, where the donor's total ODA holds no breakdown by recipient over the years asked, and where the population is unavailable (C66i). A fresh part with no provider axis has no option to state why, so wherever the default reaches it and its bilateral option is not available, it carries the info warning `bilateral_reading_unavailable`, stamped with its spec_id, with `params` `{why}`: the option's `why`, `destination_unclassified` for an unavailable population or the slot the composite refuses where it cannot serve what the part binds, else its reason code, `option_not_covered` where it does not cover every year the answer holds, and `scope_undecided` where the part's scope decision raised and the answer states no scope (`scope_decision.guarded_scope_decision`), and a message saying the figures include core contributions to multilateral agencies (`provider.bilateral_default_withheld`, `scope_decision.bilateral_withheld_warning`). Each reading a candidate set or refusal shows that keeps total ODA so carries it too (`composite_answer.with_bilateral_readings`). A part with a provider axis, or whose provider entry asks for total ODA, carries none. Where the part has no provider axis, no table can apply a `provider:contributions` or `provider:bilateral` entry, so the default serves it and the part does not report it in `dropped_entries`. A part holding a composite over bilateral-side destinations alone is not offered the total itself (`provider_not_offered`, `why` `destination_bilateral`), since those destinations hold no core contribution. The bilateral option is labelled by what it keeps and leaves out, the total by `BilateralSource.total_label`; the composite's measure noun reads "(bilateral flows)", its citation "OECD DAC2a", and its label counts the recipients its file holds. A `latest` window compiles to the one year the composite serves (`provider.elect_latest_window`). "Bilateral plus multilateral" on DAC2a reads the inclusive composite. A stored bare 206 identity reads back as total ODA under its own spec_id and offers the bilateral composite and the inclusive scope as switches | implemented | Pinned by `tests/scope/test_provider.py` (`test_a_fresh_dac2a_total_reads_its_bilateral_destinations_unless_it_names_an_agency`, `test_a_fresh_dac2a_part_switched_to_gross_from_its_base_keeps_its_bilateral_reading`, `test_a_fresh_dac2a_part_at_a_country_offers_no_switch_to_total_oda`, `test_bilateral_plus_multilateral_on_dac2a_reads_the_inclusive_composite`, `test_a_resubmitted_bare_dac2a_total_reads_as_total_oda`, `test_a_donor_with_no_breakdown_by_recipient_keeps_total_oda`, `test_the_latest_bilateral_reading_elects_the_year_its_composite_reads`, `test_a_dac2a_bilateral_answer_counts_the_recipients_its_file_holds`, `test_each_table_names_its_provider_options_by_what_they_count`, `test_a_dac2a_composite_over_bilateral_destinations_offers_no_total_oda`, `test_an_inclusive_composite_over_providers_it_counts_twice_serves_the_bilateral_line`), `tests/engine/test_dac2a_destination_populations.py` (`test_a_fresh_dac2a_answer_reads_the_composite_its_destinations_allow`, `test_a_ranking_over_every_donor_says_when_it_cannot_read_bilateral_destinations`, `test_a_part_with_a_provider_axis_states_an_unavailable_population_on_its_scope`, `test_a_candidate_over_every_donor_says_when_it_cannot_read_bilateral_destinations`), `tests/engine/test_composite_answer.py` (`test_a_candidate_set_restating_a_dac2a_reading_keeps_its_other_readings_options`). | +| C66h | A DAC2a part naming a multilateral-side destination reads total ODA: a named agency, `DACAllMultilateralRecipients` or one of the DAC's unnamed multilateral recipients. Its bilateral and inclusive options are `unavailable` with the scope reason `multilateral_destination`, the destination in `detail`, and a stated bilateral scope settles on total ODA. Any DAC2a part naming such a destination, with or without a provider axis, carries the `core_contribution_read` departure (`departures._core_contribution_facts`), which names the destination and is worded for an agency, for the DAC's total over agencies or for its line of agencies it does not name. A part bound to `DACAllRecipients`, a total that spans both sides, holds total ODA. Under the bilateral or inclusive scope its destination rebinds to `DACDevelopingCountries` (`scope.edits.rebound_destinations`), the bilateral side of all recipients, and the option labels say so (`OPTION_LABELS.provider_rebound`); a switch back reads all recipients again. A posted identity binding a composite to either destination is refused rather than settled (C66f) | implemented | Pinned by `tests/scope/test_provider.py` (`test_a_dac2a_total_to_an_agency_offers_no_bilateral_reading_and_says_why`, `test_a_dac2a_answer_to_an_agency_says_its_figures_are_core_contributions`, `test_every_donor_s_dac2a_aid_to_an_agency_says_its_figures_are_core_contributions`, `test_a_dac2a_total_to_all_recipients_reads_developing_countries_when_cut`), `tests/render/test_render_departures.py` (`test_departure_text`), `tests/engine/test_dac2a_population_reads.py` (`test_a_composite_posted_with_a_destination_off_its_side_refuses`). | +| C66i | A population is available only while every destination it could leave out is placed on a side. `destination_population` raises `PopulationUnavailable` (`found_in` `code_index`), a `CompositeNotServable` at `/facets/provenance` naming the composite and carrying the destinations in `unclassified`, so every handler of `CompositeNotServable` catches it, when the code index publishes a destination `destination_side` cannot place: a party that is neither a country nor an organisation the entity resolver knows, a total other than `DACAllRecipients`, `DACAllMultilateralRecipients` and the declared bilateral totals (`DACDevelopingCountries`, the DAC regions of `REGION_NAMES`, `DAC_BILATERAL_GROUP_TOTALS`), and a residual other than `DAC_MULTILATERAL_RECIPIENTS` and the declared bilateral residuals (the unallocated lines of `TOTAL_GROUPS`, `DAC_BILATERAL_UNALLOCATED`). `code_index.composites.population_check(index, coverage)` also reads the coverage snapshot: every recipient coverage holds for the population's variables must be a destination the code index publishes, else the population is unavailable with `found_in` `coverage`, since a list of members would leave that recipient out without a word. The check is memoised per code index, then per coverage index, each entry dropped with its own index. The release service runs the check as a release opens (`graph_release.service.build_release_views`), logs `destination_population_unavailable` for each failing population, and still serves the release. A head's freshness check (`materialise.dependencies.refreshed_dependencies`) gives a composite whose population is unavailable no digest and logs `composite_digest_unavailable`, so the head rebuilds and the mint answers its refusal. While the check fails, the mint refuses a composite reading the population before any read with `DestinationUnclassifiedError`, a server fault (500 `internal_error`) whose detail names the destinations (`datasets_api.serve._materialise_components`, `datasets_api.adapter.composite_components`), and Search reports the bilateral and inclusive options `provider_not_offered` with `why` `destination_unclassified` and keeps total ODA on a fresh part (`provider._population_unavailable`), which warns `bilateral_reading_unavailable` where it has no provider axis (C66g). A posted identity binding the composite then refuses 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `provider_not_offered` naming the unplaced destinations, with the plain DAC2a table as the alternative, and a reader of the composite's table, its pins or its family query refuses it alike | implemented | Pinned by `tests/code_index/test_destination_side.py` (`test_an_unknown_party_that_is_no_country_is_unclassified`, `test_a_total_or_residual_no_declaration_places_is_unclassified`, `test_a_party_no_dataset_publishes_as_a_recipient_is_unclassified`, `test_every_dac2a_destination_of_the_recorded_index_is_classified`, `test_the_organisation_records_read_from_a_zipped_package_without_running_its_init`), `tests/code_index/test_destination_population.py` (`test_a_destination_the_code_index_cannot_place_refuses_the_composite_reading_it`, `test_coverage_whose_recipients_the_population_places_leaves_it_available`, `test_a_covered_recipient_the_code_index_does_not_publish_makes_the_population_unavailable`, `test_the_population_check_memo_holds_one_index_finalizer_however_many_coverages`), `tests/materialise/test_dependencies.py` (`test_a_composite_whose_population_is_unavailable_rebuilds_without_raising`), `tests/graph_release/test_service.py` (`test_opening_a_release_checks_its_destination_populations_against_its_coverage`), `tests/datasets_api/test_mint_dac2a_destinations.py` (`test_a_composite_whose_population_coverage_cannot_place_refuses_before_any_read`, `test_a_composite_whose_code_index_cannot_place_a_destination_is_a_server_fault`), `tests/datasets_api/test_mint_error_classification.py` (`test_an_unclassified_destination_is_a_server_fault_whose_detail_names_it`), `tests/scope/test_provider.py` (`test_a_dac2a_total_naming_a_destination_on_no_side_keeps_total_oda`, `test_a_dac2a_total_naming_a_destination_on_no_side_offers_neither_cut_reading`), `tests/engine/test_dac2a_population_reads.py` (`test_a_composite_whose_coverage_holds_a_destination_on_no_side_refuses_at_resubmit`, `test_a_composite_whose_code_index_places_no_side_refuses_at_resubmit`, `test_a_composite_whose_coverage_holds_a_destination_on_no_side_refuses_its_options`, `test_a_reader_of_a_composite_s_table_refuses_an_unavailable_population`). | | C67 | A singleton set normalises to the value form; empty sets are invalid everywhere | implemented | | | C68 | A slot is identified by its property alone; the sentence axis it renders under is declared by the shape, not stored per-slot | partial | Derive a slot's sentence axis from the shape's grammar instead of storing it on `SlotKey`. Issue: none | | C69 | entities is a role-keyed map, sorted/deduplicated, OR within a role and AND across roles; always present, never an empty per-role list; a role in `open_roles` is not a key of `entities` | implemented | | @@ -215,7 +221,7 @@ Gap rows state what is needed to close the clause and name the open issue that t | C143 | resolved.open_axis_facts: one entry per constraint axis this Spec's own served StatVar members declare (an arc naming the property, resolved or not) that the query left unbound, each naming the axis property and what those members exhibit on it: varies=false when they agree on a declared value (or all decline to declare one), varies=true when they differ, value for the single declared value they agree on (present only when varies is false and at least one member declares one), its label resolved from the packaged code index when the dcid names a declared code system's row and the bare dcid otherwise (neither path is a graph lookup this clause requires), distinct_count for how many distinct declared values the served members actually exhibit (0 when every member declares nothing), counted over the served members' own declarations independent of this response's observation window (a query's applied when-slot narrows which observation points are returned, not which constraint values a member declares), and undeclared_count for how many declare none at all (a present-but-unresolved arc counts as undeclared too). None when the served members declare no constraint property the query left open at all, distinct from an open axis every member declines to declare, which still publishes its own entry. Never a member of BaseIdentity, never hashed. Carried forward across a same-shape merge (re-derived over the merged member set) and on a dataset_resubmit identity entry's re-resolve response | implemented | | C144 | resolved.measure_label_axes: the declared measure-axis identifiers (code_index.types.MeasureAxis.axis, a shape-field name such as measuredProperty or measurementQualifier, or a constraint-property dcid such as a financial-flow-type or netting-basis axis) that resolved.measure_label was composed from: every axis the served partition carries a value on, whether that value ended up folded into measure_label's own text, appended as its trailing parenthetical qualifier, or, for the price basis (measurementQualifier), left to the byline's unit token, which states it on every answer ("current prices" for Nominal, or the requested basis under a unit request). [] when measure_label is None. Never a member of BaseIdentity, never hashed. | implemented | | | C145 | For a shape with a measure family, resolved.shape.label is composed from resolved.measure_label plus every other axis the served group agrees on, never from one served StatVar member's own catalog name: a bound slot's value on a property outside measure_label_axes, and an open_axis_facts entry with varies=false and a value, each fold in as a trailing parenthetical qualifier in measure_label's own style; an open_axis_facts entry with varies=true is never named, since the served group does not agree on one value for it. A qualifier whose value text already appears is stated once. When measure_label is None because the served partition carries no authored label, the same qualifiers compose the label without a head phrase, comma-joined, falling back to a confirmed-served StatVar's own name only when the served group shares no bound slot value and no constant open_axis_facts entry either. When this Spec's shape carries no measure family at all, a reading that serves exactly one StatVar with a graph name takes that StatVar's name, and every other reading's label passes through exactly as built upstream. | implemented | | -| C201 | resolved.scope_choices states how the part a Spec answers was counted, as a PartScope (`search.models.scope`): `base`, the identity the part resolved; `rule_version`, the hash of the declared scope conventions (`code_index.scope_conventions`), the live axes and their conflicts; `entries`, the explicit and carried entries applied, and `dropped_entries`, each carried entry the part cannot take with code `carried_choice_not_applied` and its cause, and on a fresh search each entry the request's words stated that the part cannot serve with code `stated_choice_not_applied` and the reason as its cause, the other axes' defaults derived without it, since a parsed or raw_text search never refuses its scope; `choices`, one ScopeChoice per axis a declared rule settled (sector, region, measure_basis, provider), each with its id `:`, the rule, its provenance (`explicit`, `carried` or `default`), the active option id and every option with a label naming its subject ("health, CRS 120 alone", "Africa's countries only"), its binding and availability (`available`, `partial` with its years, or `unavailable` with a `ScopeReasonCode`); `compiled`, the spec_id, identity and components the active options compile to; and `ledger`, which the server leaves empty. Option ids are `:` on every part. `search.scope.compile` derives each default after the explicit and carried entries, never moves an explicit or carried entry, and compiles a selection the base already holds to the base and its spec_id. A dataset_resubmit identity entry posting `scope_entries` beside the scope's `base.identity` as `identity` resolves the identity they compile to and carries its scope. A switch posts the scope's `entries` with the entry on the switched axis replaced by the switch, an explicit entry whose origin names the choice, so each axis holds one entry; an entry posted with no concept on an axis asked for a concept takes that concept, echoed in the response's `entries`. It refuses `requested_option_unavailable` (422) for an explicit entry naming an axis the part does not settle, an option the axis does not offer or an unavailable option, and `conflicting_requested_options` (422) for two entries that cannot hold together, two entries on one axis and a switch blocked by an entry on another axis included, each pointed at `/input/entries/{i}/scope_entries/{j}`. Absent when no axis was settled and no scope entries were posted. The sector axis is live (`search.scope.sector`): a sector concept the request names binds its declared grouping (`code_index.scope_conventions.SECTOR_GROUPINGS`; "health" binds CRS 120 and 130, slot provenance `declared_grouping`) and states a `default` choice offering the grouping and each of its codes alone, unless an explicit or carried entry made for the concept chose one of them; a code the request names ("basic health", 122) keeps its binding and states an `explicit` choice whose entry quotes the words; an option whose codes no covered member of the part's family declares is `unavailable` with `option_not_covered`. Never a member of BaseIdentity, never hashed | implemented | | +| C201 | resolved.scope_choices states how the part a Spec answers was counted, as a PartScope (`search.models.scope`): `base`, the identity the part resolved; `rule_version`, the hash of the declared scope conventions (`code_index.scope_conventions`), the live axes and their conflicts; `entries`, the explicit and carried entries applied, and `dropped_entries`, each carried entry the part cannot take with code `carried_choice_not_applied` and its cause, and on a fresh search each entry the request's words stated that the part cannot serve with code `stated_choice_not_applied` and the reason as its cause, the other axes' defaults derived without it, since a parsed or raw_text search never refuses its scope; `choices`, one ScopeChoice per axis a declared rule settled (sector, region, measure_basis, provider), each with its id `:`, the rule, its provenance (`explicit`, `carried` or `default`), the active option id and every option with a label naming its subject ("health, CRS 120 alone", "Africa's countries only"), its binding and availability (`available`, `partial` with its years, or `unavailable` with a `ScopeReasonCode`); `compiled`, the spec_id, identity and components the active options compile to, each component's identity the one the mint reads, so a role bound to a population carries its members as a list (C66d); and `ledger`, which the server leaves empty. Option ids are `:` on every part. `search.scope.compile` derives each default after the explicit and carried entries, never moves an explicit or carried entry, and compiles a selection the base already holds to the base and its spec_id. A dataset_resubmit identity entry posting `scope_entries` beside the scope's `base.identity` as `identity` resolves the identity they compile to and carries its scope. A switch posts the scope's `entries` with the entry on the switched axis replaced by the switch, an explicit entry whose origin names the choice, so each axis holds one entry; an entry posted with no concept on an axis asked for a concept takes that concept, echoed in the response's `entries`. It refuses `requested_option_unavailable` (422) for an explicit entry naming an axis the part does not settle, an option the axis does not offer or an unavailable option, and `conflicting_requested_options` (422) for two entries that cannot hold together, two entries on one axis and a switch blocked by an entry on another axis included, each pointed at `/input/entries/{i}/scope_entries/{j}`. Empty `choices` when no axis was settled but the part is served from a composite, which `compiled.identity` and `compiled.components` name (a DAC2a ranking read to its bilateral destinations); absent when no axis was settled, no composite serves the part and no scope entries were posted. The sector axis is live (`search.scope.sector`): a sector concept the request names binds its declared grouping (`code_index.scope_conventions.SECTOR_GROUPINGS`; "health" binds CRS 120 and 130, slot provenance `declared_grouping`) and states a `default` choice offering the grouping and each of its codes alone, unless an explicit or carried entry made for the concept chose one of them; a code the request names ("basic health", 122) keeps its binding and states an `explicit` choice whose entry quotes the words; an option whose codes no covered member of the part's family declares is `unavailable` with `option_not_covered`. Never a member of BaseIdentity, never hashed | implemented | Pinned by `tests/engine/test_composite_answer.py` (`test_a_merge_of_composite_parts_with_no_axis_states_the_composite_scope`), `tests/scope/test_provider.py` (`test_a_fresh_dac2a_total_reads_its_bilateral_destinations_unless_it_names_an_agency`). | | C153 | resolved.measure_axis_values: one {axis, value} entry per axis in resolved.measure_label_axes, in the same order, naming the value the served partition holds on that axis as a GraphRef labelled from the packaged measure index, including a value the query never named (a declared default such as Gross or Nominal). The measurementQualifier entry names the served partition's native price basis, under a unit request too; measure_label never states it. [] when measure_label is None. A partition value the measure index does not declare raises rather than publishing its dcid as a label. Never a member of BaseIdentity, never hashed. | implemented | Pinned by `tests/engine/test_shape_label_measure_agreement.py` and `tests/engine/test_measure_alternatives.py`. | | C154 | Each resolved.alternative_measures entry carries measure_axis_values, the same projection as C153 over that entry's own partition, on both the sibling-partition path and a folded measure-sibling candidate. | implemented | Pinned by `tests/engine/test_measure_alternatives.py` and `tests/engine/test_shape_label_measure_agreement.py`. | | C155 | Each resolved.open_axis_facts entry carries values: every declared value as {value, count}, value labelled as for C143's value, count the number of distinct served StatVars declaring it, ordered by count descending then dcid. The counts plus undeclared_count equal the distinct served StatVars, and len(values) equals distinct_count. A same-shape merge counts a StatVar that several merged answers serve once, and counts a merged answer's served StatVars as undeclared on a property none of them asserts. Never a member of BaseIdentity, never hashed. | implemented | Pinned by `tests/engine/test_open_axis_facts.py` and `tests/engine/test_conjoin_open_axis_merge.py`. | @@ -261,7 +267,7 @@ Gap rows state what is needed to close the clause and name the open issue that t | C107 | Named codes: date_coverage_partial, binding_folded, observations_omitted | partial | `date_coverage_partial` is minted when a unit conversion's tables cover only part of the served years (C178), and when a span counted back from now answers from data whose newest year lies more than `SEARCH_TRAILING_STALE_YEARS` before the current one (C178a). `binding_folded` and `observations_omitted` are minted and reachable. The registry keeps `default_not_declared` active on search.resolve, and nothing emits it, because unit completion reads declared sources or refuses (C100). Issue: none | | C107a | Origin catalogue warnings on search.resolve: `un_system_unavailable` on a part the UN pass could change, one whose own pass ends no_data for a reason other than its entities or answers doubted (C51e), when the UN resolve or its shape derivation failed upstream, the two were not in within SEARCH_UN_FIRST_PASS_WAIT_S of the start of derivation, or the UN pass does not complete, and with SEARCH_UN_ON_FIRST_MENU on any part whose weak first pass waited for the UN leg and found it failed or not in, and on any part when the UN front door refuses places a read named, such as `country/GRC`, `country/SVN` or `country/ESH` (the catalogue's `un.json` lists them) or its part of a read that also names our variables fails while our part serves, including in a trailing observations or slot-options read, whose warning carries its spec_id in the done event, and whose `params` hold the refused place dcids as `places`, the failed reads' variables as `variables` and each variable's name as `variable_labels`, parallel to `variables`, taken from the members of the Specs the part holds with no graph read, and "" where it holds none (a member the graph names no label for, and every variable on search.options), so a client tells a refusal from a failed read and names the variables without the message, and in search.options diagnostics; `withheld_by_registry` on the request's first part that ends no_data with reason `no_observations` and gate `no_member_has_data_at_place` after electing a subject-only shape, naming the provenances the catalogue withholds that hold data for its elected StatVars at the places the part names (one unfiltered facet read per request at most); no other no-data reason triggers it; `origin_catalogue_stale` in diagnostics when the loaded catalogue is more than 14 days old, added after the response cache so a cached answer reports the catalogue's age when served | implemented | Search has no door that pins a variable dcid (a posted `stat_var` shape refuses), so the pinned-variable trigger applies to Datasets only. Pinned by `tests/engine/test_origin_wire.py` (`test_trailing_observations_report_places_the_un_refused`, `test_refused_places_alone_name_no_variable`) and, for a failed, slow or unreadable UN pool and the UN pass's failures, `tests/engine/test_pool_election.py` and `tests/engine/test_recall_transfer.py` (`test_a_un_resolve_that_never_answers_leaves_a_transferred_answer_intact`). | | C107b | Artifact warnings on search.resolve, one per response at most each: `artifact_pair_stale` in diagnostics when the bound origin catalogue and coverage builds record different `ingestion.workflow_execution_id` values (a build recording none never warns), naming both builds and their ingestions, added after the response cache, whose key holds both builds; `catalogue_index_drift` in diagnostics when, for a StatVar a definite or candidate Spec serves that the packaged code index lists and the catalogue's arc tables hold, the two disagree on the five-tuple, the constraint properties and their values, or the roles (`observationProperties`), naming each such StatVar with the aspects, added before the response cache stores the answer, whose key holds the catalogue build and the build commit that packages the code index. Each also logs one WARNING line, `artifact_pair_stale` or `catalogue_index_drift` followed by a JSON object. Neither fails the request or readiness. `ORIGIN_CATALOGUE_WARN_PAIR_STALE=false` and `ORIGIN_CATALOGUE_WARN_INDEX_DRIFT=false` silence the warning and keep the log line | implemented | Pinned by `tests/engine/test_artifact_checks.py`. | -| C149 | A warning carries `params` only for `observations_series_truncated` (C150), `elected_shape_no_data` (C151), `un_system_unavailable` (C107a), `whole_total_available`, `currency_unmapped_omitted`, `deflator_missing_omitted` and `deflator_projected`, holding the counts and labels its message states so a client composes its own text without parsing the message; for `currency_unmapped_omitted` and `deflator_missing_omitted` the first ten entities the message counts, with their labels (`""` where the conversion tables hold none, as for most entities no currency names) and, for `deflator_missing_omitted`, the years left out; for `deflator_projected` `{count, examples}`, the rows that took a projected rate or deflator, and at most ten distinct `entity@year` keys among them, sorted, where `count` counts rows, not keys, and `examples` may leave keys out; every other code's warning omits the key | implemented | Pinned by the C150 and C151 tests. | +| C149 | A warning carries `params` only for `observations_series_truncated` (C150), `elected_shape_no_data` (C151), `un_system_unavailable` (C107a), `whole_total_available`, `bilateral_reading_unavailable` (C66g), `currency_unmapped_omitted`, `deflator_missing_omitted` and `deflator_projected`, holding the counts and labels its message states so a client composes its own text without parsing the message; for `currency_unmapped_omitted` and `deflator_missing_omitted` the first ten entities the message counts, with their labels (`""` where the conversion tables hold none, as for most entities no currency names) and, for `deflator_missing_omitted`, the years left out; for `deflator_projected` `{count, examples}`, the rows that took a projected rate or deflator, and at most ten distinct `entity@year` keys among them, sorted, where `count` counts rows, not keys, and `examples` may leave keys out; every other code's warning omits the key | implemented | Pinned by the C150 and C151 tests. | | C150 | `observations_series_truncated` carries `params: {shown, total, order}`: the series the preview ships and the confirmed series behind them, both integers, `shown` < `total`, and `order`, `"ranked"` when the preview read every series and ships the most recent and largest (latest year, then latest value, then identity) or `"fixed"` when the observations the answer keeps in each member's years exceed `role_bulk_rank_max_rows` or the read converts into each entity's own currency and the preview ships a selection made before any value was read, StatVars newest year first and taken round-robin. The message states the same order. A fixed selection takes only series with data in their member's years, StatVars ordered by their newest year with data there. The observations themselves carry the same `order` whenever they show fewer series than the answer holds, and omit it otherwise. The Datasets observations door selects a snapshot share's series by the same rule over its frame's rows and states its own `order` on the observations it serves | implemented | Pinned by `tests/engine/test_bulk_role_series_observations.py`, `tests/engine/test_population_headline.py` and, for the door against the live read on each window, `tests/datasets_api/test_observations_door.py` (`test_the_door_and_the_live_role_bulk_preview_agree_on_each_series_axes`). | | C151 | A part carries at most one `elected_shape_no_data` per election, however many empty readings it dropped, with `params: {no_data, elected}`: the shape labels of the readings with no data and of the readings answered instead, each listed once in first-seen order. A single-sibling re-election names the elected shape and its sibling; a candidates answer of several siblings names the elected shape and each candidate's shape label. A same-shape merge folds its parts' warnings that share a message into one, whose `no_data` and `elected` are the first-seen unions over the parts | implemented | Pinned by `tests/engine/test_zero_observation_election.py`, `tests/engine/test_candidates_measure_sibling_collapse.py`, and `tests/engine/test_conjoin_warnings_and_primary.py`. | | C194 | A price-basis requirement the query names on `measurementQualifier` ("constant", "real", "at constant prices") is met by an answer served under a unit request whose price basis reads as one of the requirement's codes (`requirements._serves_price_basis`): constant prices meet `RealValue` and `InflationAdjusted`, current prices meet `Nominal`, and no unit request meets `PurchasingPowerParity`. A definite answer, or a candidate set every member of which is served so, carries no `query_qualifier_dropped` warning, `measure_not_applied` departure or sentence fact for it. The requirement still elects a publisher's series at that basis where one exists. An answer served under no unit request, or at the other basis, leaves it unmet, so "constant prices" over a series holding only current prices still warns | implemented | Pinned by `tests/engine/test_pinned_code_refusal.py`. | diff --git a/packages/dfo-analysis-engine/pyproject.toml b/packages/dfo-analysis-engine/pyproject.toml index e9155bd73..c4ac74dc3 100644 --- a/packages/dfo-analysis-engine/pyproject.toml +++ b/packages/dfo-analysis-engine/pyproject.toml @@ -20,8 +20,8 @@ dependencies = [ # See the root pyproject.toml's dfo-spec-kit source comment for the # pin-consistency invariant these two revs must satisfy: every spec-kit source in the # workspace moves together as one rev. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/spec-kit" } -dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/frame-grammar" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/spec-kit" } +dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/frame-grammar" } [build-system] requires = ["uv_build>=0.11,<0.12"] diff --git a/packages/dfo-data-tools/pyproject.toml b/packages/dfo-data-tools/pyproject.toml index 351f2982c..91e68a43c 100644 --- a/packages/dfo-data-tools/pyproject.toml +++ b/packages/dfo-data-tools/pyproject.toml @@ -94,11 +94,11 @@ graph-release = ["google-cloud-spanner>=3.57", "dfo-data-tools[coverage]"] dfo-analysis-engine = { workspace = true } # See the root pyproject.toml's dfo-spec-kit source comment for the # pin-consistency invariant this rev must satisfy. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/spec-kit" } -dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/frame-grammar" } -dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/dfo-spec-kit-components" } -dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/provenance-docs" } -dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/descriptions" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/spec-kit" } +dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/frame-grammar" } +dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/dfo-spec-kit-components" } +dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/provenance-docs" } +dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/descriptions" } [build-system] requires = ["uv_build>=0.11,<0.12"] diff --git a/packages/dfo-data-tools/src/assembler/pipeline/export_component.py b/packages/dfo-data-tools/src/assembler/pipeline/export_component.py index 8587ed783..85f993f5e 100644 --- a/packages/dfo-data-tools/src/assembler/pipeline/export_component.py +++ b/packages/dfo-data-tools/src/assembler/pipeline/export_component.py @@ -47,10 +47,13 @@ `description` and README row say which file is which, with "The card shows N of M; holds all M." on the shown file. `ComponentExportMismatch` guards the shown file alone, and a bare file holds the shown rows. Each provenance ref the kept rows carry is cited from the frame manifest's -citation, its licence and documentation included, through `materialise.bundle_sources`. Columns are -documented by `materialise.bundle_columns`, and the codes table is labelled from -`ResolvedFrame.labels` and the cited source names with no graph read, so a role's codes keep the -kind `entity`. Links and the reproducing call sit under the public site origin the caller passes. +citation, its licence and documentation included, through `materialise.bundle_sources`. A +declared composite is cited by its component tables, as its citation names them, and named with +what it adds or reads in the README, SOURCES.md and the datapackage description +(`bundle_sources.composite_notes`). Columns are documented by `materialise.bundle_columns`, and +the codes table is labelled from `ResolvedFrame.labels`, the declared composites' bundle names +and the cited source names with no graph read, so a role's codes keep the kind `entity`. Links +and the reproducing call sit under the public site origin the caller passes. The revision's stored `created_at` is the bundle's prepared date and dates the file name, so one revision always serves one name. An export on a target frame takes both from that frame's @@ -89,6 +92,8 @@ from assembler.pipeline.render import render_on_frame from assembler.pipeline.stored_revision import read_stored_revision from assembler.ports import FrameRef, FrameStore, ReleaseRegistry, ResolvedFrame, RevisionStore +from code_index.composites import provenance_datasets +from code_index.shared import get_index from dfo_analysis_engine.place_type import ( PARTY_ROLES, PLACE_ROLE, @@ -117,7 +122,11 @@ revision_tag, zip_file_name, ) -from materialise.bundle_sources import source_provenance +from materialise.bundle_sources import ( + composite_bundle_labels, + composite_notes, + source_provenance, +) from materialise.formats import ( Rendering, Sink, @@ -735,7 +744,7 @@ def _export_table( generated[label_column] = column out_columns.append(label_column) - labels = resolved_frame.labels + labels = _frame_labels(resolved_frame) grain = period_grain(row[index[TIME_PERIOD_COLUMN]] for _, row in rows) out_rows: list[tuple[object, ...]] = [] for series_label, row in rows: @@ -819,6 +828,7 @@ def _bundled_member( sources=_sources(resolved_frame=resolved_frame, provenance_refs=refs), columns=_column_docs(scan, generated=generated, partly_shown=partly_shown), description=description, + composites=composite_notes(refs, index=get_index()), ) return provenance, scan @@ -826,25 +836,39 @@ def _bundled_member( def _sources( *, resolved_frame: ResolvedFrame, provenance_refs: Sequence[str | None] ) -> tuple[SourceProvenance, ...]: - """One source per provenance ref, cited from the frame manifest's citation. Rows without - a provenance cite one unattributed source.""" + """One source per provenance ref, cited from the frame manifest's citation. A declared + composite's rows are cited by its component tables (`code_index.composites + .provenance_datasets`), as its citation names them, each once. Rows without a provenance + cite one unattributed source.""" cited_by_ref = _citation_sources_by_ref(resolved_frame.manifest) + cited = dict.fromkeys( + dataset + for ref in provenance_refs + for dataset in ((None,) if ref is None else provenance_datasets(ref)) + ) return tuple( source_provenance(ref, cited_by_ref.get(ref)) if ref is not None else SourceProvenance(ref=None, name=_UNATTRIBUTED_SOURCE_NAME) - for ref in provenance_refs + for ref in cited ) +def _frame_labels(resolved_frame: ResolvedFrame) -> dict[str, str]: + """`ResolvedFrame.labels`, with each declared composite named by its bundle name + (`materialise.bundle_sources.composite_bundle_labels`), so its `provenance` cells and + codes-table row read apart from its component tables.""" + return {**resolved_frame.labels, **composite_bundle_labels(get_index())} + + def _known_labels( resolved_frame: ResolvedFrame, *, sources: Sequence[SourceProvenance] ) -> dict[str, str]: - """The codes table's labels: the frame's resolved labels, with each cited source's name - in place of its provenance ref's graph name. A source the citation lacks is named by its - ref, so it keeps the frame's label.""" + """The codes table's labels: the frame's resolved labels (`_frame_labels`), with each + cited source's name in place of its provenance ref's graph name. A source the citation + lacks is named by its ref, so it keeps the frame's label.""" cited_by_ref = _citation_sources_by_ref(resolved_frame.manifest) - labels = dict(resolved_frame.labels) + labels = _frame_labels(resolved_frame) labels.update( (source.ref, source.name) for source in sources diff --git a/packages/dfo-data-tools/src/code_index/composites.py b/packages/dfo-data-tools/src/code_index/composites.py index 5f3e39330..3ce3ebfea 100644 --- a/packages/dfo-data-tools/src/code_index/composites.py +++ b/packages/dfo-data-tools/src/code_index/composites.py @@ -13,13 +13,22 @@ import dataclasses import hashlib import json +import weakref from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import TYPE_CHECKING, cast +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Literal, cast from code_index.declarations import COMPOSITES from code_index.measure_basis import edit_basis, with_slot -from code_index.scope_conventions import BASIS_MEASURES, ODA_FLOW, ODA_FLOW_SYSTEM +from code_index.scope_conventions import ( + BASIS_MEASURES, + BILATERAL_SOURCES, + ODA_FLOW, + ODA_FLOW_SYSTEM, + RolePopulation, + destination_side, +) +from code_index.shared import per_index from code_index.types import ( ComponentRecipe, ComponentRole, @@ -27,13 +36,16 @@ StatVarRow, YearPeriod, ) +from dfo_platform.roles import role_key_axis if TYPE_CHECKING: from code_index.query import CodeIndex - from dfo_platform.roles import FlowAccount + from dfo_platform.roles import FlowAccount, RoleAxis + from series_coverage.index import CoverageIndex _PROVENANCE = "provenance" _PINNED_SHAPE = "stat_var" +_DESTINATION: RoleAxis = "destination" DECLARED_COMPOSITES: Mapping[str, CompositeDeclaration] = { composite.provenance: composite for composite in COMPOSITES @@ -57,38 +69,133 @@ def _jsonable(value: object) -> object: raise TypeError(f"cannot digest {type(value).__name__}") -def composite_digest(composite: CompositeDeclaration) -> str: - """`sha256:` and the hex digest of what a run composes `composite` from: its declaration - (each component's recipe with its periods, binds, dropped slots, zero absence, measure and - basis), and, when a component sets a basis, the basis measures and ODA flow codes - `measure_basis.edit_basis` reads.""" - payload: dict[str, object] = {"composite": dataclasses.asdict(composite)} +def composite_digest(composite: CompositeDeclaration, *, index: CodeIndex) -> str: + """`sha256:` and the hex digest of what a run composes `composite` from, over sorted-key + JSON of: + + - its declaration, every field of it and of each component's recipe + (`ComponentRecipe`), with `destinations` left out of a recipe where it is None, so a + composite whose components bind no destinations digests as its other fields alone; + - where any component sets a basis, the basis measures and ODA flow codes + `measure_basis.edit_basis` reads; + - where any component binds its destinations, for each distinct (dataset, side) bound, + the dataset, the side, and the members and total of the population it reads in + `index` (`destination_population`), the code index the caller reads the composite + under, so a release that moves a party across sides moves the digest. + + Raises `PopulationUnavailable` as `destination_population` does.""" + declaration = dataclasses.asdict(composite) + for component in cast("list[dict[str, object]]", declaration["components"]): + if component["destinations"] is None: + del component["destinations"] + payload: dict[str, object] = {"composite": declaration} if any(component.basis is not None for component in composite.components): payload["basis_measures"] = [dataclasses.asdict(row) for row in BASIS_MEASURES] payload["oda_flow"] = [ODA_FLOW_SYSTEM, ODA_FLOW] + bound = sorted( + { + (recipe.dataset, recipe.destinations) + for recipe in composite.components + if recipe.destinations is not None + } + ) + if bound: + populations = [ + destination_population(index, dataset, composite=composite.provenance) + for dataset, _side in bound + ] + payload["destination_population"] = [ + [dataset, population.side, list(population.members), population.total] + for (dataset, _side), population in zip(bound, populations, strict=True) + ] encoded = json.dumps(payload, sort_keys=True, default=_jsonable).encode("utf-8") return f"sha256:{hashlib.sha256(encoded).hexdigest()}" -def declared_composite_digest(provenance: str) -> str | None: - """`composite_digest` of the composite this deployment declares for `provenance` - (`DECLARED_COMPOSITES`), or `None` where it declares none.""" +def declared_composite_digest(provenance: str, *, index: CodeIndex) -> str | None: + """`composite_digest` under `index` of the composite this deployment declares for + `provenance` (`DECLARED_COMPOSITES`), or `None` where it declares none. Raises + `PopulationUnavailable` as `composite_digest` does.""" composite = DECLARED_COMPOSITES.get(provenance) - return composite_digest(composite) if composite is not None else None + return composite_digest(composite, index=index) if composite is not None else None class CompositeNotServable(ValueError): """A composite identity whose components cannot be derived, because it binds a slot a component's rows do not declare, pins a StatVar, names a composite in a provenance set, or - names one the index serves no component table of. `ref` is the composite's provenance and `pointer` - the identity member at fault, relative to the identity.""" - - def __init__(self, message: str, *, ref: str, pointer: str) -> None: + names one the index serves no component table of, or binds a component reading one side's + destinations to destinations off that side. `ref` is the composite's provenance and + `pointer` the identity member at fault, relative to the identity. `refs` holds the refs at + fault in that member, in the identity's order, where the refusal names some: the + destinations off the side for the last case, and empty otherwise. `key` is the `entities` + key of the destination role those destinations bind, for the last case alone, and None + otherwise.""" + + def __init__( + self, + message: str, + *, + ref: str, + pointer: str, + refs: tuple[str, ...] = (), + key: str | None = None, + ) -> None: self.ref = ref self.pointer = pointer + self.refs = refs + self.key = key super().__init__(message) +UnclassifiedIn = Literal["code_index", "coverage"] +"""Where a destination no side declaration places was found: among the parties the code index +publishes in the dataset, or among the recipients coverage holds for the population's +variables (`population_check`).""" + + +@dataclass(frozen=True) +class UnclassifiedDestinations: + """Destinations of `dataset`, a dataset a composite component binds the destinations of + (`ComponentRecipe.destinations`), which `destination_side` cannot place: the declarations + have drifted from the graph. `found_in` says where they were found. While any is + unclassified the population is unavailable, since a list of members would leave the + drifted destination out without a word. + + `unlisted` names the population's variables whose recipients coverage cannot list: it + holds rows of the variable, but no facet of it carries `dataset` as its provenance. A + destination there could sit off every side unseen, so the population is unavailable too; + `destinations` is then empty and `found_in` is `coverage`.""" + + dataset: str + destinations: tuple[str, ...] + found_in: UnclassifiedIn + unlisted: tuple[str, ...] = () + + def message(self) -> str: + if self.unlisted: + return ( + f"coverage lists no recipients of {self.dataset} for the population variables " + f"{list(self.unlisted)!r}" + ) + where = "publishes" if self.found_in == "code_index" else "holds coverage of" + return ( + f"{self.dataset} {where} destinations no side declaration places: " + f"{list(self.destinations)!r}" + ) + + +class PopulationUnavailable(CompositeNotServable): + """A composite, provenance `ref`, whose component reads a population of destinations + (`destination_population`) that is unavailable, for the reason `unclassified` gives: the + side declarations have drifted from the graph, a server fault. `pointer` is the + composite's provenance facet, since no edit of the identity serves it. A handler of + `CompositeNotServable` that reads every refusal as the caller's fault catches this first.""" + + def __init__(self, unclassified: UnclassifiedDestinations, *, ref: str) -> None: + self.unclassified = unclassified + super().__init__(unclassified.message(), ref=ref, pointer=f"/facets/{_PROVENANCE}") + + class CompositeProvenanceRead(ValueError): """A provenance pin read as a dataset when it binds a composite. The caller reads the composite's components (`component_identities`) instead.""" @@ -97,11 +204,28 @@ class CompositeProvenanceRead(ValueError): @dataclass(frozen=True) class ComponentIdentity: """One component of a composite identity: the role it plays, the recipe it was derived - by, and its own identity document, which reads the component's table alone.""" + by, and its own identity document, which reads the component's table alone. + + `populations` maps each `entities` key of `identity` whose binding is a population + (`RolePopulation.members`) in place of the destinations the caller named, the key an + open role took. It is empty for a component that binds no destinations, and for one + whose identity named its destinations or left them out. A reader of rows reads + `identity` alone; a reader of the names the caller asked for reads the key as open within + its population.""" role: ComponentRole recipe: ComponentRecipe identity: Mapping[str, object] + populations: Mapping[str, RolePopulation] = field(default_factory=dict) + + +@dataclass(frozen=True) +class TableIdentity: + """The identity a reader of one table's family reads (`table_identity`), and the + populations its `entities` bind in place of names (`ComponentIdentity.populations`).""" + + identity: Mapping[str, object] + populations: Mapping[str, RolePopulation] = field(default_factory=dict) def composite_ref(identity: Mapping[str, object]) -> str | None: @@ -127,6 +251,16 @@ def composite_ref(identity: Mapping[str, object]) -> str | None: return None +def binds_composite(identity: Mapping[str, object]) -> bool: + """Whether `identity`'s provenance facet binds a declared composite, or names one in a + provenance set, which `composite_ref` refuses and every reader treats as a composite: + deriving its components refuses it there.""" + try: + return composite_ref(identity) is not None + except CompositeNotServable: + return True + + def provenance_datasets(ref: str) -> tuple[str, ...]: """The datasets a provenance `ref` serves from: a declared composite's component tables, each once in declaration order, else `ref` itself.""" @@ -171,10 +305,18 @@ def component_identities( recipe's periods. A spliced component whose periods the window does not reach is left out. Slots stay sorted by property. + A component that binds its destinations (`ComponentRecipe.destinations`) reads its + dataset's population (`destination_population`): an open destination binds the + population's members and is listed in `ComponentIdentity.populations`, a named one passes + when the dataset publishes no ref of it off the population's side, and an identity that + leaves the destination out binds the population's total. + Raises `CompositeNotServable` when `identity` pins a StatVar, binds a slot a component drops (`ComponentRecipe.drop_unbound`), names a composite in a provenance set, names one - whose component tables `index` does not serve (`CodeIndex.composite`), or reaches none of - its components' periods.""" + whose component tables `index` does not serve (`CodeIndex.composite`), names a + destination off a component's population (at `/entities/`), or reaches none of its + components' periods, and `PopulationUnavailable`, a `CompositeNotServable`, as + `destination_population` does.""" ref = composite_ref(identity) if ref is None: return None @@ -198,12 +340,13 @@ def component_identities( clipped = _clipped_window(window, recipe.periods) if clipped is None: continue - derived = _derived_identity(identity, recipe, ref=ref) + derived, populations = _derived_identity(index, identity, recipe, ref=ref) components.append( ComponentIdentity( role=recipe.role, recipe=recipe, identity={**derived, "requested_window": clipped}, + populations=populations, ) ) if not components: @@ -215,19 +358,38 @@ def component_identities( return tuple(components) -def table_identity(index: CodeIndex, identity: Mapping[str, object]) -> Mapping[str, object]: +def table_identity(index: CodeIndex, identity: Mapping[str, object]) -> TableIdentity: """The identity a reader of one table's family reads for `identity`. For a composite that is the identity its composite option was set on (`without_composite`), once its components - derive (`component_identities`), and otherwise `identity` itself. The result is the - bilateral component of a provider-inclusive composite, and the net identity over the whole - window of the OECD headline, whose shape the part elected. Search reads a composite's family, pins, dataset and - provenance scope here, so its readers elect over the same rows the composite's option - edits. - - Raises `CompositeNotServable` as `component_identities` does.""" - if component_identities(index, identity) is None: - return identity - return without_composite(identity) + derive (`component_identities`), and otherwise `identity` itself, with no populations. The + result is the bilateral component of a provider-inclusive composite, and the net identity + over the whole window of the OECD headline, whose shape the part elected. Search reads a + composite's family, pins, dataset and provenance scope here, so its readers elect over the + same rows the composite's option edits. + + A composite whose first component binds its destinations (`ComponentRecipe.destinations`) + binds them here as that component does, so Search elects within the population the mint + reads, and `TableIdentity.populations` names the keys bound to a population. The read + holds the first component's rows alone: a party only another component holds, such as a + recipient of the imputed line with no total ODA row, is not in it, so a reader counting + what the composite serves reads every component (`component_identities`). + + The first component is the first `component_identities` derives, whose `entities` and + `open_roles` the read takes as they were bound there. + + Raises `CompositeNotServable` and `PopulationUnavailable` as `component_identities` + does.""" + components = component_identities(index, identity) + if components is None: + return TableIdentity(identity=identity) + read = without_composite(identity) + first = components[0] + if first.recipe.destinations is None: + return TableIdentity(identity=read) + roles = ("entities", "open_roles") + bound = {key: value for key, value in read.items() if key not in roles} + bound.update({key: first.identity[key] for key in roles if key in first.identity}) + return TableIdentity(identity=bound, populations=first.populations) def component_of(composite: CompositeDeclaration, row: StatVarRow) -> ComponentRole | None: @@ -268,10 +430,12 @@ def without_composite(identity: Mapping[str, object]) -> Mapping[str, object]: def _derived_identity( - identity: Mapping[str, object], recipe: ComponentRecipe, *, ref: str -) -> dict[str, object]: - """`identity` derived by `recipe`, its window unchanged.""" + index: CodeIndex, identity: Mapping[str, object], recipe: ComponentRecipe, *, ref: str +) -> tuple[dict[str, object], Mapping[str, RolePopulation]]: + """`identity` derived by `recipe`, its window unchanged, and the `entities` keys it binds + to a population (`_bind_destinations`).""" derived = dict(without_composite(identity)) + populations = _bind_destinations(index, derived, recipe=recipe, ref=ref) if recipe.provenance is not None: facets = dict(cast("Mapping[str, object]", derived.get("facets", {}))) facets[_PROVENANCE] = {"state": "value", "ref": recipe.provenance} @@ -298,7 +462,297 @@ def _derived_identity( for prop, code in sorted(recipe.bind.items()): slots = with_slot(slots, {"property": prop, "binding": {"state": "value", "ref": code}}) derived["slots"] = sorted(slots, key=lambda slot: cast(str, slot["property"])) - return derived + return derived, populations + + +def _bound_datasets() -> frozenset[str]: + """The datasets a declared composite component binds the destinations of.""" + return frozenset( + recipe.dataset + for composite in COMPOSITES + for recipe in composite.components + if recipe.destinations is not None + ) + + +@dataclass(frozen=True) +class _Classified: + """One dataset's destinations as `index` publishes them: its population, or the + destinations no side declaration places, with no population.""" + + population: RolePopulation | None + unclassified: tuple[str, ...] + + +@per_index +def _classified(index: CodeIndex) -> Mapping[str, _Classified]: + """`_Classified` for each dataset a declared component binds the destinations of.""" + sources = {source.dataset: source for source in BILATERAL_SOURCES} + classified: dict[str, _Classified] = {} + for dataset in sorted(_bound_datasets()): + source = sources.get(dataset) + if source is None: + raise LookupError( + f"a composite component binds the destinations of {dataset!r}, which no " + "BILATERAL_SOURCES row declares" + ) + members: list[str] = [] + named: set[str] = set() + unclassified: list[str] = [] + for dcid, row in index.parties.items(): + if not row.publishes(_DESTINATION, (dataset,)): + continue + side = destination_side(row, datasets=(dataset,)) + if side is None: + unclassified.append(dcid) + elif side == "bilateral": + named.add(dcid) + if row.kind_over(_DESTINATION, (dataset,)) != "total": + members.append(dcid) + classified[dataset] = ( + _Classified(population=None, unclassified=tuple(sorted(unclassified))) + if unclassified + else _Classified( + population=RolePopulation( + side="bilateral", + members=tuple(sorted(members)), + named=frozenset(named), + source=source, + ), + unclassified=(), + ) + ) + return classified + + +def destination_population( + index: CodeIndex, dataset: str, *, composite: str | None = None +) -> RolePopulation: + """The population of destinations a component reading `dataset` binds its destinations + to (`ComponentRecipe.destinations`), from the parties `index` publishes there as + destinations, built once per index. + + Raises `PopulationUnavailable` (`found_in="code_index"`) when `destination_side` places + none of a destination `dataset` publishes, whatever its kind. It names `composite` as the + composite refused, or by default the first declared composite whose component binds the + destinations of `dataset`. Raises `KeyError` for a dataset no declared component binds + the destinations of.""" + classified = _classified(index)[dataset] + if classified.population is None: + raise PopulationUnavailable( + UnclassifiedDestinations( + dataset=dataset, destinations=classified.unclassified, found_in="code_index" + ), + ref=composite if composite is not None else _first_binding(dataset), + ) + return classified.population + + +def _first_binding(dataset: str) -> str: + """The provenance of the first declared composite whose component binds the destinations + of `dataset`.""" + return next( + composite.provenance + for composite in COMPOSITES + if any( + recipe.dataset == dataset and recipe.destinations is not None + for recipe in composite.components + ) + ) + + +def _bind_destinations( + index: CodeIndex, derived: dict[str, object], *, recipe: ComponentRecipe, ref: str +) -> Mapping[str, RolePopulation]: + """Bind `derived`'s destination role, in place, as `recipe` reads it, and return the + `entities` key bound to a population, empty where none is. A recipe that binds no + destinations leaves `derived` as it is. + + An open destination binds the population's members (`destination_population`). A named + one keeps its refs when the dataset publishes none of them as a destination off the + population (`RolePopulation.named`), and refuses with `CompositeNotServable` otherwise; a + ref the dataset does not publish as a destination passes, and reads no rows. An identity + that neither names nor opens the destination binds the population's total + (`RolePopulation.total`), since the dataset's total over every destination spans both + sides.""" + if recipe.destinations is None: + return {} + population = destination_population(index, recipe.dataset, composite=ref) + entities = dict(cast("Mapping[str, Sequence[str]]", derived.get("entities", {}))) + open_roles = list(cast("Sequence[str]", derived.get("open_roles", ()))) + bound_key = next((key for key in entities if role_key_axis(key) == _DESTINATION), None) + open_key = next((key for key in open_roles if role_key_axis(key) == _DESTINATION), None) + if bound_key is not None: + refused = [ + dcid + for dcid in entities[bound_key] + if dcid not in population.named + and (row := index.parties.get(dcid)) is not None + and row.publishes(_DESTINATION, (recipe.dataset,)) + ] + if refused: + raise CompositeNotServable( + f"composite {ref!r} reads {population.side} destinations alone, and " + f"{refused!r} {'is' if len(refused) == 1 else 'are'} not on that side", + ref=ref, + pointer=f"/entities/{bound_key}", + refs=tuple(refused), + key=bound_key, + ) + return {} + populations: dict[str, RolePopulation] = {} + if open_key is not None: + open_roles.remove(open_key) + entities[open_key] = list(population.members) + populations[open_key] = population + else: + entities[_DESTINATION] = [population.total] + derived["entities"] = dict(sorted(entities.items())) + if open_roles: + derived["open_roles"] = open_roles + else: + derived.pop("open_roles", None) + return populations + + +@dataclass(frozen=True) +class PopulationCheck: + """Whether the populations of one code index (`destination_population`) hold every + destination one coverage index serves for their variables. `unclassified` maps each + dataset whose population is unavailable to why: the destinations `destination_side` + cannot place, found in the code index or, where it places every party it publishes, in + coverage, or the variables whose recipients coverage cannot list (`population_check`). + An empty map means every population is available.""" + + unclassified: Mapping[str, UnclassifiedDestinations] + + def refusal(self, composite: str) -> UnclassifiedDestinations | None: + """Why the composite with provenance `composite` cannot be read, for the first of its + components whose population is unavailable in declaration order, or `None` where + every population it reads is available or it binds none.""" + declaration = DECLARED_COMPOSITES.get(composite) + if declaration is None: + return None + return next( + ( + self.unclassified[recipe.dataset] + for recipe in declaration.components + if recipe.destinations is not None and recipe.dataset in self.unclassified + ), + None, + ) + + +@per_index +def _checks(index: CodeIndex) -> dict[int, PopulationCheck]: + """The population checks of `index` built so far, by the `id` of the coverage index each + was built against (`population_check`).""" + return {} + + +def population_check(index: CodeIndex, coverage: CoverageIndex) -> PopulationCheck: + """Check every population `index` declares (`destination_population`) against the + recipients `coverage` holds for the population's variables: the rows of the dataset that + hold every code a component binding the population binds (`ComponentRecipe.bind`). A + recipient the code index does not publish there as a destination is one no side + declaration places, and the population's members would leave it out without a word, so + the population is unavailable until the code index places it. A population the code + index itself cannot place (`destination_population`) is unavailable too, and so is one + with a variable whose recipients coverage cannot list (`UnclassifiedDestinations + .unlisted`): the check fails closed. + + Built once per pair of indexes: memoised per code index (`per_index`), then per coverage + index within it by `id`. Each coverage index carries one finalizer per code index it was + checked against, which drops its entry when the coverage index is collected; the code + index's own memo goes with the code index. The release + service builds it when a release opens (`graph_release.service.build_release_views`), so + a request reads it from the memo. The mint and Search refuse a composite reading an + unavailable population (`PopulationCheck.refusal`).""" + checks = _checks(index) + key = id(coverage) + found = checks.get(key) + if found is None: + found = _population_check(index, coverage) + checks[key] = found + weakref.finalize(coverage, checks.pop, key, None) + return found + + +def _population_check(index: CodeIndex, coverage: CoverageIndex) -> PopulationCheck: + unclassified: dict[str, UnclassifiedDestinations] = {} + for dataset, classified in _classified(index).items(): + if classified.population is None: + unclassified[dataset] = UnclassifiedDestinations( + dataset=dataset, destinations=classified.unclassified, found_in="code_index" + ) + continue + held: set[str] = set() + unlisted: list[str] = [] + for variable in _population_variables(index, dataset): + recipients = _covered_recipients(coverage, variable, dataset) + if recipients is None: + unlisted.append(variable) + else: + held |= recipients + if unlisted: + unclassified[dataset] = UnclassifiedDestinations( + dataset=dataset, destinations=(), found_in="coverage", unlisted=tuple(unlisted) + ) + continue + drifted = sorted(held - _published_destinations(index, dataset)) + if drifted: + unclassified[dataset] = UnclassifiedDestinations( + dataset=dataset, destinations=tuple(drifted), found_in="coverage" + ) + return PopulationCheck(unclassified=unclassified) + + +def _covered_recipients( + coverage: CoverageIndex, variable: str, dataset: str +) -> frozenset[str] | None: + """The recipients `coverage` holds for `variable` in the facets whose provenance is + `dataset`, or None when it holds rows of `variable` but no facet of it names `dataset`: + rows with no provenance, or another one, whose recipients coverage cannot attribute. + + A variable coverage holds no row of contributes no recipients. The check reads the + snapshot, so it is only as fresh as the snapshot: where the source publishes rows of a + variable the snapshot does not yet hold, a destination there goes unchecked until the + next coverage build.""" + summary = coverage.facet_summary.variable(variable) + if summary is None: + return frozenset() + facets = [facet for facet in summary.facets if facet.metadata.provenance_id == dataset] + if not facets: + return None + return frozenset().union(*(facet.parties.get("recipient", frozenset()) for facet in facets)) + + +def _published_destinations(index: CodeIndex, dataset: str) -> frozenset[str]: + """The parties `index` publishes as destinations of `dataset`, each of which a population + `index` builds places on a side.""" + return frozenset( + dcid for dcid, row in index.parties.items() if row.publishes(_DESTINATION, (dataset,)) + ) + + +def _population_variables(index: CodeIndex, dataset: str) -> tuple[str, ...]: + """The StatVars of `dataset` a component binding its destinations reads: those holding + every code such a component binds (`ComponentRecipe.bind`).""" + binds = [ + recipe.bind + for composite in COMPOSITES + for recipe in composite.components + if recipe.destinations is not None and recipe.dataset == dataset + ] + return tuple( + row.dcid + for row in index.statvars + if dataset in row.provenances + and any( + all(row.constraint_properties.get(prop) == code for prop, code in bind.items()) + for bind in binds + ) + ) def _clipped_window( diff --git a/packages/dfo-data-tools/src/code_index/declarations.py b/packages/dfo-data-tools/src/code_index/declarations.py index 58a2af574..8aded3520 100644 --- a/packages/dfo-data-tools/src/code_index/declarations.py +++ b/packages/dfo-data-tools/src/code_index/declarations.py @@ -110,6 +110,9 @@ class CodeSystemConfig: CRS_INCLUSIVE = "provenance/ONEBilateralAndImputedMultilateral" DAC2A_INCLUSIVE = "provenance/ONEDAC2aBilateralAndImputedMultilateral" +# DAC2a's total ODA read to its bilateral recipients alone (`COMPOSITES`). +DAC2A_BILATERAL = "provenance/ONEDAC2aBilateral" + # The DAC's published total of the multilateral organisations' own outflows, a provider of DAC1, # DAC2a and DAC2b. DAC_MULTILATERALS = "DACMultilaterals" @@ -508,7 +511,10 @@ def _aid_notes(note: str) -> tuple[NotedWord, ...]: kind="stacked", components=( ComponentRecipe( - role="bilateral", dataset=DAC2A, bind={"dac2aAidType": DAC2A_TOTAL_ODA} + role="bilateral", + dataset=DAC2A, + bind={"dac2aAidType": DAC2A_TOTAL_ODA}, + destinations="bilateral", ), ComponentRecipe( role="imputed", @@ -516,6 +522,21 @@ def _aid_notes(note: str) -> tuple[NotedWord, ...]: provenance=DAC2A, basis=("attributedAmount", "Gross"), bind={"dac2aAidType": DAC2A_IMPUTED_MULTILATERAL_ODA}, + destinations="bilateral", + ), + ), + ), + CompositeDeclaration( + provenance=DAC2A_BILATERAL, + label="OECD DAC2a", + measure_noun="(bilateral flows)", + kind="stacked", + components=( + ComponentRecipe( + role="bilateral", + dataset=DAC2A, + bind={"dac2aAidType": DAC2A_TOTAL_ODA}, + destinations="bilateral", ), ), ), diff --git a/packages/dfo-data-tools/src/code_index/labels.py b/packages/dfo-data-tools/src/code_index/labels.py index 8d4fa345d..12c5e6d33 100644 --- a/packages/dfo-data-tools/src/code_index/labels.py +++ b/packages/dfo-data-tools/src/code_index/labels.py @@ -214,9 +214,13 @@ def family_label( with no provenance pin (`code_index.composites.without_composite`), and is named by its declaration (`CompositeDeclaration.label`): after the measure label for a stacked composite, whose components share the measure, and in its place for a spliced one, whose - measure changes between its periods (the OECD headline). Returns `None` when neither part - has a label. The presentation door labels a collapsed family's `dcid:` ref with - this, and the datasets door's `stat_vars_summed` warning carries the same string as `stat_var_label`. + measure changes between its periods (the OECD headline). A stacked composite of one + component reads a cut of one table's rows, so its measure label names the cut by the + declaration's `measure_noun`, as a sentence does ("... (bilateral flows)" for DAC2a's + bilateral destinations), and its declaration label, the table's own, adds nothing + unless no measure label leads. Returns `None` when neither part has a label. The + presentation door labels a collapsed family's `dcid:` ref with this, and the + datasets door's `stat_vars_summed` warning carries the same string as `stat_var_label`. """ shape = cast("Mapping[str, object]", identity.get("shape", {})) slots = cast("Sequence[Mapping[str, object]]", identity.get("slots", ())) @@ -240,7 +244,13 @@ def family_label( ref = composite_ref(identity) composite = index.composite(ref) if ref is not None else None if composite is not None: - parts = [*(parts if composite.kind == "stacked" else []), composite.label] + if composite.kind == "spliced": + parts = [composite.label] + elif len(composite.components) > 1: + parts = [*parts, composite.label] + else: + lead = parts or [composite.label] + parts = [f"{label} {composite.measure_noun}" for label in lead] pinned = elected_pins(index, identity_family_query(index, without_composite(identity))) named = frozenset(match.named_values) if match is not None else frozenset() diff --git a/packages/dfo-data-tools/src/code_index/multilaterals.py b/packages/dfo-data-tools/src/code_index/multilaterals.py new file mode 100644 index 000000000..4b964348a --- /dev/null +++ b/packages/dfo-data-tools/src/code_index/multilaterals.py @@ -0,0 +1,78 @@ +"""Which parties are multilateral organisations, read from the code index and the entity +resolver's organisation records, so Search (donors) and the mint (DAC2a destinations) classify +a party the same way. + +A multilateral is an official party the graph types `Organization`: the organisation records +the entity resolver exports from the graph (`organisation_dcids`), less the private providers, +which the code index classes `private` (`PartyRow.party_class`). The EU institutions count, +since they spend their member states' contributions +(`code_index.scope_conventions.REDISTRIBUTED_PROVIDERS`). +""" + +from __future__ import annotations + +import functools +import importlib.resources +import importlib.util +import json +from collections.abc import Collection +from typing import Final, NamedTuple + +from code_index.types import PartyClass, PartyKind, PartyRow, RoleAxis + +COUNTRY_PREFIX: Final[str] = "country/" +"""The dcid prefix of a country node.""" + +_RECORDS_PACKAGE = "entity_resolution" +_RECORDS = ("data", "graph_entities.json") + + +@functools.cache +def organisation_dcids() -> frozenset[str]: + """The dcids of the entity resolver's committed records of kind `organisation`: the nodes + the graph types `Organization`, multilaterals and private foundations alike. The one + reader of them, which `entity_resolution.pack` imports. + + The records are read through `importlib.resources` from a module built from the package's + import spec and never executed, so the read works from a wheel or a zip as from disk, and + `import code_index` runs no `entity_resolution.__init__` and loads no resolvekit.""" + spec = importlib.util.find_spec(_RECORDS_PACKAGE) + if spec is None: + raise RuntimeError(f"the {_RECORDS_PACKAGE} package is not installed") + package = importlib.util.module_from_spec(spec) + records = json.loads( + importlib.resources.files(package).joinpath(*_RECORDS).read_text(encoding="utf-8") + ) + return frozenset(record["dcid"] for record in records if record["kind"] == "organisation") + + +class PartyFacts(NamedTuple): + """What one party is where some datasets publish it on one axis: its kind + (`PartyRow.kind_over`), its class (`PartyRow.class_of`), and whether it is a multilateral, + each `None` where those datasets give it different kinds.""" + + kind: PartyKind | None + party_class: PartyClass | None + multilateral: bool | None + + +def party_facts(row: PartyRow, *, axis: RoleAxis, datasets: Collection[str] = ()) -> PartyFacts: + """The facts of `row` where `datasets` publish it on `axis`. A multilateral is an + organisation that is an official party there; a total, such as the World Bank Group beside + its members in DAC2a, is none.""" + kind = row.kind_over(axis, datasets) + if kind is None: + return PartyFacts(None, None, None) + party_class = row.class_of(kind) + multilateral = ( + kind == "party" and party_class == "official" and row.dcid in organisation_dcids() + ) + return PartyFacts(kind, party_class, multilateral) + + +def is_multilateral(row: PartyRow | None, *, datasets: Collection[str] = ()) -> bool: + """Whether `row` is a multilateral where `datasets` publish it as a donor, on the origin + axis (`party_facts`).""" + return ( + row is not None and party_facts(row, axis="origin", datasets=datasets).multilateral is True + ) diff --git a/packages/dfo-data-tools/src/code_index/scope_conventions.py b/packages/dfo-data-tools/src/code_index/scope_conventions.py index d32582b58..15ecf1c82 100644 --- a/packages/dfo-data-tools/src/code_index/scope_conventions.py +++ b/packages/dfo-data-tools/src/code_index/scope_conventions.py @@ -9,7 +9,9 @@ `REDISTRIBUTED_PROVIDERS` the providers no pool takes a scope counting multilateral aid with, and the group whose members fund each, `PUBLISHED_PROVIDER_SCOPES` the provider scope a published donor total counts, `INCLUSIVE_SOURCES` the composite the inclusive scope binds on -each table that offers it, and `DONOR_ORGANISATIONS` the organisations a published donor +each table that offers it, `BILATERAL_SOURCES` the composite the bilateral scope binds on a +published total that counts core contributions as recipients and the population of +destinations its components read, and `DONOR_ORGANISATIONS` the organisations a published donor total counts as donors. `OPTION_LABELS` words the options the rules build from code and group labels, and `REGION_READINGS` names the graph regions a DAC dataset counts as the OECD's region. `search.scope` reads them all, and `graph_release.declarations` hashes the tables @@ -19,7 +21,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Collection, Iterable from dataclasses import dataclass from typing import Literal @@ -36,9 +38,13 @@ DAC1_MULTILATERAL_ODA, DAC1_TOTAL_ODA, DAC2A, + DAC2A_BILATERAL, DAC2A_INCLUSIVE, DAC2A_TOTAL_ODA, + TOTAL_GROUPS, ) +from code_index.multilaterals import COUNTRY_PREFIX, organisation_dcids, party_facts +from code_index.types import ComponentDestinations, PartyRow MeasureBasis = Literal["gross", "net", "grant_equivalent", "commitments", "oecd_headline"] """Every measure basis a part can be counted on. `oecd_headline` is the OECD's headline @@ -137,6 +143,39 @@ class InclusiveSource: note: str +@dataclass(frozen=True) +class BilateralSource: + """The stacked composite (`code_index.declarations.COMPOSITES`) the bilateral scope binds + on a part counting the published total `code` on `code_system` of `dataset` + (`PUBLISHED_PROVIDER_SCOPES`), which counts a donor's bilateral flows to countries and + its core contributions to the multilateral agencies it lists as recipients. A component + of `dataset` that binds its destinations to the bilateral side + (`ComponentRecipe.destinations`) reads the population this row declares + (`RolePopulation`). + + `rebound` is the dataset's published total of the bilateral side, which a component + binds when the identity leaves its destination out, and `spanning` the total of both + sides, which no such component reads; `rebound_words` and `spanning_words` name the two + for a reader. `rule` names the default a fresh part counting the total settles on, and + `note` says what the composite counts. `label` names the bilateral option in place of + `OptionLabels.provider_bilateral`, and `total_label` the option holding the total itself + in place of `OptionLabels.provider_contributions`, since the total counts every + destination the table lists.""" + + dataset: str + code_system: str + code: str + composite: str + spanning: str + rebound: str + spanning_words: str + rebound_words: str + rule: str + note: str + label: str + total_label: str + + @dataclass(frozen=True) class ProviderScopeRule: """The rule a provider option of `scope` names where no convention @@ -257,7 +296,9 @@ class OptionLabels: provider scopes, `provider_inclusive_net` the inclusive scope on a part counted on net flows, which adds the imputed line as published, and `provider_years` a scope's `label` followed by the `years` its option serves ("2011 to 2024", "2023") where they set the - part's window.""" + part's window. `provider_rebound` names a scope's `label` read at the total of the + bilateral side in place of the total of both sides the part names + (`BilateralSource.rebound_words`, `BilateralSource.spanning_words`).""" sector_code: str sector_code_alone: str @@ -274,6 +315,7 @@ class OptionLabels: provider_contributions: str provider_multilateral: str provider_inclusive_net: str + provider_rebound: str OPTION_LABELS = OptionLabels( @@ -293,6 +335,7 @@ class OptionLabels: provider_contributions="bilateral plus core contributions to multilaterals", provider_multilateral="core contributions to multilaterals", provider_inclusive_net="net bilateral plus imputed multilateral as published, by convention", + provider_rebound="{label}, to {rebound} in place of {spanning}", ) CONVENTION_CURRENCY = "USD" @@ -411,9 +454,10 @@ class RegionReading: # which attributes the multilateral organisations' own spending in the sector to the donors in # proportion to their core contributions. Every other part counts bilateral flows. DAC1 donor # totals already count core contributions to multilaterals, so they take no inclusive scope. -# DAC2a offers it from its own imputed line (`INCLUSIVE_SOURCES`), which it publishes on a -# gross basis alone, so a DAC2a total keeps the bilateral flows it counts unless a switch or -# the request's words ask for the imputed share. +# DAC2a's total ODA counts core contributions too, wherever it lists a multilateral agency as a +# recipient, so a fresh DAC2a part reads the bilateral destinations alone (`BILATERAL_SOURCES`) +# and keeps them unless a switch or the request's words ask for the imputed share +# (`INCLUSIVE_SOURCES`) or the total itself. PROVIDER_CONVENTIONS: tuple[ProviderConvention, ...] = ( ProviderConvention( rule="named_donor_total", @@ -470,8 +514,11 @@ class RegionReading: # timing and in amount. DAC1's bilateral ODA counts the bilateral flows alone, and its # multilateral ODA the core contributions alone. A part reading one row is offered every other # row of its dataset and code system as an option that binds that row's code. DAC2a's total -# ODA to a recipient counts the donor's bilateral flows, and offers the inclusive scope from -# DAC2a's own imputed line (`INCLUSIVE_SOURCES`). +# ODA counts the donor's bilateral flows to countries and regions and its core contributions to +# the multilateral agencies it lists as recipients, so over every destination it is DAC1's +# total ODA by recipient. Its bilateral scope binds the composite that reads the bilateral +# destinations alone (`BILATERAL_SOURCES`), and its inclusive scope DAC2a's own imputed line +# (`INCLUSIVE_SOURCES`). PUBLISHED_PROVIDER_SCOPES: tuple[PublishedProviderScope, ...] = ( PublishedProviderScope( dataset=DAC1, @@ -501,9 +548,12 @@ class RegionReading: dataset=DAC2A, code_system="dac2aAidType", code=DAC2A_TOTAL_ODA, - scope="bilateral", - rule="bilateral_recipient_total", - note="DAC2a ODA to a recipient counts the donor's bilateral flows", + scope="contributions", + rule="contribution_recipient_total", + note=( + "DAC2a total ODA counts bilateral flows to countries and regions plus core " + "contributions to the multilateral agencies it lists as recipients" + ), ), ) @@ -534,6 +584,182 @@ class RegionReading: # no provider record. DONOR_ORGANISATIONS: tuple[str, ...] = (_EU_INSTITUTIONS,) +# The DAC2a recipients that are no named organisation but stand for the multilaterals as a +# group. The DAC lists them as unspecified parties, so the graph types them as residuals and +# `party_facts` reads none of them as a multilateral. Each holds a donor's core contributions. +DAC_MULTILATERAL_RECIPIENTS: tuple[str, ...] = ( + # Core contributions to multilateral agencies the DAC does not itemise. + "DACOtherMultilateralRecipient", + # Core contributions to United Nations agencies other than those the DAC itemises. + "DACOtherUnitedNations", + # Core contributions to regional development banks other than those the DAC itemises. + "DACOtherRegionalDevelopmentBanks", + # Core contributions to World Bank Group members other than those the DAC itemises. + "DACOtherWorldBank", + # Core contributions to EU institutions other than those the DAC itemises. + "DACOtherEUInstitutions", +) + +# The organisations the DAC counts as bilateral recipients: DACDevelopingCountries holds them +# in 206, though the graph types them `Organization` and `party_facts` reads them as +# multilaterals on the origin axis. Read on the destination axis only, ahead of `party_facts`. +DAC_BILATERAL_ORGANISATIONS: tuple[str, ...] = ( + # A regional union of partner states the DAC lists among developing-country recipients. + "EastAfricanCommunity", + # A river-basin fund between India and Pakistan that the DAC lists as a country recipient. + "IndusBasinDevelopmentFund", +) + +# The DAC2a recipient total of every multilateral recipient: its named organisations and the +# unspecified ones in `DAC_MULTILATERAL_RECIPIENTS`. +DAC_ALL_MULTILATERAL_RECIPIENTS = "DACAllMultilateralRecipients" + +# The DAC2a total of all recipients, "Developing Countries, Total" plus the multilateral +# agencies, so it spans both sides. +DAC_ALL_RECIPIENTS = "DACAllRecipients" + +# The DAC2a total of every developing-country recipient: the countries, regions and the DAC's +# unallocated lines, the bilateral side of `DAC_ALL_RECIPIENTS`. +DAC_DEVELOPING_COUNTRIES = "DACDevelopingCountries" + +# The DAC2a recipient totals on the bilateral side that `REGION_NAMES` does not name: groups of +# developing countries by income, by the DAC's or the UN's lists, or by the World Bank's income +# group. Each counts countries alone, and none counts a multilateral agency. +DAC_BILATERAL_GROUP_TOTALS: tuple[str, ...] = ( + "DACACPCountries", + "DACCentralAndEasternEuropeanCountries", + "DACClubDuSahelCountries", + "DACFragileStates", + "DACLowerMiddleIncomeCountries", + "DACMoreAdvancedDevelopingCountriesAndTerritories", + "DACNewIndependentStates", + "DACNorthernAmerica", + "DACOtherLowIncomeCountries", + "DACSmallIslandStates", + "DACUnallocatedByIncome", + "DACUpperMiddleIncomeCountries", + "HeavilyIndebtedPoorCountries", + "UNLandlockedDevelopingCountries", + "UNLeastDevelopedCountries", + "WorldBankHighIncome", + "WorldBankLowIncome", + "WorldBankLowerMiddleIncome", + "WorldBankNotClassifiedByIncome", + "WorldBankUpperMiddleIncome", +) + +# The DAC2a residual of bilateral flows the donor allocates to no country or region, which +# `DAC_DEVELOPING_COUNTRIES` counts beside the regional residuals `TOTAL_GROUPS` lists. +DAC_BILATERAL_UNALLOCATED = "DACBilateralUnallocated" + +# The DAC2a recipient totals on the bilateral side: `DAC_DEVELOPING_COUNTRIES`, the DAC regions +# (`REGION_NAMES`) and `DAC_BILATERAL_GROUP_TOTALS`. +_BILATERAL_TOTALS = frozenset( + (DAC_DEVELOPING_COUNTRIES, *(dcid for dcid, _ in REGION_NAMES), *DAC_BILATERAL_GROUP_TOTALS) +) + +# The DAC2a residuals on the bilateral side: the unallocated lines each regional total counts +# beside its members (`TOTAL_GROUPS`) and `DAC_BILATERAL_UNALLOCATED`. +_BILATERAL_RESIDUALS = frozenset( + ( + DAC_BILATERAL_UNALLOCATED, + *(residual for group in TOTAL_GROUPS.values() for residual in group.residuals), + ) +) + +# The composite each published total counting core contributions over every destination binds +# for its bilateral scope, and the population every composite component binding the dataset's +# destinations reads (`RolePopulation`), the bilateral and the inclusive composite's alike. +# DAC2a's total ODA lists the multilateral agencies among its recipients, so an open +# destination binds the bilateral-side parties and residuals alone, and a named agency, the +# DAC's total of them or its total of all recipients, which holds both sides, is refused +# (`code_index.composites.component_identities`). An identity naming no destination binds the +# DAC's total of developing countries, the bilateral side of all recipients. +BILATERAL_SOURCES: tuple[BilateralSource, ...] = ( + BilateralSource( + dataset=DAC2A, + code_system="dac2aAidType", + code=DAC2A_TOTAL_ODA, + composite=DAC2A_BILATERAL, + spanning=DAC_ALL_RECIPIENTS, + rebound=DAC_DEVELOPING_COUNTRIES, + spanning_words="all recipients", + rebound_words="developing countries", + rule="bilateral_destinations", + note="DAC2a total ODA to countries, regions and the DAC's unallocated lines alone", + label=( + "bilateral flows to countries and regions, without core contributions to " + "multilateral agencies" + ), + total_label="total ODA, including core contributions to multilateral agencies", + ), +) + +DestinationSide = Literal["bilateral", "multilateral", "spanning"] +"""Which side of a donor's aid a DAC2a destination is on: a country or a group of countries +(`bilateral`), a multilateral agency (`multilateral`), or `DACAllRecipients`, the total of both +(`spanning`).""" + + +def destination_side(row: PartyRow, *, datasets: Collection[str]) -> DestinationSide | None: + """The side of `row` as a recipient in `datasets`, or `None` when it is unclassified: no + dataset publishes it as a recipient, they give it different kinds, or the classifier + cannot place it. + + A total is `spanning` when `DAC_ALL_RECIPIENTS`, `multilateral` when + `DAC_ALL_MULTILATERAL_RECIPIENTS`, and `bilateral` when declared on that side + (`DAC_DEVELOPING_COUNTRIES`, `REGION_NAMES`, `DAC_BILATERAL_GROUP_TOTALS`). A residual is + `multilateral` when in `DAC_MULTILATERAL_RECIPIENTS` and `bilateral` when declared on that + side (`TOTAL_GROUPS` residuals, `DAC_BILATERAL_UNALLOCATED`). A party is `bilateral` when + in `DAC_BILATERAL_ORGANISATIONS`, `multilateral` where `party_facts` says so, `bilateral` + when a country or an organisation the entity resolver knows, and unclassified otherwise. + """ + kind = row.kind_over("destination", datasets) + if kind is None: + return None + if kind == "total": + if row.dcid == DAC_ALL_RECIPIENTS: + return "spanning" + if row.dcid == DAC_ALL_MULTILATERAL_RECIPIENTS: + return "multilateral" + return "bilateral" if row.dcid in _BILATERAL_TOTALS else None + if kind == "residual": + if row.dcid in DAC_MULTILATERAL_RECIPIENTS: + return "multilateral" + return "bilateral" if row.dcid in _BILATERAL_RESIDUALS else None + if row.dcid in DAC_BILATERAL_ORGANISATIONS: + return "bilateral" + if party_facts(row, axis="destination", datasets=datasets).multilateral: + return "multilateral" + if row.dcid.startswith(COUNTRY_PREFIX) or row.dcid in organisation_dcids(): + return "bilateral" + return None + + +@dataclass(frozen=True) +class RolePopulation: + """The destinations a composite component reads where it binds its destinations to one + side (`ComponentRecipe.destinations`), derived from one code index + (`code_index.composites.destination_population`) and declared by `source`. + + `members` is what an open destination reads: the sorted dcids of the `side` parties and + residuals the dataset publishes as destinations, its totals left out as an open read + leaves them out. `named` is what a named destination may be: every destination the + dataset publishes on `side`, its totals included. `total` is the dataset's published + total of `side` (`BilateralSource.rebound`), which an identity naming no destination + reads.""" + + side: ComponentDestinations + members: tuple[str, ...] + named: frozenset[str] + source: BilateralSource + + @property + def total(self) -> str: + """The dataset's published total of `side`.""" + return self.source.rebound + + ODA_FLOW_SYSTEM = "crsTypeOfFlow" """The code system an ODA flow binding binds.""" ODA_FLOW = "CRSTypeOfFlow10" diff --git a/packages/dfo-data-tools/src/code_index/types.py b/packages/dfo-data-tools/src/code_index/types.py index 8b5c13f1e..45e22a481 100644 --- a/packages/dfo-data-tools/src/code_index/types.py +++ b/packages/dfo-data-tools/src/code_index/types.py @@ -476,6 +476,11 @@ def holds(self, year: int) -> bool: ) +ComponentDestinations = Literal["bilateral"] +"""The side of a donor's aid a composite component binds its destinations to +(`ComponentRecipe.destinations`).""" + + @dataclass(frozen=True, slots=True) class ComponentRecipe: """How one component's identity derives from its composite's identity. @@ -498,7 +503,13 @@ class ComponentRecipe: writes a zero row for each of its members holding no row in a cell another component fills, inside the composite's years. A component without it contributes nothing where it holds no row. Only a stacked composite's component sets it: a spliced one's zeros would - fill years it does not cover, and the loader refuses it.""" + fill years it does not cover, and the loader refuses it. + + `destinations` binds the component's destination role to one side of a donor's aid. + `"bilateral"` reads the destinations whose `code_index.scope_conventions.destination_side` + is `bilateral`: countries, regions, groups of countries and the DAC's unallocated lines + (`code_index.composites.component_identities`). `None` reads the destinations the + identity binds or opens.""" role: ComponentRole dataset: str @@ -509,6 +520,7 @@ class ComponentRecipe: bind: Mapping[str, str] = field(default_factory=dict) drop_unbound: frozenset[str] = frozenset() zero_absent: bool = False + destinations: ComponentDestinations | None = None @dataclass(frozen=True, slots=True) diff --git a/packages/dfo-data-tools/src/dc_sdmx/__init__.py b/packages/dfo-data-tools/src/dc_sdmx/__init__.py index 2f67956f2..37bf7b4aa 100644 --- a/packages/dfo-data-tools/src/dc_sdmx/__init__.py +++ b/packages/dfo-data-tools/src/dc_sdmx/__init__.py @@ -13,6 +13,7 @@ from dc_sdmx.client import ( DATA_HEDGE_AFTER_SECONDS, DEFAULT_PLANNING_LANES, + Among, ContainedIn, EntityFilter, SdmxClient, @@ -59,6 +60,7 @@ "DIMENSION_COLUMNS", "VALUE_COLUMN", "VARIABLE_COLUMN", + "Among", "ContainedIn", "EntityFilter", "SdmxAvailabilityResult", diff --git a/packages/dfo-data-tools/src/dc_sdmx/client.py b/packages/dfo-data-tools/src/dc_sdmx/client.py index a1ca91dee..fb4e1b3a4 100644 --- a/packages/dfo-data-tools/src/dc_sdmx/client.py +++ b/packages/dfo-data-tools/src/dc_sdmx/client.py @@ -544,7 +544,21 @@ class ContainedIn: child_type: str -type EntityFilter = Sequence[str] | ContainedIn +@pydantic.dataclasses.dataclass(frozen=True) +class Among: + """Select the entities in `members`, a declared population (`side`) bound in place of + names a caller gave, as an open DAC2a destination under a bilateral composite binds the + bilateral-side recipients. + + Requests send `members` as the role's OR-list, as a list of dcids is sent. A reader of + rows reads `members`; a reader of the names a caller asked for reads the role as open.""" + + members: tuple[str, ...] + side: str + + +type EntityFilter = Sequence[str] | ContainedIn | Among +"""A role's binding: an OR-list of dcids, a containment filter, or a population.""" @dataclass(frozen=True, slots=True) @@ -2698,9 +2712,10 @@ def _entity_params(entities: Mapping[str, EntityFilter]) -> dict[str, str]: params[f"c[{role}.containedInPlace+]"] = filt.parent params[f"c[{role}.typeOf]"] = filt.child_type else: - if not filt: + refs = filt.members if isinstance(filt, Among) else filt + if not refs: raise SdmxQueryError(f"entity role {role!r} has no values") - params[f"c[{role}]"] = _or_list(role, filt) + params[f"c[{role}]"] = _or_list(role, refs) return params diff --git a/packages/dfo-data-tools/src/entity_resolution/pack.py b/packages/dfo-data-tools/src/entity_resolution/pack.py index ffc2c44a6..af22224c8 100644 --- a/packages/dfo-data-tools/src/entity_resolution/pack.py +++ b/packages/dfo-data-tools/src/entity_resolution/pack.py @@ -15,6 +15,10 @@ from pathlib import Path from typing import TYPE_CHECKING +# `pack.organisation_dcids` is the reader `code_index.multilaterals` defines, so the code index +# and the mint read the organisation records without loading resolvekit. +from code_index.multilaterals import organisation_dcids as organisation_dcids + if TYPE_CHECKING: from resolvekit.core.model.result import CandidateSummary @@ -243,17 +247,6 @@ def longest_name_words() -> int: return max((len(key.split()) for key in _committed_alias_index().owners), default=0) -@functools.cache -def organisation_dcids() -> frozenset[str]: - """The dcids of the committed records of kind `organisation`: the nodes the graph types - `Organization`, multilaterals and private foundations alike.""" - return frozenset( - record["dcid"] - for record in _load_records(records_path()) - if record["kind"] == "organisation" - ) - - def build_pack( *, records_path: Traversable | Path, curated_path: Traversable | Path, out_dir: Path ) -> Path: diff --git a/packages/dfo-data-tools/src/graph_release/declarations.py b/packages/dfo-data-tools/src/graph_release/declarations.py index 024df486e..a4c15697f 100644 --- a/packages/dfo-data-tools/src/graph_release/declarations.py +++ b/packages/dfo-data-tools/src/graph_release/declarations.py @@ -58,7 +58,15 @@ (code_index_declarations, "SLOT_PLAIN_NAMES"), (scope_conventions, "BASIS_MEASURES"), (scope_conventions, "BASIS_RESTRICTIONS"), + (scope_conventions, "BILATERAL_SOURCES"), (scope_conventions, "CONVENTION_CURRENCY"), + (scope_conventions, "DAC_ALL_MULTILATERAL_RECIPIENTS"), + (scope_conventions, "DAC_ALL_RECIPIENTS"), + (scope_conventions, "DAC_BILATERAL_GROUP_TOTALS"), + (scope_conventions, "DAC_BILATERAL_ORGANISATIONS"), + (scope_conventions, "DAC_BILATERAL_UNALLOCATED"), + (scope_conventions, "DAC_DEVELOPING_COUNTRIES"), + (scope_conventions, "DAC_MULTILATERAL_RECIPIENTS"), (scope_conventions, "DONOR_ORGANISATIONS"), (scope_conventions, "INCLUSIVE_SOURCES"), (scope_conventions, "MEASURE_BASIS_CONVENTIONS"), diff --git a/packages/dfo-data-tools/src/graph_release/service.py b/packages/dfo-data-tools/src/graph_release/service.py index d939d841c..47b01c03a 100644 --- a/packages/dfo-data-tools/src/graph_release/service.py +++ b/packages/dfo-data-tools/src/graph_release/service.py @@ -70,6 +70,7 @@ import pyarrow as pa +from code_index.composites import population_check from code_index.query import CodeIndex from code_index.types import MeasureIndex from dfo_platform.cloud_logging import JSON_FIELDS_ATTR @@ -191,13 +192,26 @@ class ReleaseViews: def build_release_views(loaded: LoadedRelease) -> ReleaseViews: """The serving views of `loaded`. Raises `graph_release.views.ViewRefused` when one does - not build.""" + not build. + + Checks the destination populations of the code index against the coverage view + (`code_index.composites.population_check`) as they open, and logs + `destination_population_unavailable` for each population that does not hold every + destination coverage serves, or whose variables' recipients coverage cannot list. The release still serves: the composites reading that + population refuse until a release places every destination.""" code_index, measure_index = release_code_indexes(loaded) - return ReleaseViews( - code_index=code_index, - measure_index=measure_index, - coverage=release_coverage_index(loaded), - ) + coverage = release_coverage_index(loaded) + for unclassified in population_check(code_index, coverage).unclassified.values(): + _event( + logging.ERROR, + "destination_population_unavailable", + release_id=loaded.manifest.release_id, + dataset=unclassified.dataset, + found_in=unclassified.found_in, + destinations=list(unclassified.destinations), + unlisted=list(unclassified.unlisted), + ) + return ReleaseViews(code_index=code_index, measure_index=measure_index, coverage=coverage) @dataclass(frozen=True, eq=False) diff --git a/packages/dfo-data-tools/src/materialise/bundle.py b/packages/dfo-data-tools/src/materialise/bundle.py index caef89fa6..10e30c19c 100644 --- a/packages/dfo-data-tools/src/materialise/bundle.py +++ b/packages/dfo-data-tools/src/materialise/bundle.py @@ -28,7 +28,13 @@ from dfo_platform.codes import CodeCatalogue from dfo_platform.publisher import DFO_NAME from materialise.bundle_citation import build_citation_cff -from materialise.bundle_docs import blank_to_none, build_readme, build_sources, site_path +from materialise.bundle_docs import ( + blank_to_none, + build_readme, + build_sources, + composite_line, + site_path, +) from materialise.cell_safety import spreadsheet_safe from materialise.errors import MaterialisationUsageError from materialise.license_type import licence_ids, licence_title @@ -187,6 +193,20 @@ def credited_publisher(self) -> str | None: return blank_to_none(documented) or blank_to_none(self.publisher) +@dataclass(frozen=True) +class CompositeNote: + """A composite provenance (`code_index.composites`) a member's rows carry in their + `provenance` column. Its component tables are the member's `sources`.""" + + ref: str + """The composite's provenance dcid, as the `provenance` column holds it.""" + label: str + """The composite's bundle name (`bundle_sources.composite_bundle_label`): its declared + label, followed by its measure noun where a component table shares the label.""" + summary: str + """One sentence on what the composite adds or reads (`bundle_sources.composite_summary`).""" + + @dataclass(frozen=True) class ColumnDoc: """The meaning of one column of a member's data file.""" @@ -232,6 +252,8 @@ class MemberProvenance: """One entry per documented column, in file order.""" description: str | None = None """What one row of the member's data holds, written as the resource `description`.""" + composites: tuple[CompositeNote, ...] = () + """The composite provenances the member's rows carry, in first-seen order.""" @dataclass(frozen=True) @@ -269,7 +291,9 @@ class BundleContext: """The bundle title, from `bundle_naming.heading` or a card's title.""" summary: str """One line of coverage facts, such as "1,382 rows · 309 variables · 1 donor". - Written as the package `description` and CFF `abstract`. Empty omits both.""" + Written as the CFF `abstract` and as the package `description`, which goes on to name + each composite the members' rows carry (`bundle_docs.composite_line`). Empty omits the + abstract, and the description where no composite follows.""" slug: str """The package `name`, from `bundle_naming.sources_slug` or `card_slug`.""" version: str @@ -397,9 +421,11 @@ def _assemble( members=members, context=context, codes_csv=codes_csv, dfo_property=dfo_property ) sources = _distinct_sources(members) + composites = distinct_composites(members) readme = build_readme( members=members, sources=sources, + composites=composites, context=context, codes=codes, codes_path=CODES_PATH if codes_csv is not None else None, @@ -415,7 +441,9 @@ def _assemble( sources=sources, context=context, dfo_entity=_dfo_entity(context) ), "README.md": readme.encode("utf-8"), - "SOURCES.md": build_sources(sources=sources, context=context).encode("utf-8"), + "SOURCES.md": build_sources(sources=sources, composites=composites, context=context).encode( + "utf-8" + ), } if codes_csv is not None: files[CODES_PATH] = codes_csv @@ -470,6 +498,15 @@ def _reproducing_call_entry(call: ReproducingCall, context: BundleContext) -> di } +def distinct_composites(members: Sequence[MemberProvenance]) -> list[CompositeNote]: + """Every member's composites, first-seen by `ref`.""" + seen: dict[str, CompositeNote] = {} + for member in members: + for composite in member.composites: + seen.setdefault(composite.ref, composite) + return list(seen.values()) + + def _distinct_sources(members: Sequence[MemberProvenance]) -> list[SourceProvenance]: """Every member's sources, first-seen by `ref`.""" seen: dict[str | None, SourceProvenance] = {} @@ -511,8 +548,16 @@ def _build_datapackage( "id": context.links.version, "title": context.heading, } - if context.summary: - descriptor["description"] = context.summary + description = "\n\n".join( + part + for part in ( + context.summary, + *(composite_line(composite) for composite in distinct_composites(members)), + ) + if part + ) + if description: + descriptor["description"] = description descriptor |= { "version": context.version, "created": _rfc3339(context.prepared_at), diff --git a/packages/dfo-data-tools/src/materialise/bundle_docs.py b/packages/dfo-data-tools/src/materialise/bundle_docs.py index d31079fd1..1ae83d8b0 100644 --- a/packages/dfo-data-tools/src/materialise/bundle_docs.py +++ b/packages/dfo-data-tools/src/materialise/bundle_docs.py @@ -25,6 +25,7 @@ CodeEntry, ColumnDoc, ComponentExport, + CompositeNote, Licence, MemberProvenance, ReproducingCall, @@ -132,6 +133,15 @@ def _query_url(base: str, options: Mapping[str, object]) -> str: return f"{base}?{urlencode(encoded, safe=',')}" +def composite_line(composite: CompositeNote) -> str: + """A composite named by its label, then its summary and its provenance code, on one + line.""" + return ( + f"{_inline(composite.label)}: {_inline(composite.summary)} " + f"Its `provenance` code is {_code(composite.ref)}." + ) + + def site_path(context: BundleContext, path: str) -> str: """`path` on the bundle's public site, an absolute URL.""" return context.site_url.rstrip("/") + path @@ -149,6 +159,7 @@ def build_readme( codes: Sequence[CodeEntry], codes_path: str | None, definition_id: str | None, + composites: Sequence[CompositeNote] = (), reproducing_call: ReproducingCall | None, export: ComponentExport | None, ) -> str: @@ -157,12 +168,14 @@ def build_readme( is. `sources` are the distinct sources across `members`, and `codes_path` is the codes table's - bundle path, `None` when the bundle has none. `definition_id` is the spec id the Version - section names. A card export (`export` set) + bundle path, `None` when the bundle has none. `composites` are the distinct composites + the members' rows carry, each named with what it adds or reads (`_composites_section`). + `definition_id` is the spec id the Version section names. A card export (`export` set) names the card revision and its frame's identity in place of a Version section. """ sections = [ _readme_heading(context, export), + _composites_section(composites), _files_section(members, codes=codes, codes_path=codes_path), _variables_section(codes, codes_path=codes_path, files=len(members)), _columns_section(members, codes_path=codes_path), @@ -197,6 +210,21 @@ def _readme_heading(context: BundleContext, export: ComponentExport | None) -> s return "\n".join(lines) +def _composites_section(composites: Sequence[CompositeNote]) -> str: + """The composites the data's `provenance` column names, each with what it adds or reads. + Empty for data that names none.""" + if not composites: + return "" + intro = ( + "The DFO composes these figures from the tables in SOURCES.md:" + if len(composites) > 1 + else "The DFO composes this figure from the tables in SOURCES.md:" + ) + lines = ["## How the figures are composed", "", intro, ""] + lines += [f"- {composite_line(composite)}" for composite in composites] + return "\n".join(lines) + + def _files_section( members: Sequence[MemberProvenance], *, @@ -483,11 +511,18 @@ def _version_section(context: BundleContext, *, definition_id: str | None) -> st # SOURCES.md -def build_sources(*, sources: Sequence[SourceProvenance], context: BundleContext) -> str: - """The bundle's SOURCES.md: one section per source with its publisher, description, links, - licence, attribution, coverage, the steps the DFO applied, the DFO pipeline's releases, and - links to the full documentation. Absent fields are left out. A source with no documentation - snapshot gets its name, publisher, licence and source page only.""" +def build_sources( + *, + sources: Sequence[SourceProvenance], + context: BundleContext, + composites: Sequence[CompositeNote] = (), +) -> str: + """The bundle's SOURCES.md: one section per composite the data names, with its + provenance code and what it adds or reads from the sources below, then one section per + source with its publisher, description, links, licence, attribution, coverage, the steps + the DFO applied, the DFO pipeline's releases, and links to the full documentation. Absent + fields are left out. A source with no documentation snapshot gets its name, publisher, + licence and source page only.""" intro = ( "# Sources\n\n" "Who publishes the data in this download, the terms for its reuse, and each step the " @@ -495,10 +530,22 @@ def build_sources(*, sources: Sequence[SourceProvenance], context: BundleContext ) if not sources: return f"{intro}\n\nThe data names no source.\n" - blocks = [intro, *(_source_block(source, context=context) for source in sources)] + blocks = [ + intro, + *(_composite_block(composite) for composite in composites), + *(_source_block(source, context=context) for source in sources), + ] return "\n\n".join(blocks) + "\n" +def _composite_block(composite: CompositeNote) -> str: + return ( + f"## {_inline(composite.label)}\n\n" + f"{_inline(composite.summary)} The DFO composes it from the sources below, and the " + f"`provenance` column names it {_code(composite.ref)}." + ) + + def _source_block(source: SourceProvenance, *, context: BundleContext) -> str: documentation = source.documentation blocks = [f"## {_inline(source.name)}"] diff --git a/packages/dfo-data-tools/src/materialise/bundle_sources.py b/packages/dfo-data-tools/src/materialise/bundle_sources.py index 475fd820f..6685fa6a1 100644 --- a/packages/dfo-data-tools/src/materialise/bundle_sources.py +++ b/packages/dfo-data-tools/src/materialise/bundle_sources.py @@ -3,16 +3,21 @@ Both bundle producers cite sources from the same stored shape: a Datasets version body's `citation.sources[]` entry, as `datasets_api.manifest_models.CitationSource` writes it in JSON mode. `render` dumps its parsed citation to that form, and a card export reads it from the -frame manifest, where derived frames carry the same shape. Nothing here reads the graph. +frame manifest, where derived frames carry the same shape. A composite provenance's rows are +cited by its component tables, and `composite_notes` names the composite itself from its +declaration. Nothing here reads the graph. """ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import cast +from typing import TYPE_CHECKING, cast +from code_index.composites import DECLARED_COMPOSITES +from code_index.scope_conventions import BASIS_MEASURES, BILATERAL_SOURCES from materialise.bundle import ( ChangeDoc, + CompositeNote, Licence, ReleaseDoc, SourceDocumentation, @@ -20,6 +25,117 @@ TransformationDoc, ) +if TYPE_CHECKING: + from code_index.query import CodeIndex + from code_index.types import ComponentRecipe, CompositeDeclaration + + +def composite_bundle_label(index: CodeIndex, composite: CompositeDeclaration) -> str: + """The name a bundle gives `composite`: its label, followed by its `measure_noun` where + the label is one of its component tables' labels in `index`, so the composite and the + table read apart in one bundle ("OECD DAC2a (bilateral flows)" beside "OECD DAC2a"). + Citations keep the composite's label.""" + tables = { + declared.label + for recipe in composite.components + if (declared := index.datasets.get(recipe.dataset)) is not None + } + if composite.label in tables: + return f"{composite.label} {composite.measure_noun}" + return composite.label + + +def composite_bundle_labels(index: CodeIndex) -> dict[str, str]: + """Each declared composite's ref and its bundle name (`composite_bundle_label`).""" + return { + ref: composite_bundle_label(index, composite) + for ref, composite in DECLARED_COMPOSITES.items() + } + + +def composite_summary(index: CodeIndex, composite: CompositeDeclaration) -> str: + """One sentence on what `composite` adds or reads, derived from its declaration for a + bundle reader: a stacked composite of several components adds them for each provider, + recipient and year, one of one component reads it, and a spliced one joins its + components. Each component is named by its table's label in `index`, after the labels of + the codes it binds and the words of the declared basis it sets (`BASIS_MEASURES`), with + the years its periods hold. A component binding its destinations to the bilateral side + (`ComponentRecipe.destinations`) states what that reading counts + (`BILATERAL_SOURCES`), once for the whole composite where every component binds them.""" + shared = all(recipe.destinations is not None for recipe in composite.components) + phrases = [ + _component_phrase(index, recipe, reading=not shared) for recipe in composite.components + ] + joined = " and ".join(phrases) + if composite.kind == "spliced": + sentence = f"Joins {joined}" + elif len(phrases) > 1: + sentence = f"Adds {joined} for each provider, recipient and year" + else: + sentence = f"Reads {joined}" + if shared and (reading := _destination_reading(composite.components[0])): + sentence += f", counting {reading}" + return sentence + "." + + +def _destination_reading(recipe: ComponentRecipe) -> str | None: + """What a component binding its destinations to the bilateral side counts, as the + `BILATERAL_SOURCES` row of its dataset words it.""" + if recipe.destinations is None: + return None + return next((row.label for row in BILATERAL_SOURCES if row.dataset == recipe.dataset), None) + + +def _component_phrase(index: CodeIndex, recipe: ComponentRecipe, *, reading: bool) -> str: + """`recipe`'s table by its label, after what it binds and its basis, with its years, and + its destination reading when `reading`.""" + declared = index.datasets.get(recipe.dataset) + table = declared.label if declared is not None else recipe.dataset + qualifiers = [ + row.label if (row := index.code_by_dcid(code)) is not None else code + for code in recipe.bind.values() + ] + if recipe.basis is not None: + basis = next( + ( + row.words + for row in BASIS_MEASURES + if (row.measured_property, row.netting_basis) == recipe.basis + ), + None, + ) + if basis is not None: + qualifiers.append(basis) + phrase = f"{' as '.join(qualifiers)} from {table}" if qualifiers else table + periods = recipe.periods + if periods is not None and periods.first is not None and periods.last is not None: + phrase += f" for {periods.first} to {periods.last}" + elif periods is not None and periods.last is not None: + phrase += f" for years to {periods.last}" + elif periods is not None and periods.first is not None: + phrase += f" for years from {periods.first}" + if reading and (counted := _destination_reading(recipe)): + phrase += f", counting {counted}" + return phrase + + +def composite_notes( + provenance_refs: Sequence[str | None], *, index: CodeIndex +) -> tuple[CompositeNote, ...]: + """Each declared composite (`code_index.composites.DECLARED_COMPOSITES`) among + `provenance_refs`, the provenances a member's rows carry, once each in first-seen order, + with its bundle name (`composite_bundle_label`) and what it adds or reads + (`composite_summary`).""" + return tuple( + CompositeNote( + ref=ref, + label=composite_bundle_label(index, composite), + summary=composite_summary(index, composite), + ) + for ref in dict.fromkeys(provenance_refs) + if ref is not None and (composite := DECLARED_COMPOSITES.get(ref)) is not None + ) + def source_provenance(ref: str, cited: Mapping[str, object] | None) -> SourceProvenance: """One bundle source from the stored citation entry of `ref`, in its JSON form. A ref the diff --git a/packages/dfo-data-tools/src/materialise/composite.py b/packages/dfo-data-tools/src/materialise/composite.py index 6608ee6f3..904dd0695 100644 --- a/packages/dfo-data-tools/src/materialise/composite.py +++ b/packages/dfo-data-tools/src/materialise/composite.py @@ -53,7 +53,6 @@ ) from dfo_frame_grammar.arrow import build_frame_from_arrow -from code_index.composites import composite_digest from code_index.members import SourceNotInFamily, identity_family_query, index_family from code_index.types import ComponentRecipe, ComponentRole, CompositeDeclaration, PartyRow from dfo_platform.columns import PROVENANCE_COLUMN @@ -228,10 +227,13 @@ def compose_components( role_columns: Sequence[str], enrichment: EnrichmentLookup, parties: Mapping[str, PartyRow], + digest: str, ) -> MaterialisationResult: """One result composed from `components`, each materialised under its own role, as the - module docstring states. `window` is the composite's requested window and `role_columns` - the role columns every component's frame holds. + module docstring states. `window` is the composite's requested window, `role_columns` + the role columns every component's frame holds, and `digest` the composite's digest under + the code index the run read (`code_index.composites.composite_digest`), which the + dependency fingerprint records. The frame's hash and `content_version` come from the composed chunks, in the one pass that composes them. The trace states @@ -343,7 +345,9 @@ def counted(chunks: Iterator[Frame]) -> Iterator[Frame]: for component in components for stat_var, record in component.result.member_conversion_records.items() }, - dependencies=_dependencies(components, composite=composite, requests=all_requests), + dependencies=_dependencies( + components, composite=composite, requests=all_requests, digest=digest + ), default_scope=tuple( dict.fromkeys( place for component in components for place in component.result.default_scope @@ -867,13 +871,14 @@ def _dependencies( *, composite: CompositeDeclaration, requests: Sequence[MaterialisationRequest], + digest: str, ) -> Dependencies: """Every component's fingerprint as one: its members over every component, and each source, conversion source, recipe and latest year any component recorded. The window is the first component's, as `dependencies.refreshed_dependencies` reads the first request's. `imported_data` is the components' when every one records the same key, and `None` when any records none or another. `composites` holds `composite`'s provenance and - the digest of the declaration it was composed by (`code_index.composites.composite_digest`).""" + `digest`, the digest of the declaration it was composed by.""" fingerprints = [component.result.dependencies for component in components] imported = {fingerprint.imported_data for fingerprint in fingerprints} @@ -891,7 +896,7 @@ def merged(name: str) -> tuple[tuple[str, object], ...]: window=fingerprints[0].window, latest_years=cast("tuple", merged("latest_years")), imported_data=imported.pop() if len(imported) == 1 else None, - composites=((composite.provenance, composite_digest(composite)),), + composites=((composite.provenance, digest),), ) diff --git a/packages/dfo-data-tools/src/materialise/dependencies.py b/packages/dfo-data-tools/src/materialise/dependencies.py index aa43aef85..99d2d5572 100644 --- a/packages/dfo-data-tools/src/materialise/dependencies.py +++ b/packages/dfo-data-tools/src/materialise/dependencies.py @@ -37,11 +37,12 @@ import hashlib import json +import logging from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, replace from typing import TYPE_CHECKING, cast -from code_index.composites import declared_composite_digest +from code_index.composites import PopulationUnavailable, declared_composite_digest from graph_release.context import bound_release_or_none from materialise.content_version import recipe_dependency_fingerprint from materialise.convert import ConversionRecord @@ -53,6 +54,7 @@ from origin_catalogue.origins import DFO if TYPE_CHECKING: + from code_index.query import CodeIndex from materialise.pipeline import MaterialisationRequest from series_coverage.index import CoverageIndex @@ -72,6 +74,8 @@ # A `latest` member that elected no year, over no facet. _NO_ELECTION = LatestYear(elected=None, served=None) +logger = logging.getLogger(__name__) + @dataclass(frozen=True) class Dependencies: @@ -343,6 +347,7 @@ def refreshed_dependencies( *, graph: GraphReader, coverage: CoverageIndex | None, + code_index: CodeIndex, ) -> Dependencies: """The fingerprint a run over `requests` would record now against the refs and election arguments `stored` names, without materialising. @@ -362,8 +367,12 @@ def refreshed_dependencies( `stored` lacks and so differs in `latest_years`. If it elects none, as a StatVar outside the index or a subject-only request does, it gains none and matches. - Each stored composite takes the digest of the declaration deployed now - (`declared_composite_digest`), `None` for one no longer declared. `imported_data` + Each stored composite takes the digest of the declaration deployed now under + `code_index` (`declared_composite_digest`), `None` for one no longer declared or one + reading a population of destinations `code_index` cannot place + (`code_index.composites.PopulationUnavailable`), which is logged as + `composite_digest_unavailable`. Either differs from the stored digest, so the head + rebuilds and the mint answers the refusal. `imported_data` keeps `stored`'s. The head rule compares it with the bound release's (`datasets_api.current`), and `dependency_changes` never does. """ @@ -409,12 +418,27 @@ def _elected_now(request: MaterialisationRequest) -> LatestYear | None: ), imported_data=stored.imported_data, composites=tuple( - (provenance, declared_composite_digest(provenance)) + (provenance, _composite_digest_now(provenance, code_index=code_index)) for provenance, _ in stored.composites ), ) +def _composite_digest_now(provenance: str, *, code_index: CodeIndex) -> str | None: + """The digest of the composite `provenance` as deployed now under `code_index` + (`declared_composite_digest`), or `None` where it is no longer declared or its population + is unavailable.""" + try: + return declared_composite_digest(provenance, index=code_index) + except PopulationUnavailable as exc: + logger.warning( + "composite_digest_unavailable: %s reads a population that is unavailable; %s", + provenance, + exc.unclassified.message(), + ) + return None + + # The pair fields whose tokens are refresh dates. A current `None` date says nothing about the # source, so it never differs from a stored one (`dependency_changes`). _DATE_FIELDS = ("sources", "conversion_sources") diff --git a/packages/dfo-data-tools/src/materialise/errors.py b/packages/dfo-data-tools/src/materialise/errors.py index 7927dbfd6..c536f277f 100644 --- a/packages/dfo-data-tools/src/materialise/errors.py +++ b/packages/dfo-data-tools/src/materialise/errors.py @@ -177,6 +177,36 @@ def __init__(self, detail: str, *, dataset: str, stat_var: str) -> None: super().__init__(detail, detail=detail, issues=(ProblemIssue(member=stat_var),)) +class DestinationUnclassifiedError(MaterialisationError): + """500 `internal_error`. Raised before any read of a composite whose component binds its + destinations to one side (`code_index.types.ComponentRecipe.destinations`) while that + side's population is unavailable (`code_index.composites.population_check`): the code + index publishes, or coverage holds rows at, a destination no side declaration places, + or coverage cannot list the recipients of a population variable. The graph release and + the code index or the side declarations have drifted apart, and a population leaving + the destination out would understate the figure with no trace. `destinations` holds the + dcids, sorted, `variables` the unlisted variables, and `dataset` the component's + table.""" + + code = "internal_error" + status = 500 + + def __init__( + self, *, destinations: Sequence[str], dataset: str, variables: Sequence[str] = () + ) -> None: + self.destinations = tuple(sorted(destinations)) + self.variables = tuple(variables) + self.dataset = dataset + detail = ( + f"coverage lists no recipients of {dataset} for {len(self.variables)} population " + f"variable(s), so no side's rows can be kept: {list(self.variables)!r}" + if self.variables + else f"{len(self.destinations)} destination(s) of {dataset} sit on no known side " + f"of its flows, so no side's rows can be kept: {list(self.destinations)!r}" + ) + super().__init__(detail, detail=detail) + + class MixedGrainError(MaterialisationError): """500 `internal_error`. Raised when the rows a materialisation serves hold periods of more than one grain, as `2019` beside `2019-01`, which no frame binds diff --git a/packages/dfo-data-tools/src/materialise/facets.py b/packages/dfo-data-tools/src/materialise/facets.py index 0e317e0c4..dbad5aa29 100644 --- a/packages/dfo-data-tools/src/materialise/facets.py +++ b/packages/dfo-data-tools/src/materialise/facets.py @@ -32,7 +32,7 @@ from dc_graph import earliest_date as _earliest from dc_graph import latest_date as _latest -from dc_sdmx import ContainedIn, EntityFilter +from dc_sdmx import Among, ContainedIn, EntityFilter from dfo_platform.lru_memo import LruMemo from dfo_platform.named import cut from dfo_platform.problem import Alternative @@ -411,8 +411,8 @@ def _bound_roles_for_coverage( ) -> BoundRoles | None: """Maps declared graph-role bindings to coverage's `donor`/`recipient` labels. Returns `None` when `stat_var` has no `RoleNames` entry. `ContainedIn` is unbound because - its expansion requires a graph read. This copy keeps `materialise` independent of - `datasets_api`.""" + its expansion requires a graph read; `Among` binds its members. This copy keeps + `materialise` independent of `datasets_api`.""" names = coverage.role_names.get(stat_var) if names is None: return None @@ -421,7 +421,7 @@ def _bound_roles_for_coverage( refs = role_bindings.get(declared_role) if refs is None or isinstance(refs, ContainedIn): continue - bound[role] = frozenset(refs) + bound[role] = frozenset(refs.members if isinstance(refs, Among) else refs) return bound @@ -1183,12 +1183,15 @@ def entity_filter_key(role_bindings: Mapping[str, EntityFilter]) -> tuple[object """Builds a scope key preserving role and filter order. SDMX applies each `c[role]` filter to every variable in a request, so - members with different scopes cannot share one. + members with different scopes cannot share one. An `Among` keys as its members, the + dcids it sends. """ def _filter_key(filt: EntityFilter) -> tuple[object, ...]: if isinstance(filt, ContainedIn): return ("contained_in", filt.parent, filt.child_type) + if isinstance(filt, Among): + return ("dcids", filt.members) return ("dcids", tuple(filt)) return tuple(sorted((role, _filter_key(filt)) for role, filt in role_bindings.items())) @@ -1556,11 +1559,12 @@ def _probe_provenance_refresh( def _flatten_bound_entities(role_bindings: Mapping[str, EntityFilter]) -> tuple[str, ...]: """Returns unique plain dcids in first-seen order. Skips `ContainedIn` because expansion - requires a graph read. Kept local because `generation` imports this module.""" + requires a graph read, and `Among`, whose members no caller named. Kept local because + `generation` imports this module.""" seen: set[str] = set() entities: list[str] = [] for filt in role_bindings.values(): - if isinstance(filt, ContainedIn): + if isinstance(filt, ContainedIn | Among): continue for dcid in filt: if dcid not in seen: diff --git a/packages/dfo-data-tools/src/materialise/finish.py b/packages/dfo-data-tools/src/materialise/finish.py index 894cc70e9..ccd3bf639 100644 --- a/packages/dfo-data-tools/src/materialise/finish.py +++ b/packages/dfo-data-tools/src/materialise/finish.py @@ -11,7 +11,7 @@ from dfo_frame_grammar import FRAME_GRAMMAR_VERSION, frame_sha256 from code_index.types import PartyRow -from dc_sdmx import ContainedIn, EntityFilter +from dc_sdmx import Among, ContainedIn, EntityFilter # `pipeline` imports this module during initialisation. `MaterialisationResult` resolves when # `_finish_member` runs, after `pipeline` has loaded. @@ -828,6 +828,12 @@ def deflator_missing_omitted_warning(omitted: Iterable[tuple[str, int]]) -> Mate def _plain_role_bindings(role_bindings: Mapping[str, EntityFilter]) -> dict[str, Sequence[str]]: - """Returns plain DCID lists for coverage checks. `ContainedIn` scopes require graph - expansion and stay outside this mapping.""" - return {role: filt for role, filt in role_bindings.items() if not isinstance(filt, ContainedIn)} + """Returns the DCID lists a caller named, for coverage checks. `ContainedIn` scopes require + graph expansion and stay outside this mapping, as does `Among`: its members are a + population bound in place of an open role, and an open role names no entity it could + leave unserved.""" + return { + role: filt + for role, filt in role_bindings.items() + if not isinstance(filt, ContainedIn | Among) + } diff --git a/packages/dfo-data-tools/src/materialise/generation.py b/packages/dfo-data-tools/src/materialise/generation.py index a6ead09f6..736e672cd 100644 --- a/packages/dfo-data-tools/src/materialise/generation.py +++ b/packages/dfo-data-tools/src/materialise/generation.py @@ -22,6 +22,7 @@ from dc_sdmx import ( DEFAULT_PLANNING_LANES, + Among, ContainedIn, EntityFilter, SdmxCapExceededError, @@ -3509,12 +3510,13 @@ def _flatten_bound_entities(role_bindings: Mapping[str, EntityFilter]) -> tuple[ """Every plain dcid across every `Sequence[str]`-bound role, deduped (first-seen order). A `ContainedIn`-bound role is skipped entirely (its expansion isn't known upfront), a documented limitation shared with - `probe_facets`' own note. + `probe_facets`' own note. An `Among`-bound role is skipped too: its members are a + population no caller named. """ seen: set[str] = set() entities: list[str] = [] for filt in role_bindings.values(): - if isinstance(filt, ContainedIn): + if isinstance(filt, ContainedIn | Among): continue for dcid in filt: if dcid not in seen: diff --git a/packages/dfo-data-tools/src/materialise/observation_rows.py b/packages/dfo-data-tools/src/materialise/observation_rows.py index b8f8a796a..7f5f98580 100644 --- a/packages/dfo-data-tools/src/materialise/observation_rows.py +++ b/packages/dfo-data-tools/src/materialise/observation_rows.py @@ -25,6 +25,7 @@ from dc_graph import Facet, GraphCapExceededError, GraphDeadlineExceededError, GraphReadError from dc_sdmx import ( + Among, ContainedIn, EntityFilter, SdmxCapExceededError, @@ -139,19 +140,20 @@ def _stream_place_slices( The first slice's stream supplies the roles. Each later slice opens once the one before it is exhausted, names those roles, and gets what is left of `max_rows`.""" subject = entities.get(_SUBJECT_ROLE) + places = subject.members if isinstance(subject, Among) else subject slice_entities: list[Mapping[str, EntityFilter]] = ( [ {**entities, _SUBJECT_ROLE: place_slice} for place_slice in place_slices( variables, role=_SUBJECT_ROLE, - places=subject, + places=places, facet_ids=facet_ids, time_period=time_period, url_budget_bytes=sdmx.url_budget_bytes, ) ] - if subject and not isinstance(subject, ContainedIn) + if places and not isinstance(places, ContainedIn) else [entities] ) @@ -287,7 +289,7 @@ def _subject_places(entities: Mapping[str, EntityFilter]) -> list[str]: "a UN System Data Commons read names its places; a containment filter " f"({subject.parent!r}, {subject.child_type!r}) reached it" ) - return list(dict.fromkeys(subject)) + return list(dict.fromkeys(subject.members if isinstance(subject, Among) else subject)) def _kept_series( diff --git a/pyproject.toml b/pyproject.toml index 16b89eac9..8ddb02d96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,7 +254,8 @@ dfo-analysis-engine = { workspace = true } # The registry refuses a part's `source_code` on search.resolve with # `source_code_not_held`, `source_code_ambiguous`, and # `source_code_shares_variable`. -# The pin is spec-kit 2f7462a, which registers the organisation reading codes, +# The pin is spec-kit 9092fec, which registers `bilateral_reading_unavailable` on +# search.resolve and the organisation reading codes, # defining the tie between them by the role the query's reading states, # and the Presentation limit names # (`LIMIT_NAMES`), publishes `resource_exhausted` as `correct_and_retry`, @@ -325,8 +326,8 @@ dfo-analysis-engine = { workspace = true } # role-keys rule names the frame party columns of each role key. # Follow-up: move all nine to a real spec-kit tag once one is cut, so a future # force-push or GC on spec-kit's dev can't silently break this lock. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/spec-kit" } -dfo-spec-kit-client-contract = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "2f7462a91408a18ff2bf97872fed735a8c052382", subdirectory = "python/client-contract" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/spec-kit" } +dfo-spec-kit-client-contract = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/client-contract" } [tool.pytest.ini_options] testpaths = ["tests", "packages/dfo-analysis-engine/tests"] diff --git a/src/datasets_api/adapter.py b/src/datasets_api/adapter.py index fadd3bf10..b42b8ff3d 100644 --- a/src/datasets_api/adapter.py +++ b/src/datasets_api/adapter.py @@ -27,8 +27,10 @@ from code_index.composites import ( ComponentIdentity, CompositeNotServable, + PopulationUnavailable, component_identities, composite_ref, + destination_population, ) from code_index.members import ( FamilyQuery, @@ -66,11 +68,13 @@ ) from datasets_api.models import CanonicalDefinition from datasets_api.shape import StatVarShape, resolve_shape -from dc_sdmx import EntityFilter +from dc_sdmx import Among, EntityFilter from dfo_platform.named import cut +from dfo_platform.problem import Alternative, ProblemIssue from graph_release.context import bound_release_or_none from materialise import MaterialisationRequest from materialise.errors import ( + DestinationUnclassifiedError, FacetNotAvailableError, OverCeilingError, ) @@ -389,9 +393,13 @@ def to_materialisation_request( Validation covers facet-binding shape only. A shape resolved for a component of a composite (`StatVarShape.component`) reads that - component's identity in place of `identity`, and the request names the component. + component's identity in place of `identity`, and the request names the component. A role + the component binds to a population (`ComponentIdentity.populations`) is sent as + `dc_sdmx.Among` its members, so readers of rows read the members and readers of the names + a caller asked for read the role as open. """ component = shape.component + populations = component.populations if component is not None else {} if component is not None: identity = component.identity entities = cast("Mapping[str, list[str]]", identity.get("entities", {})) @@ -403,7 +411,12 @@ def to_materialisation_request( f"declared_roles {shape.declared_roles!r} no longer declares; " "check_reproducible runs before materialisation and refuses this" ) - role_bindings[declared_role] = list(refs) + population = populations.get(role) + role_bindings[declared_role] = ( + Among(members=tuple(refs), side=population.side) + if population is not None + else list(refs) + ) facets = cast("Mapping[str, Mapping[str, object]]", identity.get("facets", {})) facet_bindings: dict[str, FacetBinding] = {} for axis, binding in facets.items(): @@ -498,13 +511,102 @@ def composite_components( (`code_index.composites.component_identities`), or `None` for a plain identity. Every reader of an identity's provenance pin in this package asks this first. - Raises `CompositeUnservable` where the components cannot be derived.""" + Raises `FacetNotAvailableError` for a composite that reads one side's destinations bound + to a destination off that side (`_off_side_refusal`), `CompositeUnservable` where the + components cannot be derived otherwise, and `DestinationUnclassifiedError` where the + side's population is unavailable.""" try: return component_identities(code_index, identity) + except PopulationUnavailable as exc: + raise DestinationUnclassifiedError( + destinations=exc.unclassified.destinations, dataset=exc.unclassified.dataset + ) from exc except CompositeNotServable as exc: + refusal = _off_side_refusal(exc, code_index=code_index) + if refusal is not None: + raise refusal from exc raise CompositeUnservable(str(exc), identity_pointer=exc.pointer) from exc +def _off_side_refusal( + exc: CompositeNotServable, *, code_index: CodeIndex +) -> FacetNotAvailableError | None: + """The 422 `facet_not_available` for a composite whose component reads one side's + destinations (`code_index.types.ComponentRecipe.destinations`) and whose identity binds + destinations off that side (`CompositeNotServable.key`, `refs`), the client's fault: the + composite holds no rows there. `None` for every other derivation refusal. + + Each such destination is one issue with the fix. The total spanning both sides names the + population's total (`RolePopulation.total`) on the destination role. Any other, a + multilateral agency or the DAC's total of them, names the component's own table on the + provenance axis, which holds the donor's core contributions to it. The refusal points at + the destination role when every such destination is the spanning total, whose fix is + there, and at the provenance facet otherwise.""" + key = exc.key + if key is None: + return None + composite = code_index.composite(exc.ref) + recipe = ( + next((recipe for recipe in composite.components if recipe.destinations is not None), None) + if composite is not None + else None + ) + if recipe is None: + return None + population = destination_population(code_index, recipe.dataset, composite=exc.ref) + off_side = list(exc.refs) + source = population.source + issues: list[ProblemIssue] = [] + for ref in off_side: + if ref == source.spanning: + total = code_index.parties.get(population.total) + issues.append( + ProblemIssue( + axis=key, + member=ref, + code="facet_not_available", + detail=( + f"{ref} counts both sides of a donor's aid; the {population.side} " + f"line holds its {population.side} side at {population.total}" + ), + alternatives=[ + Alternative( + ref=population.total, + label=(total.label if total else None) or population.total, + ) + ], + truncated=False, + ) + ) + continue + table = code_index.datasets.get(source.dataset) + issues.append( + ProblemIssue( + axis=_PROVENANCE_AXIS, + member=ref, + code="facet_not_available", + detail=( + f"{ref} is not a {population.side} destination; {source.dataset} itself " + "counts a donor's core contributions to it" + ), + alternatives=[ + Alternative( + ref=source.dataset, + label=(table.label if table else None) or source.dataset, + ) + ], + truncated=False, + ) + ) + spanning_only = all(ref == source.spanning for ref in off_side) + return FacetNotAvailableError( + f"provenance binds the {population.side} line of {recipe.dataset}, which holds no " + f"rows at {off_side!r}", + facet_pointer=f"/entities/{key}" if spanning_only else f"/facets/{_PROVENANCE_AXIS}", + issues=tuple(issues), + ) + + def identity_composite( identity: Mapping[str, object], *, code_index: CodeIndex ) -> CompositeDeclaration | None: diff --git a/src/datasets_api/admission.py b/src/datasets_api/admission.py index 1ac166b34..dfa7be5ce 100644 --- a/src/datasets_api/admission.py +++ b/src/datasets_api/admission.py @@ -62,7 +62,7 @@ from datasets_api.mint_models import MintOptions from datasets_api.output_options import movable_totals_roles from datasets_api.timings import count, phase -from dc_sdmx import ContainedIn +from dc_sdmx import Among, ContainedIn from graph_release.context import current_release from materialise.errors import OverCeilingError from materialise.formats import EXCEL_MAX_ROWS, ExcelLimitError, check_excel_rows @@ -337,7 +337,7 @@ def _bare_dcid(stat_var: str) -> str: def bound_roles(request: MaterialisationRequest, *, index: CoverageIndex) -> BoundRoles: """Map `request`'s bound declared roles to coverage's donor/recipient labels. - Omit unbound roles and `ContainedIn` filters. + Omit unbound roles and `ContainedIn` filters. An `Among` binds its members. """ names = index.role_names.get(_bare_dcid(request.stat_var)) if names is None: @@ -347,7 +347,7 @@ def bound_roles(request: MaterialisationRequest, *, index: CoverageIndex) -> Bou refs = request.role_bindings.get(declared_role) if refs is None or isinstance(refs, ContainedIn): continue - bound[role] = frozenset(refs) + bound[role] = frozenset(refs.members if isinstance(refs, Among) else refs) return bound diff --git a/src/datasets_api/current.py b/src/datasets_api/current.py index f60412ec8..94a1e4b66 100644 --- a/src/datasets_api/current.py +++ b/src/datasets_api/current.py @@ -1166,17 +1166,20 @@ def _fingerprint_changes( *, graph: GraphReader, coverage: CoverageIndex, + code_index: CodeIndex, ) -> tuple[str, ...]: """The members of `head`'s dependency fingerprint that differ from the one a run over - `requests` would record now, read in one graph call and elected from `coverage` - (`refreshed_dependencies`). A head with no readable fingerprint differs on `schema` and - reads nothing.""" + `requests` would record now, read in one graph call, elected from `coverage` and digested + under `code_index` (`refreshed_dependencies`). A head with no readable fingerprint differs + on `schema` and reads nothing.""" stored = decode_dependencies(head.dependencies) if head.dependencies is not None else None if stored is None: return ("schema",) return dependency_changes( head.dependencies, - refreshed_dependencies(stored, requests, graph=graph, coverage=coverage), + refreshed_dependencies( + stored, requests, graph=graph, coverage=coverage, code_index=code_index + ), ) @@ -1238,7 +1241,9 @@ def _judge_head( resolved = requests() if _catalogue_unkeys(head, resolved): return _Freshness(head=None, rebuild_reason=_IMPORTED_DATA) - changes = _fingerprint_changes(head, resolved, graph=deps.graph, coverage=coverage) + changes = _fingerprint_changes( + head, resolved, graph=deps.graph, coverage=coverage, code_index=deps.code_index + ) if changes: return _Freshness(head=None, rebuild_reason=",".join(changes)) written = deps.head_store.confirm_head( diff --git a/src/datasets_api/derived.py b/src/datasets_api/derived.py index 5b5876bea..ffb2f5f45 100644 --- a/src/datasets_api/derived.py +++ b/src/datasets_api/derived.py @@ -25,7 +25,10 @@ `render_derived_file` writes the frame in the format the file selector names (`doors.files.selector_options`): `csv`, `xlsx` or `parquet`, alone or in a bundle zip beside the Data Package documents a dataset bundle holds (`materialise.bundle.assemble_bundle`), citing the -sources the frame's `provenance` column names from its roots' citations. A derived frame's files +sources the frame's `provenance` column names from its roots' citations. A declared composite +there is cited by its component tables (`code_index.composites.provenance_datasets`), as its +root's citation names them, and named with what it adds or reads +(`materialise.bundle_sources.composite_notes`). A derived frame's files hold the frame's own grammar columns, the frame layout, so `layout` takes only `frame`. Bare Parquet is the frame file the publish stored beside the canonical frame (`render.render_and_store_frame_file`), the file a format 2 ref's `artifact.digest` names, as a @@ -51,6 +54,8 @@ from artifact_store import ArtifactStore, Tier from artifact_store.errors import ArtifactNotFoundError +from code_index.composites import provenance_datasets +from code_index.shared import get_index from dataset_store.clock import Clock from dataset_store.snapshot import ( ManifestFacts, @@ -86,7 +91,7 @@ ) from materialise.bundle_columns import TableScan, column_docs, summary_line from materialise.bundle_naming import bare_file_name, sources_slug, zip_file_name -from materialise.bundle_sources import source_provenance +from materialise.bundle_sources import composite_notes, source_provenance from materialise.formats import ( Rendering, Table, @@ -479,11 +484,14 @@ def render_derived_file( rows, columns = table.num_rows, len(table.columns) record = reads.recipe_reader.read_recipe(spec_id=spec_id, content_version=content_version) citation = recipe_citation(reads.snapshot_reader, record).sources_by_ref() + provenance_refs = _provenance_refs(table) sources = tuple( source_provenance( ref, cited.model_dump(mode="json") if (cited := citation.get(ref)) else None ) - for ref in _provenance_refs(table) + for ref in dict.fromkeys( + dataset for ref in provenance_refs for dataset in provenance_datasets(ref) + ) ) slug = sources_slug([source.name for source in sources]) tag = content_version.removeprefix("ocv_")[:_TAG_LENGTH] @@ -524,6 +532,7 @@ def render_derived_file( ), sources=sources, columns=column_docs(scan, code_system_names={}), + composites=composite_notes(provenance_refs, index=get_index()), ) origin = site.public_site_url documents = assemble_bundle( diff --git a/src/datasets_api/doors/definition.py b/src/datasets_api/doors/definition.py index 985982cad..529913a42 100644 --- a/src/datasets_api/doors/definition.py +++ b/src/datasets_api/doors/definition.py @@ -159,6 +159,7 @@ CollectionProbeCeilingExceededError, DatasetDisagreementError, DefaultFacetUndefinedError, + DestinationUnclassifiedError, FacetBindingUnsupportedError, FacetNotAvailableError, MembersNotHeldError, @@ -810,9 +811,12 @@ async def _definition_members_impl( MemberReadDeadline, CurrencyRoleUndeclared, # A composite identity's components are derived and resolved with its members - # (`adapter.resolve_definition_members`). + # (`adapter.resolve_definition_members`). A composite reading one side's destinations + # refuses while that side's population is unavailable, there and before its run reads + # (`serve._materialise_components`). CompositeUnservable, CompositeWindowUnsupported, + DestinationUnclassifiedError, ) # The failures of a materialisation run, which the definition and options routes make. diff --git a/src/datasets_api/doors/mint_inventory.py b/src/datasets_api/doors/mint_inventory.py index a777f75f5..90f0a2cf9 100644 --- a/src/datasets_api/doors/mint_inventory.py +++ b/src/datasets_api/doors/mint_inventory.py @@ -51,6 +51,7 @@ CollectionProbeCeilingExceededError, DatasetDisagreementError, DefaultFacetUndefinedError, + DestinationUnclassifiedError, FacetBindingUnsupportedError, FacetNotAvailableError, MembersNotHeldError, @@ -112,6 +113,7 @@ OverCeilingError, MembersNotHeldError, DatasetDisagreementError, + DestinationUnclassifiedError, FrameError, CollectionProbeCeilingExceededError, ExcelLimitError, @@ -204,6 +206,7 @@ OverCeilingError("exemplar", ceiling_kind="rows", count=1, cap=1), MembersNotHeldError(stat_vars=("exemplar",)), DatasetDisagreementError("exemplar", dataset="provenance/Exemplar", stat_var="dcid:exemplar"), + DestinationUnclassifiedError(destinations=("Exemplar",), dataset="provenance/Exemplar"), FrameError("exemplar", "exemplar"), CollectionProbeCeilingExceededError("exemplar"), ExcelLimitError("exemplar"), diff --git a/src/datasets_api/kinds/spec_id.py b/src/datasets_api/kinds/spec_id.py index cb2ccd1f2..5d19b7fc4 100644 --- a/src/datasets_api/kinds/spec_id.py +++ b/src/datasets_api/kinds/spec_id.py @@ -36,6 +36,7 @@ from dfo_spec_kit.identity import IdentityError +from code_index.composites import binds_composite from code_index.flow_family import flow_family from code_index.members import canonical_identity from code_index.query import CodeIndex @@ -139,13 +140,17 @@ def _resolve_members( replaces the identity's own before resolution, as scope surgery does, and one that supplies `entities` supplies the role keys the role class is elected under, and the open roles the scope leaves open (`open_roles_left_open`), so the family is the one the surgered - identity resolves. `bounds` bounds the graph walk + identity resolves. A composite identity takes the whole scope before resolution, since + its components bind its entities and window (`adapter.composite_components`), so each + shape's component reads the surgered identity. `bounds` bounds the graph walk (`adapter.resolve_definition_members`). Returns the antichain-minimal shapes. """ if scope is not None and scope.facets is not None: identity = apply_scope_surgery( identity, scope=scope.model_copy(update={"entities": None, "window": None}) ) + if scope is not None and binds_composite(identity): + identity = apply_scope_surgery(identity, scope=scope) try: resolved = resolve_definition_members( identity, diff --git a/src/datasets_api/origin.py b/src/datasets_api/origin.py index 32f9ad282..5ce20c7b5 100644 --- a/src/datasets_api/origin.py +++ b/src/datasets_api/origin.py @@ -61,7 +61,7 @@ from datasets_api.manifest_models import CountedRefsModel, WarningModel from datasets_api.warnings import DATASETS_WARNING_CATALOGUE from dc_graph import GraphCapExceededError, GraphReadError -from dc_sdmx import ContainedIn +from dc_sdmx import Among, ContainedIn from materialise import MaterialisationRequest from materialise.facets import binds_no_place from materialise.finish import MAX_COUNTED_REFS @@ -389,7 +389,8 @@ def _subject_binding(request: MaterialisationRequest) -> tuple[str, list[str]] | or isinstance(bound, ContainedIn) ): return None - return request.stat_var.removeprefix(_DCID_REF_PREFIX), list(dict.fromkeys(bound)) + refs = bound.members if isinstance(bound, Among) else bound + return request.stat_var.removeprefix(_DCID_REF_PREFIX), list(dict.fromkeys(refs)) def _refusal_scopes( diff --git a/src/datasets_api/render.py b/src/datasets_api/render.py index 12dec4ff4..b3aa02ae3 100644 --- a/src/datasets_api/render.py +++ b/src/datasets_api/render.py @@ -91,6 +91,7 @@ Tier, ) from artifact_store.errors import ArtifactIntegrityError +from code_index.composites import DECLARED_COMPOSITES from code_index.labels import PRICES_BY_QUALIFIER, family_label from code_index.members import absent_member_reads_zero from code_index.query import CodeIndex @@ -171,6 +172,7 @@ BundleContext, BundleLinks, CodeEntry, + CompositeNote, MemberProvenance, ReproducingCall, SourceProvenance, @@ -185,11 +187,15 @@ year_window, ) from materialise.bundle_naming import heading, member_slugs, sources_slug -from materialise.bundle_sources import source_provenance +from materialise.bundle_sources import ( + composite_bundle_labels, + composite_notes, + source_provenance, +) from materialise.codes import MaterialisationWarning from materialise.content_version import mint_version_from_digest from materialise.enrichment import PARTY_FIELDS, EnrichmentLookup, pivot_party_cell -from materialise.errors import OverCeilingError +from materialise.errors import MaterialisationUsageError, OverCeilingError from materialise.formats import ( DATA_SHEET_TITLE, FORMAT_COLUMN_CAPS, @@ -1380,9 +1386,10 @@ def _member_provenance( scan: TableScan, code_index: CodeIndex | None, description: str | None, + composites: tuple[CompositeNote, ...], ) -> MemberProvenance: """One bundle member, titled by its sources and years, with each column documented from - `scan`.""" + `scan`, naming the `composites` its rows carry.""" entry = BundleArtifactEntry( path=data_path, bytes=written.size_bytes, @@ -1402,6 +1409,7 @@ def _member_provenance( sources=tuple(sources), columns=column_docs(scan, code_system_names=code_system_names, rendering=rendering), description=description, + composites=composites, ) @@ -1474,6 +1482,7 @@ def _members( code_index: CodeIndex | None, totals: _TotalRows | None, totals_file: _TotalsFile | None, + composites: tuple[CompositeNote, ...] = (), ) -> tuple[MemberProvenance, ...]: """The bundle's members: the data file, then the totals file when the total rows are written as one (`_write_totals_entry`). Each description says where the total rows are, @@ -1502,6 +1511,7 @@ def _members( totals=totals, totals_path=totals_file.path if totals_file is not None else None, ), + composites=composites, ) if totals_file is None: return (main,) @@ -1521,6 +1531,7 @@ def _members( "figures, such as a regional total. Adding these rows to those of " f"{data_path} counts those figures twice." ), + composites=composites, ), ) @@ -1572,6 +1583,39 @@ def _table_scan( ) +def _refuse_composite_without_index( + provenance_refs: Sequence[str], *, code_index: CodeIndex | None +) -> None: + """Raise `MaterialisationUsageError` when `provenance_refs` name a declared composite + (`code_index.composites.DECLARED_COMPOSITES`) and there is no `code_index` to name and + describe it from.""" + if code_index is not None: + return + composites = sorted({ref for ref in provenance_refs if ref in DECLARED_COMPOSITES}) + if composites: + raise MaterialisationUsageError( + f"a render of the composites {composites!r} needs the code index to name them" + ) + + +def _known_labels( + citations: Mapping[str, CitationSource], + family: tuple[str, str] | None, + *, + code_index: CodeIndex | None, +) -> dict[str, str]: + """The names a render gives outright: each cited source's label; each declared + composite's bundle name (`materialise.bundle_sources.composite_bundle_labels`), which + neither a citation nor the graph holds, so its `provenance` cells and codes-table row + name it apart from its tables; and a collapsed family's `(ref, label)`.""" + known = {ref: cited.label for ref, cited in citations.items()} + if code_index is not None: + known |= composite_bundle_labels(code_index) + if family is not None: + known[family[0]] = family[1] + return known + + def _codes( scans: Sequence[TableScan], *, @@ -1585,9 +1629,7 @@ def _codes( name its `_label` cell holds. The graph names every other code through the render's `reads`, so the read shares the render's deadline and makes none after a failed name read.""" - known = {ref: cited.label for ref, cited in citations.items()} - if family is not None: - known[family[0]] = family[1] + known = _known_labels(citations, family, code_index=code_index) if code_index is not None: for scan in scans: for column in scan.axis_kinds: @@ -1815,10 +1857,9 @@ def for_render( citations: Mapping[str, CitationSource], family: tuple[str, str] | None, deadline: float | None, + code_index: CodeIndex | None, ) -> _RenderReads: - known = {ref: cited.label for ref, cited in citations.items()} - if family is not None: - known[family[0]] = family[1] + known = _known_labels(citations, family, code_index=code_index) return cls( enrichment=enrichment, parties=parties, @@ -2967,8 +3008,14 @@ def render_and_store( and mints no version. `options` keep the version's minting options, so they still name the rendering, the reproducing call and the bundle's description, and a collapse's family ref keeps its label. + + A composite's rows are named and described from its declaration over the code index + (`materialise.bundle_sources.composite_notes`), so a render of a frame whose rows carry a + declared composite raises `MaterialisationUsageError` without `code_index`, which would + write its bundle with no word of the composite. """ check_options_supported(options, code_index=code_index) + _refuse_composite_without_index(provenance_refs, code_index=code_index) parties = code_index.parties if code_index is not None else {} cited_refs = list(citation_refs) if citation_refs is not None else provenance_refs subject_id = str(spec.get("spec_id")) @@ -3005,6 +3052,7 @@ def render_and_store( name_reads = _RenderReads.for_render( enrichment=enrichment, parties=parties, + code_index=code_index, citations=citations, family=( (f"{_STAT_VAR_REF_PREFIX}{subject_id}", subject_label) @@ -3154,6 +3202,11 @@ def render_and_store( code_index=code_index, totals=totals, totals_file=totals_file, + composites=( + () + if code_index is None + else composite_notes(provenance_refs, index=code_index) + ), ) files = assemble_bundle( spec=spec, diff --git a/src/datasets_api/serve.py b/src/datasets_api/serve.py index 73149bb56..717bae47e 100644 --- a/src/datasets_api/serve.py +++ b/src/datasets_api/serve.py @@ -64,7 +64,12 @@ from artifact_store import ArtifactStore, ArtifactStoreError from artifact_store.inventory import InventoryBatch, InventoryEntry -from code_index.composites import ComponentIdentity, composite_ref +from code_index.composites import ( + ComponentIdentity, + composite_digest, + composite_ref, + population_check, +) from code_index.members import absent_member_reads_zero from code_index.query import CodeIndex from code_index.types import CompositeDeclaration, PartyRow @@ -169,7 +174,11 @@ table_years, ) from materialise.continued_series import ContinuedSeries, continued_series_of_trace -from materialise.errors import DefaultFacetUndefinedError, MaterialisationUsageError +from materialise.errors import ( + DefaultFacetUndefinedError, + DestinationUnclassifiedError, + MaterialisationUsageError, +) from materialise.graph import GraphReader from materialise.profile import OutputProfile, compute_output_profile_from_chunks from materialise.provenance_doc import ( @@ -697,12 +706,21 @@ def _materialise_components( origin findings are read only for a version not on record (`_component_findings`). A component no request names, a spliced one whose periods the window does not reach, is left out. Raises `MaterialisationUsageError` without `coverage_index`, which a composite's - years are read from.""" + years are read from, and `DestinationUnclassifiedError` before any read when a component + reads a population `coverage_index` holds a destination outside of, or cannot list the + recipients of (`code_index.composites.population_check`).""" if coverage_index is None: raise MaterialisationUsageError( "a composite reads its components' years from the coverage snapshot" ) ref = cast(str, composite_ref(stored.identity)) + unavailable = population_check(code_index, coverage_index).refusal(ref) + if unavailable is not None: + raise DestinationUnclassifiedError( + destinations=unavailable.destinations, + dataset=unavailable.dataset, + variables=unavailable.unlisted, + ) composite = cast("CompositeDeclaration", code_index.composite(ref)) covered = { component.role: table_years(code_index, coverage_index, component.identity) @@ -749,6 +767,7 @@ def _materialise_components( role_columns=requests[0].declared_roles, enrichment=enrichment, parties=code_index.parties, + digest=composite_digest(composite, index=code_index), ) check_ceiling(composed.coverage, ceiling=ceiling) return replace(composed, output_profile_seconds=seconds), tuple(runs) diff --git a/src/datasets_api/warning_backfill.py b/src/datasets_api/warning_backfill.py index 82d67c7de..b76f2d426 100644 --- a/src/datasets_api/warning_backfill.py +++ b/src/datasets_api/warning_backfill.py @@ -121,6 +121,8 @@ from materialise.errors import ( CollectionProbeCeilingExceededError, DatasetDisagreementError, + DestinationUnclassifiedError, + FacetNotAvailableError, MaterialisationUsageError, ) from materialise.facets import ( @@ -998,6 +1000,17 @@ def _probe( raise _Refused( f"the identity is refused: {type(exc).__name__}: {exc}", reason="identity_refused" ) from exc + except FacetNotAvailableError as exc: + # A composite bound to a destination off its component's side holds no rows there. + raise _Refused( + f"the composite holds no rows at the bound destination: {exc}", + reason="facet_not_available", + ) from exc + except DestinationUnclassifiedError as exc: + raise _Refused( + f"the composite's destination population is unavailable: {exc}", + reason="destination_unclassified", + ) from exc try: with ( definition_read_scope(identity, code_index=readers.code_index), diff --git a/src/presentation_store/frame_store.py b/src/presentation_store/frame_store.py index c381b5b99..80bde50f4 100644 --- a/src/presentation_store/frame_store.py +++ b/src/presentation_store/frame_store.py @@ -199,6 +199,7 @@ RollingAnchor, StructuralView, ) +from code_index.composites import provenance_datasets from code_index.labels import family_label from code_index.query import CodeIndex from code_index.shared import get_index, get_measure_index @@ -1721,10 +1722,12 @@ def _derived_citation( deadline: float, ) -> tuple[Mapping[str, object], dict[str, SourceDocumentFacts], bool]: """`stored_manifest` with the citation of a derived frame's distinct `provenance` refs, - in row order, shaped as a dataset version manifest's `citation`, each source with its - documentation and the text written for the `retrieved` date; what each source's document - says; and whether every read behind them completed. The line routes through the DFO as - the steps its description record knows (`steps`) decide + in row order, a declared composite's cited by its component tables + (`code_index.composites.provenance_datasets`) as a version of it is, shaped as a dataset + version manifest's `citation`, each source with its documentation and the text written + for the `retrieved` date; what each source's document says; and whether every read + behind them completed. The line routes through the DFO as the steps its description + record knows (`steps`) decide (`description.citation.figures_processed`). The read covers the first `_CITATION_READ_CAP` refs. With no graph, and for refs beyond the cap, each source is named by its ref alone. A failed or overdue citation read leaves the citation out and @@ -1737,7 +1740,14 @@ def _derived_citation( if PROVENANCE_COLUMN in columns: index = columns.index(PROVENANCE_COLUMN) rows = cast("Sequence[Sequence[object]]", doc["rows"]) - refs = list(dict.fromkeys(cast(str, row[index]) for row in rows if row[index])) + refs = list( + dict.fromkeys( + dataset + for row in rows + if row[index] + for dataset in provenance_datasets(cast(str, row[index])) + ) + ) live: dict[str, SourceCitation] = {} if self._graph is not None and refs: read = self._source_citations( diff --git a/src/search/README.md b/src/search/README.md index 0be60ca35..71a716f91 100644 --- a/src/search/README.md +++ b/src/search/README.md @@ -193,10 +193,28 @@ A part starts its pass with the scope its request posts (`stated_scope.part_scop The pass's terminal builds the scope in memory from the records, the release's code index and the request's coverage snapshot (`pipeline._with_scope_choices`). A part with no record carries none. Only the entries the binders applied compile (`AxisRecord.entry`), while the answer's identity holds the option the binder settled (`scope.build.build_part_scope`), so every choice states the option the answer serves. The provider axis is settled over the answer's identity once the binders have run (`scope.provider.settle_provider`). A stated or carried entry applies where its option is available, or for a stated entry partial, and the answer can serve it: the scope it holds, the inclusive composite, or the bilateral scope, which removes that composite (`scope.edits.applies_in_pass`). Otherwise the declared convention applies where the answer can serve it, else the scope the answer holds, else the bilateral scope, else the first available option, so the record holds only an option it marks available. A choice names the convention's rule only where the request matched the convention and the part holds the convention's scope. Otherwise it names the rule of the option it holds (`scope.tables.AxisTable.rules`), which describes that option's own scope: on a part the CRS serves, `bilateral_flows` for the bilateral scope and `bilateral_plus_imputed` for the inclusive one (`code_index.scope_conventions.PROVIDER_SCOPE_RULES`), and on a published total the row's rule. A resubmit matches no convention, so it names the option's own rule. `code_index.scope_conventions.PROVIDER_CONVENTIONS` counts a named donor's total of a CRS sector or recipient part as bilateral flows plus the imputed multilateral share when both tables cover every year the bilateral answer holds. Every convention reads the CRS, so a published donor total matches none and holds the provider scope its DAC1 aid type declares (`code_index.scope_conventions.PUBLISHED_PROVIDER_SCOPES`), under the row's rule, where every provider the part names is a donor: a country, the EU institutions (`DONOR_ORGANISATIONS`), or a total of a group whose membership holds countries alone (the G7, the DAC countries). A part naming any other organisation or total, the DAC's total of the multilaterals among them, or no provider, has no provider record. DAC1's total ODA counts bilateral flows plus core contributions to multilaterals in the year they are paid, so it holds `provider:contributions`, labelled "bilateral plus core contributions to multilaterals", under the rule `contribution_total`. Its `provider:bilateral` option binds DAC1's bilateral ODA (aid type 1015), judged per donor on its basis. A resubmit judges each explicit entry on the identity the other explicit entries' edits produce (`scope.build._requested_tables`), so a request switching to gross and to bilateral flows together judges bilateral ODA on gross and compiles as the two switches do one after the other. A year the total holds is covered when every donor holding the total in that year holds bilateral ODA in it, so a donor holding no total in a year places no demand on it. Where no donor holds the total in any year of the window, the option is available if every donor holds bilateral ODA in the window and unavailable otherwise. Otherwise the option is available where every year the total holds is covered, partial over the covered years where those are fewer, with the latest run of them as the window a switch reads (`SlotBinding.window`), and unavailable where none is, each with `option_not_covered`. On the OECD headline a year counts on the basis the headline reads it on, net flows before 2018 and grant equivalents from 2018. A `latest` read elects one year for the whole pool, so on a part asking for the latest year the option elects bilateral ODA's year among those every donor holding the total in its elected year holds it in, and a switch reads that one year. The option is available where that year is the total's elected year, partial over it, stated in the label, where it is another, and unavailable where the donors share no year or no donor holds the total in its elected year. One coverage scan per table reads every donor's years. Its `provider:inclusive` option is unavailable with `contributions_counted`, since it counts the contributions in place of an imputed share of the multilaterals' spending, with the total's dataset in `detail`. DAC1's bilateral ODA holds `provider:bilateral` under the rule `bilateral_total`, offers `provider:contributions` as total ODA under the same coverage check, and its inclusive option is not offered (`provider_not_offered`, why `dataset`). A pool naming the EU institutions beside an EU member state, or beside a total of a group holding one, has no provider record, since total ODA counts the member's core contributions to the institutions and the institutions' own aid. The members are those the membership policy holds, on any day of the part's window (today for a latest read), for the group `REDISTRIBUTED_PROVIDERS` declares as the institutions' funders, so the United Kingdom counts over any window holding a year before it left in 2020. Where the policy holds no members for that group, every pool with the institutions counts. A pool of the EU institutions and a donor outside the EU has a provider record. A fresh search answers the total it resolved, so a stated "bilateral" on a DAC1 total is reported in `dropped_entries` as `stated_choice_not_applied` with the cause `not_served`, and a switch to it reads bilateral ODA. A stated "including multilateral" is reported with `contributions_counted`. Words naming multilateral ODA state `provider:multilateral` only on a part whose provider table offers it, DAC1's. Elsewhere, as on a part the CRS or DAC2a serves, the entry is dropped before the axis settles (`scope.provider.without_unoffered_multilateral`), so it is neither applied nor reported. The other entries that attach to the part, those on an axis its binders settled and, of those made for a concept, the ones for the concept the axis was asked for, and the stated ones on an axis the part lacks (`ScopeState.attached`), are reported in `dropped_entries`. A carried one is reported as `carried_choice_not_applied` and a stated one as `stated_choice_not_applied`, with the cause (`entry_on_axis`, `axis_absent`, the option's partial or unavailable reason, or `not_served`). -A composite option (the inclusive provider scope, or the OECD headline per year for a donor window crossing 2018) binds a declared composite provenance as the identity's source (`code_index.composites`). When the part settles one, the scope's base is the composite, and the answer is restated as it (`scope.composite.assemble_composite`). The restated answer carries the composite's spec_id and identity, its coverage and headline over the years its components serve, its StatVar counts summed over the components, its provenance facet naming the composite, and `compiled.components` with each component's role, table, years, StatVar count and identity. Its `download` facts count the file a Datasets mint of the composite writes, from the coverage snapshot in memory (`scope.composite.composite_download`), with `rows_exact` false, since the count reads no facet the mint elects. Its observation preview reads each component identity with `resolve_spec_resubmit`, which elects the component's members as the mint does, in worker threads that run together, and composes the previews under the composite's rules (`scope.composite.compose_observations`). Each component keeps the points in the years it serves, under its own StatVar and `source_refs`, and the event or embedded preview carries the composite's spec_id. A streamed request reads it in a trailing task launched once the answer is restated (`region.composite_answer.CompositeRelaunch`), so the `response` event waits on nothing new, and a buffered request reads it inline. The answer phase decides the answer's scope from the identity the Spec will hold before any preview read launches (`region.scope_decision.decide_scope`, over `spec_build.answer_identity`), and the restatement reuses that decision, so a fresh composite answer reads no preview of its table identity, streamed or buffered. A dataset_resubmit resolves the composite's table identity with no preview and reads each component through the part memo. A composite whose components serve no point omits its preview with `observations_no_points`, or the first reason a component's own preview omits with, or `observations_probe_failed` when a component answered nothing. A component read that fails upstream, crosses the deadline or answers nothing leaves its years out and warns `observations_probe_failed`, or `observations_timed_out` for the deadline, naming the component's role, so the answer stands. Each component reads through the request's catalogue under its own pin alone, as the mint reads it, whatever source scope the part read under. A same-shape merge previews the read its merged Spec makes: composite answers, which merge only with each other, answer the composite over the merged identity (`region.composite_answer.as_merged_composite`), whose spec_id, components, years, counts, download facts and preview cover every member's entities and slots, and any other merge reads its preview from the read plan the merge rebuilds in memory, the plan its download counts, as a single answer reads its own, or, where the merge rebuilds no plan, with one resubmit of its merged identity. The inclusive option serves the years both tables cover, read from the coverage snapshot in memory. Over a window the imputed table covers in part it is `partial`, with the latest run of covered years as its own window (`ProviderBinding.window`), which a switch or the request's own words apply. A stated inclusive scope ("including imputed multilateral") is then served over that run, the scope's base keeps the years asked for, so a switch to bilateral answers them, and the part carries a `year_unavailable` departure on `when:`: "Imputed multilateral aid covers only 2011 to 2024, so this shows those years." A carried entry on a partial option is dropped, since only the user may change the period asked for. For a `latest` window it covers the latest year both tables cover: the scope's base keeps the `latest` window, and the composite compiles to that one year (`provider.elect_latest_window`), so a switch to bilateral answers the CRS's own latest year. An option a part cannot serve is not applied as its default, and a donor with no grant-equivalent series keeps net flows. Every Search reader of a composite reads the identity its option was set on (`code_index.composites.table_identity`). A fresh search does not refuse its scope. Parts a same-shape merge folds state the scope of the merged identity, built as a resubmit of it builds it from the union of the members' entries (`pipeline.merged_scope_choices`). To switch an option, post the scope's `base.identity` as a `dataset_resubmit` identity entry with its `entries` as `scope_entries`, the entry on the switched axis replaced by the switch (an explicit entry whose origin names the choice), so each axis holds one entry. The engine compiles them over the identity, resolves the result, and refuses `requested_option_unavailable` or `conflicting_requested_options` (422) at the scope entry it cannot apply. A sector part's whole, the same dataset over every sector, is asked the same way: post `base.identity` with the slot its sector options bind (`SlotBinding.slot`) set `unbound`, and as `scope_entries` an explicit entry from each other choice holding its active option. The whole then keeps the part's donor, recipient, basis, prices, window and provider scope, or refuses `requested_option_unavailable` at the entry holding an option it cannot serve, such as the inclusive scope for a donor the imputed table carries no rows for. A part whose scope states no sector choice binds no sector and has no whole. +A composite option (the inclusive provider scope, or the OECD headline per year for a donor window crossing 2018) binds a declared composite provenance as the identity's source (`code_index.composites`). When the part settles one, the scope's base is the composite, and the answer is restated as it (`scope.composite.assemble_composite`). The restated answer carries the composite's spec_id and identity, its coverage and headline over the years its components serve, its StatVar counts summed over the components, its provenance facet naming the composite, and `compiled.components` with each component's role, table, years, StatVar count and identity. Its `download` facts count the file a Datasets mint of the composite writes, from the coverage snapshot in memory (`scope.composite.composite_download`), with `rows_exact` false, since the count reads no facet the mint elects. Its observation preview reads each component identity with `resolve_spec_resubmit`, which elects the component's members as the mint does, in worker threads that run together, and composes the previews under the composite's rules (`scope.composite.compose_observations`). Each component keeps the points in the years it serves, under its own StatVar and `source_refs`, and the event or embedded preview carries the composite's spec_id. A streamed request reads it in a trailing task launched once the answer is restated (`region.composite_answer.CompositeRelaunch`), so the `response` event waits on nothing new, and a buffered request reads it inline. The answer phase decides the answer's scope from the identity the Spec will hold before any preview read launches (`region.scope_decision.decide_scope`, over `spec_build.answer_identity`), and the restatement reuses that decision, so a fresh composite answer reads no preview of its table identity, streamed or buffered. A dataset_resubmit resolves the composite's table identity with no preview and reads each component through the part memo. A composite whose components serve no point omits its preview with `observations_no_points`, or the first reason a component's own preview omits with, or `observations_probe_failed` when a component answered nothing. A component read that fails upstream, crosses the deadline or answers nothing leaves its years out and warns `observations_probe_failed`, or `observations_timed_out` for the deadline, naming the component's role, so the answer stands. Each component reads through the request's catalogue under its own pin alone, as the mint reads it, whatever source scope the part read under. A same-shape merge previews the read its merged Spec makes: composite answers, which merge only with each other, answer the composite over the merged identity (`region.composite_answer.as_merged_composite`), whose spec_id, components, years, counts, download facts and preview cover every member's entities and slots, and any other merge reads its preview from the read plan the merge rebuilds in memory, the plan its download counts, as a single answer reads its own, or, where the merge rebuilds no plan, with one resubmit of its merged identity. The inclusive option serves the years both tables cover, read from the coverage snapshot in memory. Over a window the imputed table covers in part it is `partial`, with the latest run of covered years as its own window (`ProviderBinding.window`), which a switch or the request's own words apply. A stated inclusive scope ("including imputed multilateral") is then served over that run, the scope's base keeps the years asked for, so a switch to bilateral answers them, and the part carries a `year_unavailable` departure on `when:`: "Imputed multilateral aid covers only 2011 to 2024, so this shows those years." A carried entry on a partial option is dropped, since only the user may change the period asked for. For a `latest` window it covers the latest year both tables cover: the scope's base keeps the `latest` window, and the composite compiles to that one year (`provider.elect_latest_window`), so a switch to bilateral answers the CRS's own latest year. An option a part cannot serve is not applied as its default, and a donor with no grant-equivalent series keeps net flows. Search elects and resolves a composite over the identity its option was set on, with its first component's population bound (`code_index.composites.table_identity`). That read holds the first component's rows alone, so the counts of what the composite serves read every component. A fresh search does not refuse its scope. Parts a same-shape merge folds state the scope of the merged identity, built as a resubmit of it builds it from the union of the members' entries (`pipeline.merged_scope_choices`). To switch an option, post the scope's `base.identity` as a `dataset_resubmit` identity entry with its `entries` as `scope_entries`, the entry on the switched axis replaced by the switch (an explicit entry whose origin names the choice), so each axis holds one entry. The engine compiles them over the identity, resolves the result, and refuses `requested_option_unavailable` or `conflicting_requested_options` (422) at the scope entry it cannot apply. A sector part's whole, the same dataset over every sector, is asked the same way: post `base.identity` with the slot its sector options bind (`SlotBinding.slot`) set `unbound`, and as `scope_entries` an explicit entry from each other choice holding its active option. The whole then keeps the part's donor, recipient, basis, prices, window and provider scope, or refuses `requested_option_unavailable` at the entry holding an option it cannot serve, such as the inclusive scope for a donor the imputed table carries no rows for. A part whose scope states no sector choice binds no sector and has no whole. On the SSE stream, every per-variable event (`shape`, `sentence`, `step`) carries `variable_index`, the same index the terminal `response` event's `parts` list is keyed by. Trailing per-spec events (`slot_options`, `entity_options`, `facet_options`, `observations`) join on `spec_id` instead, since a same-shape collapse can put two parts on the same spec. `sentence` events stay keyed by `variable_index`, one per part; the composed envelope sentence only ever rides the terminal `response` event, alongside the rest of the envelope. A part whose shape carries a measure family bind elected gets a first `sentence` event right after bind, naming only its measured thing from that election ("gross disbursements"), without the flow type and the year, followed by a second `sentence` event once materialise resolves the served rows, which adds both and refines that phrase when the served measure family differs from what bind elected. A part with no elected family (most indicators outside the CRS-style financial-flow axes) gets only the post-materialise `sentence` event, because materialise resolves its measured thing from the served StatVar. +#### Provider scopes + +The provider axis offers up to three readings of a donor's aid, each a `provider:` option on the part's scope. + +- `provider:contributions` is the published total ODA: bilateral flows plus the donor's core contributions to multilateral organisations. On DAC1 it is aid type 1010, under the rule `contribution_total`. On DAC2a it is aid type 206 over every destination, under `contribution_recipient_total`, where a destination that is a multilateral agency carries the donor's core contribution to it, labelled as total ODA. A part holding a DAC2a composite whose named destinations are all on the bilateral side is not offered it (`provider_not_offered`, why `destination_bilateral`), since those destinations hold no core contribution and its figures equal the bilateral ones. +- `provider:bilateral` is the donor's bilateral flows alone. On DAC1 it is aid type 1015, under `bilateral_total`. On DAC2a it is the composite `ONEDAC2aBilateral`: aid type 206 over the bilateral population, under `bilateral_destinations`. The answer's measure noun and its family label in Datasets and Presentation read "(bilateral flows)", its sentence and citation name "OECD DAC2a", and its label counts the recipients the file holds. +- `provider:inclusive` is bilateral flows plus the donor's imputed share of multilateral spending. On the CRS it is the composite of the CRS and the imputed multilateral sectors table. On DAC2a it is the composite `ONEDAC2aBilateralAndImputedMultilateral`: total ODA plus DAC2a's imputed multilateral ODA, both over the bilateral population, so a core contribution is not counted once as a flow and again as an imputed share, and neither line holds a row at `DACAllRecipients`. DAC1 offers no inclusive scope. + +A DAC2a composite reads its destinations as a role open within a population (`code_index.scope_conventions.RolePopulation`). A destination is placed on a side by `code_index.scope_conventions.destination_side`: countries, the organisations the DAC counts as bilateral recipients, the declared regional, income and DAC group totals and the DAC's unallocated lines are `bilateral`, multilateral agencies, the DAC's unnamed multilateral recipients and `DACAllMultilateralRecipients` are `multilateral`, and `DACAllRecipients` is `spanning`. The population's `members` are the bilateral-side parties and residuals, its `named` set adds the bilateral totals, and its `total` is `DACDevelopingCountries`. `code_index.composites._derived_identity` binds it where a component's identity derives, so every reader of rows reads the bound list from the identity and `ComponentIdentity.populations` tags the role for the few readers of names. An open destination binds the members, a named one passes when it is in `named` and refuses with `CompositeNotServable` otherwise, and an identity naming no destination binds the total. + +The mint sends a bound population as `dc_sdmx.Among`, so the frame, the coverage, the row ceiling and the manifest count the population alone. Search discovers an open destination within it (`discover_role_scope(within=...)`), and the options door offers no agency and no total of both sides. A fresh answer that settles on the composite is read again within the population, or at the population's total where its identity names no destination (`pipeline._within_population`), so its scope, headline, coverage and warnings equal a resubmit's of its definition. That read supplies the figures, members and warnings alone: the answer keeps the identity, spec_id and scope decided on the request's own identity, so "top donors of ODA" names no destination and keeps its scope. A candidate set's readings are read again concurrently, every part's reads together within the request's concurrency limit (`concurrency.bounded_thread_permits`). A population that holds a destination no declaration places, in the code index or in the request's coverage snapshot, is unavailable: the mint refuses the composite as a server fault, and Search offers neither cut reading (`provider_not_offered`, why `destination_unclassified`) and keeps total ODA. A posted identity binding the composite then refuses 422 `requested_option_unavailable` at the resubmit and candidate doors and `facet_not_available` at the options door, cause `provider_not_offered` naming the unplaced destinations, with the plain DAC2a table as the alternative. + +A fresh DAC2a total settles on `provider:bilateral` when it names no destination or only bilateral-side ones and the composite serves every year its answer holds (`provider.bilateral_default_reading`), with or without a provider axis, so a ranking over all donors lists no agency. The scope's base is then the composite, so a switch of any other axis posted over the base keeps the bilateral reading. The composite replaces a provenance pin to DAC2a, so a part pinned to DAC2a and switched to total ODA from the composite reads the unpinned total ODA spec, with the same figures as the pinned one. It holds `provider:contributions` when a stated choice or the part's own destination asks for it. A part whose destination is a multilateral agency reads total ODA: its bilateral and inclusive options are unavailable with `multilateral_destination`, the destination in `detail`, and the answer departs `core_contribution_read`, worded for an agency, for `DACAllMultilateralRecipients` or for the DAC's lines of unnamed agencies. The departure reads the destination's side, so a part naming no donor, which has no provider axis, carries it too. A part bound to `DACAllRecipients` holds total ODA by default; under the bilateral or inclusive scope its destination rebinds to `DACDevelopingCountries`, the bilateral side of all recipients, and the option label says so. A destination no declaration places leaves both options `provider_not_offered`, why `destination_unclassified`. A fresh part with no provider axis that keeps total ODA because its bilateral option is not available warns `bilateral_reading_unavailable`, with the option's reason in `params.why`, since no option is there to say so. A part whose scope decision raised states no scope, so where the default reaches it it carries the warning with `why` `scope_undecided` (`scope_decision.guarded_scope_decision`). + +A posted identity that binds either DAC2a composite to an agency or to `DACAllRecipients` is refused at Search's doors that read a posted identity's table (`identity_edit.composite_table_identity`), 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `multilateral_destination`, with the plain DAC2a table as the alternative: a resubmit settles nothing. An identity that names no destination reads `DACDevelopingCountries`. `resolved.scope_choices.compiled.components[].identity` carries the bound list of members. + +The CRS publishes no core contributions, so its rows are bilateral already and `provider:bilateral` there is the CRS alone. + #### Editing an identity `kind: "dataset_resubmit"` with one `identity` entry posts a resolved identity back, edited or not. This is the shape a single-identity edit takes: `entries` carries one entry, and the envelope's one part is that identity re-resolved. `identity` is the complete definition. The engine derives the identity's StatVar members as the Datasets mint does, from the release's code index when it decides the identity, else from the mint's graph walk, which reads the whole filtered set up to `SEARCH_RESUBMIT_SUBJECT_CEILING` (20,000) subjects when its first page truncates without proof. A family holding a StatVar the index lists outside its code systems, a World Bank IDS series, is elected from the index's rows of its five-tuple in place of that walk, with no graph read (`code_index.members.listed_group`). A walk that runs that full read extends the request's deadline to `SEARCH_RESUBMIT_FULL_READ_TIMEOUT_S` (90 s) from its start. A family past the ceiling or the member cap answers an unsupported part, `scope_too_broad`. Reposting a Spec's `definition.identity` unchanged returns its `spec_id` again, because `spec_id` hashes only the identity core (`shape`, `slots`, `entities`, `open_roles`, `facets`, `requested_window`). Editing a bound entity role, here `identity.entities.observationAbout`, mints a new `spec_id` bound to the edited place: diff --git a/src/search/catalogue/counts.py b/src/search/catalogue/counts.py index 7109ccf38..168a40d4c 100644 --- a/src/search/catalogue/counts.py +++ b/src/search/catalogue/counts.py @@ -33,6 +33,7 @@ import numpy as np import pyarrow as pa +from code_index.multilaterals import COUNTRY_PREFIX from code_index.types import PartyRow from origin_catalogue.catalogue import CatalogueSnapshot, entry_ref from search.catalogue.artifact import AidCounts, Counts, PlaceCounts, YearRange @@ -45,8 +46,6 @@ LAST_YEAR_COLUMN, ) -COUNTRY_PREFIX: Final[str] = "country/" - _ROLES: Final[tuple[Role, ...]] = ("donor", "recipient") diff --git a/src/search/engine/app.py b/src/search/engine/app.py index edcef3ff6..3c4edef41 100644 --- a/src/search/engine/app.py +++ b/src/search/engine/app.py @@ -554,6 +554,7 @@ async def list_options(body: OptionsRequest, deps: SearchDepsDep) -> Response: materialise_graph=materialise_graph, population_memo=deps.population_memo, deadline=deadline, + coverage=coverage_snapshot, ), deadline=deadline, ) diff --git a/src/search/engine/clients/materialise_sdmx_cache.py b/src/search/engine/clients/materialise_sdmx_cache.py index f120c1c0e..fdd418b82 100644 --- a/src/search/engine/clients/materialise_sdmx_cache.py +++ b/src/search/engine/clients/materialise_sdmx_cache.py @@ -24,12 +24,25 @@ import httpx import pyarrow as pa -from dc_sdmx import ContainedIn, EntityFilter, SdmxClient +from dc_sdmx import Among, ContainedIn, EntityFilter, SdmxClient from graph_generation import UNKNOWN_GENERATION, GenerationProvider from graph_generation.stamp import ttu_from_stamp_key from search.engine.heap_weight import heap_bytes type _EntitiesKey = tuple[tuple[str, tuple[str, ...] | ContainedIn], ...] + + +def _entities_key(dcids: EntityFilter) -> tuple[str, ...] | ContainedIn: + """One role's binding as a read key: a containment filter itself, and the dcids a list or + a population (`Among.members`) reads, so a population and a list naming its members + read, and key, the same rows.""" + if isinstance(dcids, ContainedIn): + return dcids + if isinstance(dcids, Among): + return dcids.members + return tuple(dcids) + + type _ReadKey = tuple[ str, tuple[str, ...], @@ -146,10 +159,7 @@ def stream_data_batches( key: _ReadKey = ( self._generation.stamp().key, tuple(variables), - tuple( - (role, dcids if isinstance(dcids, ContainedIn) else tuple(dcids)) - for role, dcids in entities.items() - ), + tuple((role, _entities_key(dcids)) for role, dcids in entities.items()), None if facet_ids is None else tuple(facet_ids), time_period if time_period is None or isinstance(time_period, str) diff --git a/src/search/engine/clients/sdmx.py b/src/search/engine/clients/sdmx.py index 9b244b656..348c1cbd2 100644 --- a/src/search/engine/clients/sdmx.py +++ b/src/search/engine/clients/sdmx.py @@ -28,6 +28,7 @@ import search.engine.config as config # module-attribute access for runtime-resolved values from dc_sdmx import ( + Among, ContainedIn, EntityFilter, SdmxClient, @@ -324,7 +325,7 @@ def availability_values( Filters can name roles, ``observationAbout``, ``unit``, ``provenance``, or ``measurementMethod``. Role values follow the ``role_series`` dcid-sequence rules, or are a ``ContainedIn`` - containment filter. A bare ``time_period`` string + containment filter or an ``Among`` population, sent as its members. A bare ``time_period`` string selects one exact period. A sequence is an exact-period OR-list. The endpoint serves exact periods. ``LATEST`` and ranges raise ``RoleNotDeclaredError``. Caps raise ``SdmxCapExceededError``. @@ -654,7 +655,7 @@ def availability_values( assert_no_un_variables(method="availability_values", variables=variables) filters_arg = ( { - role: filt if isinstance(filt, ContainedIn) else list(filt) + role: filt if isinstance(filt, ContainedIn | Among) else list(filt) for role, filt in filters.items() } if filters diff --git a/src/search/engine/codes.py b/src/search/engine/codes.py index e235d2777..edbab704e 100644 --- a/src/search/engine/codes.py +++ b/src/search/engine/codes.py @@ -323,6 +323,9 @@ # An answer lists each member of a flow side whose dataset publishes the side's # whole total (region/codes.py). "whole_total_available", + # A fresh DAC2a part with no provider axis reads total ODA where the bilateral + # default could not apply (region/codes.py). + "bilateral_reading_unavailable", # A replayed selection collapsed to one definite candidate. "candidates_collapsed_definite", # A replayed selection has no surviving candidates. diff --git a/src/search/engine/concurrency.py b/src/search/engine/concurrency.py index 27b1544c0..310e791f9 100644 --- a/src/search/engine/concurrency.py +++ b/src/search/engine/concurrency.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio +import threading from collections.abc import AsyncIterator, Coroutine from contextlib import asynccontextmanager from contextvars import ContextVar @@ -31,13 +32,16 @@ class RequestScope: combined work stays under the configured limit. ``create_owned_task`` registers returned tasks. Completion removes them. Scope exit cancels and awaits every unclaimed task. ``claim_withheld_read`` grants the request's one - ``withheld_by_registry`` re-read. + ``withheld_by_registry`` re-read. ``thread_permits`` bounds, at the same limit, the + reads the request's worker threads run at once where they cannot take a permit of the + semaphore, since the part leg that starts them holds one. """ - __slots__ = ("_withheld_read_claimed", "semaphore", "tasks") + __slots__ = ("_withheld_read_claimed", "semaphore", "tasks", "thread_permits") def __init__(self) -> None: self.semaphore = asyncio.Semaphore(settings().max_variable_concurrency) + self.thread_permits = threading.BoundedSemaphore(settings().max_variable_concurrency) self.tasks: set[asyncio.Task[Any]] = set() self._withheld_read_claimed = False @@ -92,6 +96,12 @@ def bounded_semaphore() -> asyncio.Semaphore: return current_scope().semaphore +def bounded_thread_permits() -> threading.BoundedSemaphore: + """Return the current request's limit on reads its worker threads run at once + (`RequestScope.thread_permits`).""" + return current_scope().thread_permits + + def create_owned_task[T](coro: Coroutine[Any, Any, T]) -> asyncio.Task[T]: """Schedule and register a coroutine for a helper to return. diff --git a/src/search/engine/departures.py b/src/search/engine/departures.py index c83835ef4..87bb1e01b 100644 --- a/src/search/engine/departures.py +++ b/src/search/engine/departures.py @@ -22,7 +22,8 @@ the Spec the part answers with and the operations it carries (`spec_departure_facts`). A no-data answer for years its series holds no point in carries `year_unavailable` (`window_gap_fact`), and a no-data answer whose check the engine narrowed carries -`scope_limited` (`scope_limited_fact`). +`scope_limited` (`scope_limited_fact`). An answer reading DAC2a's total ODA at a multilateral +recipient carries `core_contribution_read` (`_core_contribution_facts`). A part states one `entity_unavailable` per role whichever reads raised it (`merged_entity_facts`), and only the facts that tell its reader something its heading and @@ -36,9 +37,11 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Literal -from code_index.composites import provenance_datasets +from code_index.composites import binds_composite, provenance_datasets +from code_index.scope_conventions import destination_side from code_index.shared import get_index from dfo_platform.roles import role_key_axis +from search.engine.retrieval.bound_values import identity_dataset from search.heading.departure_fact import ( DepartureFact, PartyReason, @@ -69,6 +72,7 @@ Warning, role_ref, ) +from search.scope.edits import counted_bilateral_source from search.scope.region import is_residual from unit_words.currency_nouns import currency_noun @@ -331,6 +335,7 @@ def spec_departure_facts( facts.append(left_out) facts.extend(year_facts) facts.extend(_scope_window_facts(spec)) + facts.extend(_core_contribution_facts(spec)) if modelled := _modelled_fact(spec): facts.append(modelled) for role, group in (spec.resolved.entity_groups or {}).items(): @@ -933,6 +938,44 @@ def _scope_window_facts(spec: Spec) -> list[DepartureFact]: return facts +def _core_contribution_facts(spec: Spec) -> list[DepartureFact]: + """`core_contribution_read` for each destination the answer names on the multilateral + side (`code_index.scope_conventions.destination_side`) of a published total that + declares a bilateral source (`code_index.scope_conventions.BILATERAL_SOURCES`), DAC2a's + total ODA, read on its own dataset: its figures at that destination are the donor's core + contributions to it. Read from the destinations themselves, so it holds with or without + a provider axis, as for every donor's ODA to one agency, and a resubmit states it too. + `shown` is the recipient's name in the code index, and `recipient` its kind as a + destination of that dataset: an organisation, a total or a residual. An answer binding a + composite, or a provenance set naming one (`code_index.composites.binds_composite`), reads + no published total, so it carries none.""" + identity = spec.definition.identity + key = next((key for key in identity.entities if role_key_axis(key) == "destination"), None) + if key is None: + return [] + if binds_composite(identity.model_dump(mode="json", include={"facets"})): + return [] + source = counted_bilateral_source(identity, dataset_of=identity_dataset) + if source is None: + return [] + parties = get_index().parties + datasets = (source.dataset,) + facts: list[DepartureFact] = [] + for dcid in identity.entities[key]: + party = parties.get(dcid) + if party is None or destination_side(party, datasets=datasets) != "multilateral": + continue + facts.append( + DepartureFact( + code="core_contribution_read", + binding_ref=role_ref(key), + shown=party.label or dcid, + recipient=party.kind_over("destination", datasets) or party.kind, + ) + ) + return facts + + def window_gap_fact(gap: WindowGap | None) -> DepartureFact | None: """`year_unavailable` for a `no_observations` answer whose elected result holds data at the place but no point in the years read (`Materialised.window_gap`): the years asked, diff --git a/src/search/engine/entry/dataset_entries.py b/src/search/engine/entry/dataset_entries.py index f59e74eb0..0b6258873 100644 --- a/src/search/engine/entry/dataset_entries.py +++ b/src/search/engine/entry/dataset_entries.py @@ -21,6 +21,7 @@ from entity_resolution import Admission, EntityResolver from materialise import GraphReader from materialise.rate_tables import RateTableCache +from origin_catalogue import current_catalogue from search.engine.binders import Binder from search.engine.clients.graph import EngineGraphClient from search.engine.clients.llm import SupportsLLM @@ -34,12 +35,15 @@ ) from search.engine.errors import EngineInputError from search.engine.fanout import _FanOutResult, _run_variables_streaming +from search.engine.identity_edit import composite_table_identity from search.engine.options.population_memo import OptionsPopulationMemo from search.engine.region import RegionResult, resolve_spec_resubmit +from search.engine.region.composite_answer import ComponentReads, as_composite_answer from search.engine.region.place_options import PlaceOptionsPool from search.engine.region.result import any_degraded from search.engine.region.state import EngineOptions from search.engine.region.whole_totals import with_containing_total +from search.engine.spec_id import identity_spec_id from search.models import ( BaseIdentity, Entry, @@ -50,7 +54,7 @@ ResubmitParsedEntry, Warning, ) -from search.scope.provider import without_uncounted_inclusive +from search.scope.provider import elect_latest_window, without_uncounted_inclusive from search.scope.tables import ScopeFacts from series_coverage.shared import CoverageSnapshot @@ -278,14 +282,46 @@ async def resolve_candidates_entry( One semaphore bounds all promotes across up to 6 entries with 6 candidates each. Candidate edits skip observations and coverage probes. The caller checks the configured candidate cap before scheduling this coroutine. + + A candidate posting a composite, as a fresh candidate set's reading of DAC2a's bilateral + destinations is (`composite_answer.with_bilateral_readings`), resolves its table identity + with the populations its roles bind (`identity_edit.composite_table_identity`) and + answers the composite (`composite_answer.as_composite_answer`), over the one year it + serves for a `latest` window (`scope.provider.elect_latest_window`), as an identity entry + posting it does. """ + reads = ComponentReads( + graph=graph, + sdmx=sdmx, + coverage=coverage, + materialise_graph=materialise_graph, + materialise_sdmx=materialise_sdmx, + rate_cache=rate_cache, + deadline=deadline, + catalogue=current_catalogue(), + ) + facts = ScopeFacts(index=get_index(), coverage=coverage) def _promote(candidate: ResubmitCandidate) -> RegionResult: + # A composite its components cannot serve refuses before it is served, as an + # identity entry posting it does (`dataset_resubmit`). + table = composite_table_identity( + candidate.identity, pointer="/input/identity", coverage=coverage + ) # A candidate carries no scope entries, so the carried entry a served bilateral line # would report has no scope to state it. - identity, _ = served_posted_identity(candidate.identity, None, coverage=coverage) - return resolve_spec_resubmit( - inp=ResubmitIdentityPayload(identity=identity, entity_groups=candidate.entity_groups), + posted, _ = served_posted_identity(candidate.identity, None, coverage=coverage) + identity = elect_latest_window(posted, facts) + # The table identity of the posted one stands where the window election and the + # served bilateral line leave it unchanged. + if identity != candidate.identity: + table = composite_table_identity(identity, pointer="/input/identity", coverage=coverage) + region = resolve_spec_resubmit( + inp=ResubmitIdentityPayload( + identity=table.identity, + entity_groups=candidate.entity_groups, + populations=dict(table.populations), + ), graph=graph, sdmx=sdmx, coverage=coverage, @@ -299,6 +335,15 @@ def _promote(candidate: ResubmitCandidate) -> RegionResult: ), full_read_deadline=full_read_deadline, ) + if table.identity == identity or region.status != "definite": + return region + # A candidate reads no preview, so the composite's is not read either. + return dataclasses.replace( + as_composite_answer( + region, identity, identity_spec_id(identity), facts=facts, reads=reads + ), + observations_relaunch=None, + ) async def _promote_one(candidate_index: int, candidate: ResubmitCandidate) -> RegionResult: async with sem: diff --git a/src/search/engine/entry/dataset_resubmit.py b/src/search/engine/entry/dataset_resubmit.py index 054d95d46..c045aa510 100644 --- a/src/search/engine/entry/dataset_resubmit.py +++ b/src/search/engine/entry/dataset_resubmit.py @@ -112,15 +112,13 @@ from search.engine.region import RegionResult, resolve_spec_resubmit from search.engine.region.composite_answer import ( ComponentReads, - CompositeRelaunch, - composite_served, + as_composite_answer, with_composite_observations, ) from search.engine.region.place_options import PlaceOptionsPool from search.engine.region.promote_relaunch import bind_relaunches from search.engine.region.result import ( any_degraded, - with_composite_answer, with_runnable_operations, ) from search.engine.region.sentence_events import answer_events, response_sentence_events @@ -145,7 +143,6 @@ from search.scope.basis import identity_basis_record from search.scope.build import build_part_scope, refusal_error from search.scope.compile import Refusal -from search.scope.composite import assemble_composite from search.scope.provider import elect_latest_window from search.scope.region import region_groups from search.scope.tables import PartFacts, ScopeFacts @@ -234,7 +231,7 @@ def _as_composite( reads: ComponentReads, ) -> RegionResult: """`region`, resolved for composite `identity`'s table identity, answering the composite - (`scope.composite.assemble_composite`) under its own spec_id, which is the compiled + (`composite_answer.as_composite_answer`) under its own spec_id, which is the compiled scope's when `scope` compiled it. It carries the closure that reads the composite's preview per component over `reads` (`composite_answer.CompositeRelaunch`), over the components it was restated over (`composite_served`), which `_launch_trailing_tasks` @@ -242,13 +239,7 @@ def _as_composite( The closure reads the composite's identity alone, so a table identity the part memo held answers with one too.""" spec_id = scope.compiled.spec_id if scope is not None else identity_spec_id(identity) - served = composite_served(identity, reads=reads) - spec, _ = assemble_composite(region.spec, identity, spec_id, served=served, facts=facts) - return with_composite_answer( - region, - spec, - observations_relaunch=CompositeRelaunch(spec=spec, reads=reads, served=served), - ) + return as_composite_answer(region, identity, spec_id, facts=facts, reads=reads) def _served_scope(scope: PartScope | None, region: RegionResult) -> PartScope | None: @@ -446,7 +437,10 @@ def _promote(index: int, entry: ResubmitIdentityEntry) -> _Promoted: identity = ( elect_latest_window(entry.identity, facts) if scope is None else scope.compiled.identity ) - read = composite_table_identity(identity, pointer=f"/input/entries/{index}/identity") + table = composite_table_identity( + identity, pointer=f"/input/entries/{index}/identity", coverage=params.coverage + ) + read = table.identity # A composite's preview reads its components (`_as_composite`), so the table identity # it is resolved for reads none. read_options = ( @@ -463,7 +457,9 @@ def _promote(index: int, entry: ResubmitIdentityEntry) -> _Promoted: payload = ( spec_resubmit_input_for_identity(entry) if scope is None and read is identity - else ResubmitIdentityPayload(identity=read, entity_groups=groups) + else ResubmitIdentityPayload( + identity=read, entity_groups=groups, populations=dict(table.populations) + ) ) key = memo.key(payload, read_options) if memo is not None and memo.active else None stored = memo.lookup(key) if memo is not None and key is not None else None @@ -502,7 +498,9 @@ def _promote(index: int, entry: ResubmitIdentityEntry) -> _Promoted: async def _resolve_one(index: int, entry: ResubmitIdentityEntry) -> RegionResult: # A composite its components cannot serve refuses before its scope compiles. - composite_table_identity(entry.identity, pointer=f"/input/entries/{index}/identity") + composite_table_identity( + entry.identity, pointer=f"/input/entries/{index}/identity", coverage=params.coverage + ) async with sem: promoted = await asyncio.to_thread(_promote, index, entry) if promoted.stored: diff --git a/src/search/engine/identity_edit.py b/src/search/engine/identity_edit.py index f44a507ed..6fa7907a3 100644 --- a/src/search/engine/identity_edit.py +++ b/src/search/engine/identity_edit.py @@ -9,11 +9,25 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import cast from dfo_spec_kit import identity as _identity -from code_index.composites import CompositeNotServable, composite_ref, table_identity +from code_index.composites import ( + DECLARED_COMPOSITES, + CompositeNotServable, + PopulationUnavailable, + UnclassifiedDestinations, + composite_ref, + destination_population, + population_check, + table_identity, +) +from code_index.query import CodeIndex +from code_index.scope_conventions import RolePopulation from code_index.shared import get_index +from dfo_platform import Alternative, Cause from dfo_platform.named import cut from dfo_platform.roles import role_key_axis from search.engine.config import settings @@ -23,7 +37,18 @@ IdentityBindingExact, IdentityShape, IdentityShapePin, + RoleAxis, + ScopeReasonCode, ) +from series_coverage.shared import CoverageSnapshot, get_coverage_service + +_MULTILATERAL_DESTINATION: ScopeReasonCode = "multilateral_destination" +"""The cause a composite posted with a destination off its population's side refuses +under (`composite_table_identity`).""" +_PROVIDER_NOT_OFFERED: ScopeReasonCode = "provider_not_offered" +"""The cause a composite whose population is unavailable refuses under +(`composite_table_identity`), as its scope option reads (`scope.provider._bilateral_option`, +why `destination_unclassified`).""" def check_no_exact_bindings(identity: BaseIdentity, *, pointer: str) -> None: @@ -93,6 +118,38 @@ def open_requested_role(identity: BaseIdentity, *, role: str) -> BaseIdentity: ) +def population_members( + populations: Mapping[str, RolePopulation], +) -> dict[RoleAxis, frozenset[str]]: + """The members each role `populations` binds is open within, by the flow axis its key + names (`RolePopulation.members`).""" + return { + axis: frozenset(population.members) + for key, population in populations.items() + if (axis := role_key_axis(key)) is not None + } + + +def open_within( + identity: BaseIdentity, populations: Mapping[str, RolePopulation] +) -> tuple[BaseIdentity, dict[RoleAxis, frozenset[str]]]: + """`identity` with each `entities` key `populations` lists moved to `open_roles`, and the + members each such role is open within, by flow axis: a role bound to a population + (`TableRead.populations`) read as the open role the caller posted, over the parties an + open read of the composite reads. `identity` itself, and no members, when `populations` + lists none of its keys.""" + keys = [key for key in identity.entities if key in populations] + if not keys: + return identity, {} + within = population_members({key: populations[key] for key in keys}) + document = identity.model_dump(exclude_none=True) + document["entities"] = { + key: dcids for key, dcids in identity.entities.items() if key not in populations + } + document["open_roles"] = sorted({*(identity.open_roles or ()), *within}) + return BaseIdentity.model_validate(document), within + + def refuse_over_place_bound( entities: Mapping[str, Sequence[str]], *, pointer: str = "/input/identity/entities" ) -> None: @@ -122,21 +179,152 @@ def refuse_over_place_bound( ) -def composite_table_identity(identity: BaseIdentity, *, pointer: str) -> BaseIdentity: +@dataclass(frozen=True) +class TableRead: + """The identity a Search door reads for a posted identity (`composite_table_identity`). + + `populations` maps each `entities` key of `identity` that binds a population in place of + names the caller posted (`code_index.composites.TableIdentity.populations`): a reader of + the caller's names reads such a role as open within the population's members. `sides` + holds the population of each flow axis the composite reads on one side alone + (`ComponentRecipe.destinations`), whether the posted identity names that axis, opens it + or leaves it out, so an options read of the axis offers that side's destinations alone. + Both are empty for an identity that binds no such composite.""" + + identity: BaseIdentity + populations: Mapping[str, RolePopulation] = field(default_factory=dict) + sides: Mapping[RoleAxis, RolePopulation] = field(default_factory=dict) + + +def composite_table_identity( + identity: BaseIdentity, *, pointer: str, coverage: CoverageSnapshot | None +) -> TableRead: """The identity a Search door reads for `identity`. For a composite (`code_index.composites`) that is the identity its option was set on (`code_index.composites.table_identity`), whose one table every graph and index read - names, and otherwise `identity` itself. Refuses a composite whose components cannot be derived - (`CompositeNotServable`) as `request_invalid`, at `pointer`, the posted identity, joined - with the member at fault.""" + names, with the populations its roles bind, and otherwise `identity` itself. + + Refuses a composite that reads a population of destinations that is unavailable, in the + code index (`code_index.composites.PopulationUnavailable`) or in `coverage`, the + request's coverage snapshot, or the release's where the door holds none + (`code_index.composites.population_check`), with 422 `requested_option_unavailable`, + cause `provider_not_offered`, the destinations no side declaration places, at `pointer` + joined with the provenance facet; the alternative names the dataset whose total ODA reads + every destination (`_population_unavailable`), as a fresh search reads it then. Refuses a + composite that reads one side of its destinations and is posted naming a destination off + that side, an agency or the total spanning both sides, with 422 + `requested_option_unavailable`, cause `multilateral_destination`, at `pointer`, the posted + identity, joined with the role at fault; the alternative names the dataset whose total + ODA reads the destination (`_multilateral_destination`). Refuses any other composite whose + components cannot be derived (`CompositeNotServable`) as `request_invalid`, at `pointer` + joined with the member at fault.""" document = identity.model_dump(mode="json", exclude_none=True) + index = get_index() try: - read = table_identity(get_index(), document) + table = table_identity(index, document) + except PopulationUnavailable as exc: + raise _population_unavailable(index, exc.ref, exc.unclassified, pointer=pointer) from exc except CompositeNotServable as exc: - raise EngineInputError( - str(exc), code="request_invalid", pointer=f"{pointer}{exc.pointer}" - ) from exc - return identity if read is document else BaseIdentity.model_validate(read) + refused = _multilateral_destination(index, document, exc, pointer=pointer) + raise (refused or _request_invalid(exc, pointer=pointer)) from exc + if table.identity is document: + return TableRead(identity=identity) + ref = cast(str, composite_ref(document)) + if any(recipe.destinations is not None for recipe in DECLARED_COMPOSITES[ref].components): + held = (coverage or get_coverage_service().snapshot()).index + if (unclassified := population_check(index, held).refusal(ref)) is not None: + raise _population_unavailable(index, ref, unclassified, pointer=pointer) + return TableRead( + identity=BaseIdentity.model_validate(table.identity), + populations=table.populations, + sides=_sides(index, document), + ) + + +def population_refusal(exc: PopulationUnavailable, *, pointer: str) -> EngineInputError: + """The refusal of a posted composite whose population is unavailable in the code index + (`exc`), at `pointer` joined with the provenance facet, as `composite_table_identity` + refuses it, for a reader of the composite's table that runs before or without that + check.""" + return _population_unavailable(get_index(), exc.ref, exc.unclassified, pointer=pointer) + + +def _population_unavailable( + index: CodeIndex, ref: str, unclassified: UnclassifiedDestinations, *, pointer: str +) -> EngineInputError: + """The refusal of composite `ref`, whose population of `unclassified.dataset`'s + destinations is unavailable: the side declarations have drifted from the graph, so the + composite is not offered (`provider_not_offered`, why `destination_unclassified`), and + the dataset's own total ODA, which reads every destination, is the alternative.""" + dataset = unclassified.dataset + label = index.datasets[dataset].label if dataset in index.datasets else dataset + destinations = list(unclassified.destinations) + return EngineInputError( + f"composite {ref!r} is not offered (provider_not_offered, why " + f"destination_unclassified): {unclassified.message()}; total ODA to every " + f"destination reads from {label}: post this identity with facets.provenance bound to " + f"{dataset!r} in place of {ref!r}", + code="requested_option_unavailable", + status=422, + pointer=f"{pointer}/facets/provenance", + alternatives=[Alternative(ref=dataset, label=label)], + cause=Cause( + code=_PROVIDER_NOT_OFFERED, + entities=destinations[:10], + truncated=len(destinations) > 10, + ), + ) + + +def _request_invalid(exc: CompositeNotServable, *, pointer: str) -> EngineInputError: + return EngineInputError(str(exc), code="request_invalid", pointer=f"{pointer}{exc.pointer}") + + +def _sides(index: CodeIndex, document: Mapping[str, object]) -> dict[RoleAxis, RolePopulation]: + """The population of each flow axis the composite `document` binds reads on one side + alone: its first component's destinations (`ComponentRecipe.destinations`), the + component `table_identity` reads. Empty for a document binding no such composite.""" + ref = composite_ref(document) + if ref is None: + return {} + first = DECLARED_COMPOSITES[ref].components[0] + if first.destinations is None: + return {} + return {"destination": destination_population(index, first.dataset, composite=ref)} + + +def _multilateral_destination( + index: CodeIndex, document: Mapping[str, object], exc: CompositeNotServable, *, pointer: str +) -> EngineInputError | None: + """The refusal of composite `document` naming destinations off the side its population + reads (`RolePopulation.named`): agencies, or the total spanning both sides. Its message + and alternative name the dataset whose total ODA reads them, the population's own + (`BilateralSource.dataset`), which the identity can bind in place of the composite. + `None` when `exc` refuses anything else.""" + key = exc.key + if key is None: + return None + population = _sides(index, document).get("destination") + if population is None: + return None + dataset = population.source.dataset + refused = list(exc.refs) + label = index.datasets[dataset].label if dataset in index.datasets else dataset + return EngineInputError( + f"composite {exc.ref!r} reads {population.side} destinations alone, and " + f"{', '.join(refused)} {'is' if len(refused) == 1 else 'are'} not on that side; total ODA " + f"to {'it' if len(refused) == 1 else 'them'} reads from {label}: post this identity " + f"with facets.provenance bound to {dataset!r} in place of {exc.ref!r}", + code="requested_option_unavailable", + status=422, + pointer=f"{pointer}/entities/{key}", + alternatives=[Alternative(ref=dataset, label=label)], + cause=Cause( + code=_MULTILATERAL_DESTINATION, + entities=refused[:10], + truncated=len(refused) > 10, + ), + ) def check_not_composite(identity: BaseIdentity, *, pointer: str) -> None: diff --git a/src/search/engine/multilaterals.py b/src/search/engine/multilaterals.py index 68921b89c..ad38d2043 100644 --- a/src/search/engine/multilaterals.py +++ b/src/search/engine/multilaterals.py @@ -8,54 +8,18 @@ `RoleAxisFact.holds_multilaterals`). Each bound party's kind and class come with its marker (`party_facts`). -A multilateral is an official party the graph types `Organization`: the organisation records -the entity resolver exports from the graph (`entity_resolution.pack.organisation_dcids`), less -the private providers, which the code index classes `private` (`PartyRow.party_class`). The -EU institutions count, since they spend their member states' contributions -(`code_index.scope_conventions.REDISTRIBUTED_PROVIDERS`). +The classifier itself (`party_facts`, `is_multilateral`) lives in `code_index.multilaterals`, +where the mint reads it too. """ from __future__ import annotations from collections.abc import Collection, Iterable, Mapping -from typing import NamedTuple +from code_index.multilaterals import is_multilateral, party_facts from code_index.query import CodeIndex from code_index.shared import per_index -from code_index.types import PartyClass, PartyKind, PartyRow, RoleAxis -from entity_resolution.pack import organisation_dcids - - -class PartyFacts(NamedTuple): - """What one party is where some datasets publish it on one axis: its kind - (`PartyRow.kind_over`), its class (`PartyRow.class_of`), and whether it is a multilateral, - each `None` where those datasets give it different kinds.""" - - kind: PartyKind | None - party_class: PartyClass | None - multilateral: bool | None - - -def party_facts(row: PartyRow, *, axis: RoleAxis, datasets: Collection[str] = ()) -> PartyFacts: - """The facts of `row` where `datasets` publish it on `axis`. A multilateral is an - organisation that is an official party there; a total, such as the World Bank Group beside - its members in DAC2a, is none.""" - kind = row.kind_over(axis, datasets) - if kind is None: - return PartyFacts(None, None, None) - party_class = row.class_of(kind) - multilateral = ( - kind == "party" and party_class == "official" and row.dcid in organisation_dcids() - ) - return PartyFacts(kind, party_class, multilateral) - - -def is_multilateral(row: PartyRow | None, *, datasets: Collection[str] = ()) -> bool: - """Whether `row` is a multilateral where `datasets` publish it as a donor, on the origin - axis (`party_facts`).""" - return ( - row is not None and party_facts(row, axis="origin", datasets=datasets).multilateral is True - ) +from code_index.types import PartyRow, RoleAxis def holds_multilaterals( diff --git a/src/search/engine/options/options_response_cache.py b/src/search/engine/options/options_response_cache.py index 2d68bac29..8888c1ae7 100644 --- a/src/search/engine/options/options_response_cache.py +++ b/src/search/engine/options/options_response_cache.py @@ -20,8 +20,9 @@ (so a Kenya-anchored ask and a Ghana-anchored ask -- both Country -- share one entry) or the single bound anchor's own dcid for a `children` ask (which has no anchor-invariant answer, unlike peers); for a directional role -(`origin`/`destination`), the opposite axis's own bound-entity set -(`options.surface._directional_opposite_bound`'s own return value) -- reused +(`origin`/`destination`), the coverage roles its read lists under, the +opposite axis's own bound-entity set and any population a composite reads +(`options.surface._directional_bound`'s own return value) -- reused rather than re-derived, since it is already the exact fact `_directional_options_set`'s own coverage lookup filters by. @@ -82,7 +83,7 @@ from search.engine.options.surface import ( _DEFAULT_PEER_TYPE, ComputedOptions, - _directional_opposite_bound, + _directional_bound, _entities_with_role, ) from search.models import OptionsRequest @@ -118,9 +119,10 @@ def _observation_about_key_component(req: OptionsRequest, target: ResolvedOption def _directional_key_component( target: ResolvedOptionsTarget, *, axis: RoleAxis ) -> dict[str, list[str]] | None: - """The `origin`/`destination` slice of the cache key: the opposite - axis's own bound-entity set, exactly as `options.surface. - _directional_opposite_bound` computes it for `_directional_options_set`'s + """The `origin`/`destination` slice of the cache key: the coverage roles + the read lists under, the opposite axis's own bound-entity set and any + population a composite reads, exactly as `options.surface. + _directional_bound` computes it for `_directional_options_set`'s own `series_coverage.CoverageIndex.open_axis` lookup -- reused rather than re-derived so this can never drift from what the surface half actually filters by. Keyed by coverage's own "donor"/"recipient" label @@ -131,11 +133,10 @@ def _directional_key_component( reads. Every dcid set is sorted here: two requests naming the same opposite-axis dcids in a different order denote the identical filter and must share one key, mirroring the identity core's own - order-invariant entity buckets. None (no component) when the opposite - axis carries no bound entity, matching `_directional_opposite_bound`'s - own empty-mapping case. + order-invariant entity buckets. None (no component) when the read lists + under no role, matching `_directional_bound`'s own empty-mapping case. """ - bound = _directional_opposite_bound(target, axis=axis) + bound = _directional_bound(target, axis=axis) if not bound: return None return {coverage_role: sorted(dcids) for coverage_role, dcids in bound.items()} diff --git a/src/search/engine/options/resolve.py b/src/search/engine/options/resolve.py index 5cfe04655..0ed4343fa 100644 --- a/src/search/engine/options/resolve.py +++ b/src/search/engine/options/resolve.py @@ -88,10 +88,12 @@ from __future__ import annotations -from dataclasses import dataclass +from collections.abc import Mapping +from dataclasses import dataclass, field from typing import cast from code_index.flow_family import flow_family +from code_index.scope_conventions import RolePopulation from code_index.shared import get_index from dfo_platform import Alternative from dfo_platform.named import cut, named_capped @@ -103,6 +105,7 @@ check_described_arm, composite_table_identity, open_requested_role, + open_within, refuse_over_place_bound, validate_edited_identity, ) @@ -124,6 +127,7 @@ ResolvedEntity, ) from search.role_kinds import graph_types +from series_coverage.shared import CoverageSnapshot def _typed_anchors( @@ -248,6 +252,9 @@ class ResolvedOptionsTarget: `node_types_batch` call. `facets` and `requested_window` pass through the identity, with the window translated to engine form. `role_names_by_axis` maps declared flow axes to role names; it is empty for subject-only data. + `sides` holds the population of each flow axis a posted composite reads on one side + alone, and `within` the members each role it binds to a population is open within + (`identity_edit.TableRead`); both are empty for any other identity. """ identity: BaseIdentity @@ -256,6 +263,8 @@ class ResolvedOptionsTarget: facets: dict[str, IdentityFacetBinding] | None requested_window: RequestedWindow | None role_names_by_axis: dict[RoleAxis, str] + sides: Mapping[RoleAxis, RolePopulation] = field(default_factory=dict) + within: Mapping[RoleAxis, frozenset[str]] = field(default_factory=dict) def resolve_options_target( @@ -266,10 +275,13 @@ def resolve_options_target( population_memo: OptionsPopulationMemo | None = None, deadline: float | None = None, join_inflight: bool = True, + coverage: CoverageSnapshot | None = None, ) -> ResolvedOptionsTarget: """Resolve the posted request into the target used by the options surface. - `materialise_graph` is required for graph-backed population derivation. + `materialise_graph` is required for graph-backed population derivation. `coverage` is + the request's coverage snapshot, which a posted composite's population is checked + against (`composite_table_identity`), the release's where None. Args: population_memo: optional per-surface population memo; `None` derives @@ -289,7 +301,10 @@ def resolve_options_target( or at `/identity/entities`; `facet_not_available` (422) at `/identity/facets/provenance` for an identity naming a dataset that does not publish its five-tuple (`refuse_source_outside_family`), before any - read; every refusal + read, and for a composite reading one side of its destinations posted with a + destination off it, or reading a population that is unavailable, with the + cause `composite_table_identity` names (`multilateral_destination`, + `provider_not_offered`); every refusal `options.population.options_population` raises propagates unchanged (`population_empty`, `scope_too_broad`, `role_signature_incompatible`, 422 each); `invalid_role_ref` @@ -322,7 +337,24 @@ def resolve_options_target( validate_edited_identity(identity, pointer="/identity") # A composite's options are those of the identity its option was set on, whose one # table every read names. - identity = composite_table_identity(identity, pointer="/identity") + # A role the composite binds to a population is the open role the caller posted, read + # within the population's members. + try: + table = composite_table_identity(identity, pointer="/identity", coverage=coverage) + except EngineInputError as exc: + if exc.code != "requested_option_unavailable": + raise + # The options door registers no scope refusal: a composite that cannot read the + # posted destinations, or its population, is a facet binding it cannot serve. + raise EngineInputError( + str(exc), + code="facet_not_available", + status=422, + pointer=exc.pointer, + alternatives=exc.alternatives, + cause=exc.cause, + ) from exc + identity, within = open_within(table.identity, table.populations) refuse_over_place_bound(identity.entities, pointer="/identity/entities") # A pin posted bound and one posted unbound denote one population. identity = canonicalise_identity(identity) @@ -378,4 +410,6 @@ def resolve_options_target( facets=identity.facets, requested_window=_requested_window_from_identity(identity.requested_window), role_names_by_axis=role_names_by_axis, + sides=table.sides, + within=within, ) diff --git a/src/search/engine/options/surface.py b/src/search/engine/options/surface.py index 195ed5941..140f4f6e9 100644 --- a/src/search/engine/options/surface.py +++ b/src/search/engine/options/surface.py @@ -716,6 +716,24 @@ def _directional_opposite_bound(target: ResolvedOptionsTarget, *, axis: RoleAxis return {_COVERAGE_ROLE_FOR_AXIS[other_axis]: frozenset(other_dcids)} +def _directional_bound(target: ResolvedOptionsTarget, *, axis: RoleAxis) -> BoundRoles: + """The coverage roles an options read of `axis` lists under: the opposite axis's bound + entities (`_directional_opposite_bound`), or the members it is open within where a posted + composite binds it to a population (`ResolvedOptionsTarget.within`), and, where the + composite reads `axis` on one side alone (`ResolvedOptionsTarget.sides`), every + destination that side may name (`RolePopulation.named`). So a composite counting + bilateral flows offers no agency and no total spanning both sides as a destination, and + offers as donors those holding rows at the destinations it reads.""" + bound = dict(_directional_opposite_bound(target, axis=axis)) + other_axis = _OTHER_FLOW_AXIS[axis] + other_role = _COVERAGE_ROLE_FOR_AXIS[other_axis] + if other_role not in bound and (members := target.within.get(other_axis)) is not None: + bound[other_role] = members + if (side := target.sides.get(axis)) is not None: + bound[_COVERAGE_ROLE_FOR_AXIS[axis]] = side.named + return bound + + def _directional_window_years(window: RequestedWindow | None) -> tuple[int | None, int | None]: """`window` re-expressed as the `first_year`/`last_year` pair `CoverageIndex.open_axis` takes: `None` for both when `window` is `None` @@ -757,7 +775,7 @@ def _directional_options_set( coverage_index.open_axis( variables=sv_dcids, axis=_COVERAGE_ROLE_FOR_AXIS[axis], - bound=_directional_opposite_bound(target, axis=axis), + bound=_directional_bound(target, axis=axis), first_year=first_year, last_year=last_year, ) diff --git a/src/search/engine/region/candidate_collapse.py b/src/search/engine/region/candidate_collapse.py index 10badb09d..812bf835a 100644 --- a/src/search/engine/region/candidate_collapse.py +++ b/src/search/engine/region/candidate_collapse.py @@ -126,11 +126,14 @@ def _offer_place_warnings( ) -> None: """Stamp each spec in `offered`, in menu order, with the place warnings of its own context that the shape in play's do not hold (`stamp_offered_place_warnings`).""" + before = len(state.warnings) stamp_offered_place_warnings( state.warnings, [(spec.spec_id, grounded[spec.spec_id].candidate.place) for spec in offered], departure_facts=state.departure_facts, ) + # Each reading's places are read from the request's words. + state.from_request(*state.warnings[before:]) def _apply_bound_unit_filter( @@ -236,9 +239,11 @@ def _handle_sparse_survivors( return unique_specs if served_twin is not None: # A registry twin reads the elected indicator itself (`_apply_twin_diversion`). - state.warnings.append( - _elected_no_data_twin_substituted_warning(state, served_twin.resolved.shape.label) + substituted = _elected_no_data_twin_substituted_warning( + state, served_twin.resolved.shape.label ) + state.warnings.append(substituted) + state.from_request(substituted) return [served_twin] # The elected reading of a family with no total is one of its breakdowns, so the others # with data answer in its place: one alone, several as candidates. @@ -256,11 +261,11 @@ def _handle_sparse_survivors( # One stand-in answers in the elected shape's place, as re-election installs one. # Definite assembly judges every requirement on its own shape and bindings # (`_assemble_definite`). - state.warnings.append( - _elected_sibling_warning( - no_data=[state.shape_draft.label], elected=[stand_ins[0].resolved.shape.label] - ) + stand_in = _elected_sibling_warning( + no_data=[state.shape_draft.label], elected=[stand_ins[0].resolved.shape.label] ) + state.warnings.append(stand_in) + state.from_request(stand_in) return stand_ins if stand_ins: return stand_ins @@ -510,6 +515,8 @@ async def _assemble_definite( install_place_warnings( state.warnings, _survivor.place, departure_facts=state.departure_facts ) + # The survivor's places are read from the request's words. + state.from_request(*_survivor.place.warnings) _survivor_unbound_qualifiers = settle_answer( state, _survivor.shape, @@ -816,7 +823,9 @@ async def _assemble_candidate_set( ] _offer_place_warnings(state, grounded, unique_specs) if not state.elected_has_data: - state.warnings.append(_elected_no_data_warning(state, unique_specs)) + elected_no_data = _elected_no_data_warning(state, unique_specs) + state.warnings.append(elected_no_data) + state.from_request(elected_no_data) await _record_unheld_values( state, [ diff --git a/src/search/engine/region/codes.py b/src/search/engine/region/codes.py index 71c0a6eed..cad9874a2 100644 --- a/src/search/engine/region/codes.py +++ b/src/search/engine/region/codes.py @@ -179,6 +179,11 @@ # side's whole population for the answer's years (`whole_totals.whole_total_warnings`), so a # sum of the members can differ from the published figure. Its params name the total. WHOLE_TOTAL_AVAILABLE = "whole_total_available" +# A fresh part reading DAC2a total ODA with no provider axis, which the bilateral default +# reaches, could not read the bilateral destinations alone +# (`scope_decision.bilateral_withheld_warning`), so its figures include core contributions to +# multilateral agencies. Its params name why. +BILATERAL_READING_UNAVAILABLE = "bilateral_reading_unavailable" # The app has loaded no conversion catalogue yet, so a currency answer states no # unit_conversion offer; the null says nothing about whether it converts. UNIT_CONVERSION_UNAVAILABLE = "unit_conversion_unavailable" diff --git a/src/search/engine/region/composite_answer.py b/src/search/engine/region/composite_answer.py index d7c79d082..bb1fa0fb0 100644 --- a/src/search/engine/region/composite_answer.py +++ b/src/search/engine/region/composite_answer.py @@ -30,31 +30,41 @@ from __future__ import annotations import asyncio +import contextvars import dataclasses import logging from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext from dataclasses import dataclass -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, cast, get_args import dc_graph from code_index.shared import get_index from dc_sdmx import SdmxClient as MaterialiseSdmxClient +from dfo_platform.roles import role_key_axis from materialise import GraphReader from materialise.rate_tables import RateTableCache from origin_catalogue import Catalogue, bound_catalogue, recording_un_blocked_places from search.engine.clients.graph import EngineGraphClient from search.engine.clients.sdmx import EngineSdmxClient -from search.engine.concurrency import bounded_semaphore, create_owned_task +from search.engine.concurrency import ( + bounded_semaphore, + bounded_thread_permits, + create_owned_task, +) +from search.engine.config import settings from search.engine.degraded_reads import ( LOST_READ_ERRORS, note_degraded_read, note_lost_read, recording_degraded_reads, ) +from search.engine.departures import FactsByWarning, WarningKey, warning_key from search.engine.envelope import now_ms from search.engine.errors import EngineInputError from search.engine.events import ObservationsEvent +from search.engine.identity_edit import composite_table_identity from search.engine.region.codes import ( OBSERVATIONS_BULK_FETCH_FAILED, OBSERVATIONS_CLIENTS_UNAVAILABLE, @@ -68,6 +78,7 @@ OBSERVATIONS_TIMED_OUT, UN_SYSTEM_UNAVAILABLE, _ObsProbeResult, + _OptionProbeResult, ) from search.engine.region.observations.dispatch import ( PreviewRead, @@ -76,15 +87,28 @@ _deflator_projected_warning, ) from search.engine.region.resubmit import resolve_spec_resubmit -from search.engine.region.result import RegionResult, with_composite_answer +from search.engine.region.result import ( + RegionResult, + recounted_scope, + with_composite_answer, + with_restated_keys, +) +from search.engine.region.scope_decision import ( + ScopeDecision, + bilateral_withheld_warning, + decide_guarded, +) from search.engine.region.state import EngineOptions from search.engine.region.un_system import un_unavailable_warning from search.engine.response_cache_keys import is_region_storable from search.engine.spec_id import identity_spec_id from search.models import ( BaseIdentity, + Component, EntityGroup, EntityRoleKey, + NoDataReason, + Operation, ResubmitIdentityPayload, Spec, SpecObservations, @@ -102,14 +126,18 @@ compose_observations, served_components, ) +from search.scope.provider import bilateral_default_reading, elect_latest_window from search.scope.region import region_groups +from search.scope.state import ScopeState from search.scope.tables import ScopeFacts from series_coverage.shared import CoverageSnapshot if TYPE_CHECKING: from collections.abc import Mapping + from code_index.scope_conventions import RolePopulation from search.engine.response_cache_flow import PartMemoScope + from search.heading.departure_fact import DepartureFact logger = logging.getLogger(__name__) @@ -238,7 +266,12 @@ async def _read_component( was lost (`_missing`).""" identity = BaseIdentity.model_validate(dict(item.component.identity)) read = await _resubmit_preview( - identity, groups, reads=reads, semaphore=semaphore, about=f"component {item.component.role}" + identity, + groups, + reads=reads, + semaphore=semaphore, + about=f"component {item.component.role}", + populations=item.component.populations, ) if isinstance(read, _LostRead): return None, (_missing(item, read),) @@ -273,11 +306,15 @@ async def _resubmit_preview( reads: ComponentReads, semaphore: asyncio.Semaphore | None, about: str, + populations: Mapping[str, RolePopulation] | None = None, ) -> tuple[SpecObservations, tuple[Warning, ...]] | _LostRead: """The preview a resubmit of `identity` posting `groups` embeds, read with `resolve_spec_resubmit` in a worker thread through `reads.catalogue` under `reads.options`, holding a permit of `semaphore` when one is given, and the answer's warnings under `PREVIEW_WARNING_CODES`. A part memo hit reads nothing, and a miss stores the answer. + Each role `populations` lists, the roles a component binds to a population + (`ComponentIdentity.populations`), reads as open within it + (`ResubmitIdentityPayload.populations`). A read lost to `degraded_reads.LOST_READ_ERRORS` is noted (`note_lost_read`, so a read cut by its enclosing budget is noted as cut and leaves the request unstorable), logged under `about`, and is lost @@ -293,6 +330,7 @@ async def _resubmit_preview( entity_groups=None if groups is None else region_groups(groups, identity, index=get_index()), + populations=dict(populations or {}), ) options = reads.options memo = reads.memo if reads.memo is not None and reads.memo.active else None @@ -523,6 +561,601 @@ async def _preview_events( return (event.stating_degraded(lost.degraded),), tuple(warnings) +def population_table_answer( + spec: Spec, identity: BaseIdentity, *, reads: ComponentReads +) -> RegionResult | None: + """The answer a resubmit of composite `identity` resolves for its table identity, where + that read binds a population (`identity_edit.composite_table_identity`, + `TableRead.populations`) or the population's total in place of a destination `identity` + leaves out (`RolePopulation.total`): `spec`, an answer resolved over the whole table, + read again with each population role open within its members (`resolve_spec_resubmit`), + posting `spec`'s entity groups, through `reads.catalogue`. A fresh answer discovers its + open roles before its scope settles on the composite, and a merge rebuilds its Spec from + its members', so its members, scope, headline, coverage and warnings count every party + the table holds; this read counts the composite's population alone, as the resubmit door + does (`dataset_resubmit`). It answers whatever the resubmit answers, no_data included, + where the population holds no row in the window. + + Where the composite refuses `identity`, as it does a population that is unavailable, + the read answers no_data, `facet_not_available`, and logs the refusal at ERROR: a fresh + search does not refuse its scope, and the whole table's read would state every party + under the composite's spec_id. A read that fails answers no_data too + (`_failed_population_read`), so no error of a read the request did not post reaches + its caller. None where the table read binds neither, so the caller reads `spec` as the + one table it resolved.""" + try: + table = composite_table_identity( + identity, pointer="/input/identity", coverage=reads.coverage + ) + except EngineInputError as exc: + logger.error( + "population read of %s as %s: the composite refuses it (%s): %s", + spec.spec_id, + identity_spec_id(identity), + exc.code, + exc, + ) + return _unread_population("facet_not_available") + if not table.populations and not _binds_left_out_role(identity, table.identity): + return None + groups = cast("Mapping[EntityRoleKey, EntityGroup] | None", spec.resolved.entity_groups) + payload = ResubmitIdentityPayload( + identity=table.identity, + entity_groups=None + if groups is None + else region_groups(groups, table.identity, index=get_index()), + populations=dict(table.populations), + ) + try: + with bound_catalogue(reads.catalogue): + return resolve_spec_resubmit( + inp=payload, + graph=reads.graph, + sdmx=reads.sdmx, + coverage=reads.coverage, + options=EngineOptions( + materialise_graph=reads.materialise_graph, + materialise_sdmx=reads.materialise_sdmx, + rate_cache=reads.rate_cache, + deadline=reads.deadline, + ), + ) + except (EngineInputError, *LOST_READ_ERRORS) as exc: + return _failed_population_read(spec, identity, exc) + + +def _failed_population_read(spec: Spec, identity: BaseIdentity, exc: BaseException) -> RegionResult: + """The no_data answer of `spec`'s population read as composite `identity` where the read + raised `exc`. A refusal of the posted read (`EngineInputError`) answers its own code where + that code is a no_data reason, else `facet_not_available`: the request never posted the + identity the refusal points into. A lost read (`degraded_reads.LOST_READ_ERRORS`) is noted + (`note_lost_read`) and answers degraded, `timed_out` where it crossed its deadline + (`TimeoutError`, `GraphDeadlineExceededError`), else `no_observations`.""" + about = f"population read of {spec.spec_id} as {identity_spec_id(identity)}" + if isinstance(exc, EngineInputError): + logger.warning("%s refused (%s): %s", about, exc.code, exc) + reason = exc.code if exc.code in _NO_DATA_REASONS else "facet_not_available" + return _unread_population(cast("NoDataReason", reason)) + note_lost_read(about, exc) + timed_out = isinstance(exc, TimeoutError | dc_graph.GraphDeadlineExceededError) + logger.warning("%s lost: %s: %s", about, type(exc).__name__, exc, exc_info=not timed_out) + return _unread_population("timed_out" if timed_out else "no_observations", degraded=True) + + +_NO_DATA_REASONS: frozenset[str] = frozenset(get_args(NoDataReason)) + + +def _unread_population(reason: NoDataReason, *, degraded: bool = False) -> RegionResult: + """A population read answering no_data for `reason`, `degraded` where it lost a read.""" + return RegionResult( + variable_text=None, + status="no_data", + specs=(), + no_data_reason=reason, + warnings=(), + timing_by_step={}, + degraded=degraded, + ) + + +def _binds_left_out_role(identity: BaseIdentity, read: BaseIdentity) -> bool: + """Whether `read`, the table identity of composite `identity`, binds a flow axis that + `identity` neither names nor opens: the population's total a component binds in place + of a destination the identity leaves out (`RolePopulation.total`).""" + asked = {role_key_axis(key) for key in identity.entities} | set(identity.open_roles or ()) + return any(role_key_axis(key) not in asked for key in read.entities) + + +def with_population_read( + fresh: RegionResult, answered: RegionResult, *, request_warnings: Sequence[Warning] +) -> RegionResult: + """`answered`, the population read of `fresh`'s definite answer + (`population_table_answer`), definite or not, carrying what `fresh` holds from the + request rather than from the rows it read, so every figure, member, plan and warning the + read derives is `answered`'s, and a field this leaves out costs wording, not figures: + + - `variable_text`, `earliest_index` and `llm_usage`: the request's words for the part, + its position among the request's parts, and the model calls its words took; + - `timing_by_step`: the steps the request's pass ran, the population read included; + - `warnings`: `request_warnings`, those the pass raised before it interpreted its read + (`ResolveState.request_warnings`), then `answered`'s, every one about the answer keyed + to `answered`'s Spec where `fresh`'s were keyed to its own (`_grafted_warnings`); + - `departure_facts`: those of `request_warnings`, then `answered`'s; + - `reads_deadline` and `promote_relaunch`: the request door's own deadline, and the + relaunch facts only a resubmit door's part memo binds, which a fresh part has none of; + - `degraded`: also set where the pass swallowed a read before the population read. + + Every field the request's pass sets once its phases return (its words, entity words, + judged requirements, scope) is set on this result by that pass. `fresh`'s trailing tasks + read the whole table, so the caller cancels them (`result.cancel_dropped_tasks`).""" + kept = {warning_key(warning) for warning in request_warnings} + reading = fresh.spec.spec_id + warnings = _grafted_warnings( + fresh.warnings, + answered, + reading=reading, + keyed_to=answered.spec.spec_id if answered.status == "definite" else reading, + request_warnings=request_warnings, + own=True, + ) + return dataclasses.replace( + answered, + variable_text=fresh.variable_text, + earliest_index=fresh.earliest_index, + llm_usage=fresh.llm_usage, + timing_by_step=fresh.timing_by_step, + warnings=warnings, + departure_facts={ + **{key: fact for key, fact in fresh.departure_facts.items() if key in kept}, + **answered.departure_facts, + }, + reads_deadline=fresh.reads_deadline, + promote_relaunch=fresh.promote_relaunch, + degraded=fresh.degraded or answered.degraded, + ) + + +def _grafted_warnings( + held: Sequence[Warning], + answered: RegionResult, + *, + reading: str, + keyed_to: str, + request_warnings: Sequence[Warning], + own: bool, +) -> tuple[Warning, ...]: + """`held`, a fresh region's warnings, with those about the reading under spec_id + `reading` replaced by `answered`'s, its read within its population, and every warning + about the reading keyed to `keyed_to`, so none names a Spec the response does not hold. + + A warning of `held` is about the reading when keyed to `reading`, or keyed to none where + the reading is the region's `own`, its definite answer or first candidate. Of those, the + ones in `request_warnings`, raised before the pass interpreted its read + (`ResolveState.request_warnings`), stand; the rest came from the whole table's rows. A + warning of `answered` is about the reading when keyed to the Spec it answers, whose + spec_id is its own table identity's and may differ from `reading`, or to `reading`, or to + none. A warning keyed to none on the region's own reading stays keyed to none.""" + request = set(map(warning_key, request_warnings)) + read = answered.spec.spec_id if answered.status == "definite" else None + + def about(warning: Warning) -> bool: + return warning.spec_id == reading or (own and warning.spec_id is None) + + def keyed(warning: Warning) -> Warning: + if warning.spec_id is None and own: + return warning + if warning.spec_id not in (None, reading, read) or warning.spec_id == keyed_to: + return warning + return warning.model_copy(update={"spec_id": keyed_to}) + + stood = ( + keyed(warning) if about(warning) else warning + for warning in held + if not about(warning) or warning_key(warning) in request + ) + return (*stood, *map(keyed, answered.warnings)) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class RestatedReading: + """A reading restated as the composite it is served as (`restated_reading`).""" + + composite: Spec + # The composite's components (`CompiledScope.components`). + components: tuple[Component, ...] + # The closure that reads the composite's preview per component. + relaunch: CompositeRelaunch + + +def restated_reading( + table: Spec, + identity: BaseIdentity, + spec_id: str, + *, + facts: ScopeFacts, + reads: ComponentReads, +) -> RestatedReading: + """`table`, a reading resolved for composite `identity`'s table identity, restated as the + composite under `spec_id` (`scope.composite.assemble_composite`), over the components + `identity` serves (`composite_served`), read through `reads.catalogue` with the part's + `facts`, with the closure that reads its preview per component (`CompositeRelaunch`). + + Every reading a part shows is restated here: its definite answer (`with_decided_scope`), + each reading a candidate set or refusal shows (`with_bilateral_readings`), a set left + with one (`_collapsed`), a same-shape merge (`as_merged_composite`) and an identity + entry (`as_composite_answer`). A reading whose composite binds a population is read + within it first (`population_table_answer`), so `table` counts the population alone.""" + served = composite_served(identity, reads=reads) + with bound_catalogue(reads.catalogue): + composite, components = assemble_composite( + table, identity, spec_id, served=served, facts=facts + ) + return RestatedReading( + composite=composite, + components=tuple(components), + relaunch=CompositeRelaunch(spec=composite, reads=reads, served=served), + ) + + +def with_decided_scope( + result: RegionResult, decision: ScopeDecision, *, reads: ComponentReads +) -> RegionResult: + """`result`, a definite answer, stating the scope `decision` decided for it + (`RegionResult.scope_choices`) and carrying the decision's warning once + (`ScopeDecision.warning`). Where the decision compiles a composite, the answer is + restated as it (`restated_reading`, `result.with_composite_answer`) over `reads`, the + compiled scope listing the composite's components. A fresh definite answer + (`pipeline._with_scope_choices`) and a candidate set left with one reading (`_collapsed`) + both state their scope here.""" + if decision.warning is not None and warning_key(decision.warning) not in set( + map(warning_key, result.warnings) + ): + result = dataclasses.replace(result, warnings=(*result.warnings, decision.warning)) + built = decision.scope + if built is None: + return result + if not decision.composite: + return dataclasses.replace(result, scope_choices=built) + restated = restated_reading( + result.spec, + built.compiled.identity, + built.compiled.spec_id, + facts=decision.facts, + reads=reads, + ) + compiled = built.compiled.model_copy(update={"components": list(restated.components)}) + return dataclasses.replace( + with_composite_answer(result, restated.composite, observations_relaunch=restated.relaunch), + scope_choices=built.model_copy(update={"compiled": compiled}), + ) + + +def as_composite_answer( + region: RegionResult, + identity: BaseIdentity, + spec_id: str, + *, + facts: ScopeFacts, + reads: ComponentReads, +) -> RegionResult: + """`region`, a definite answer resolved for composite `identity`'s table identity, + answering the composite under `spec_id` (`restated_reading`).""" + restated = restated_reading(region.spec, identity, spec_id, facts=facts, reads=reads) + return with_composite_answer( + region, restated.composite, observations_relaunch=restated.relaunch + ) + + +def with_bilateral_readings( + region: RegionResult, + *, + reads: ComponentReads, + selection: ScopeState, + facts: ScopeFacts, + operations: Sequence[Operation] = (), + request_warnings: Sequence[Warning] = (), +) -> RegionResult: + """`region`, a fresh search's candidate set or refusal, with each reading it shows that + is not its definite answer read as a fresh definite answer is: a candidate, the alternate + a refusal offers in its place, each flagged match and each nearest real suggestion, + restated as the composite of DAC2a's bilateral destinations where the default settles it + there under the part's provider entries `selection` holds + (`scope.provider.bilateral_default_reading`), so a stated or carried + `provider:contributions` keeps total ODA, over the one year it serves for a `latest` + window (`scope.provider.elect_latest_window`). A client can fetch any of them by its + spec_id, so none serves the agencies the definite answer leaves out. `facts` are the + part's (`ScopeFacts`), which every reading's rules and restatement share. Every reading + is restated by `restated_reading`, through `reads.catalogue`, the request's catalogue + before any part's source scope narrowed it, as the definite answer is. + A reading whose composite binds a population is read again within it first + (`population_table_answer`), once per spec_id, the readings concurrently + (`_population_reads`), so its scope, headline, coverage and warnings count the + population alone: of the warnings about it, those in `request_warnings` + (`ResolveState.request_warnings`) stand and the read's replace the rest, keyed to the + reading (`_grafted_warnings`), and its `role_scope_defaulted` warning counts the + composite's parties (`result.recounted_scope`). A reading the default reaches that reads + total ODA because the default could not apply, on a part with no provider axis, carries + the `bilateral_reading_unavailable` warning a definite answer would + (`scope_decision.bilateral_withheld_warning`). A reading whose population holds no row in + the window, or whose composite refuses it, is dropped, never restated over the whole + table. A set left with no candidate refuses with the reason its first such read gives, + and a set left with one answers as a definite answer of that reading does (`_collapsed`). + + Each restated reading takes the composite's spec_id, and the warnings, unpublished-whole + facts and assumption echoes keyed to the table's spec_id are keyed to it + (`result.with_restated_keys`). A candidate set keeps its trailing slot options task, whose + events for a restated reading the caller withholds on the event loop + (`without_restated_options`). Where the region's own reading (`RegionResult.spec`) is + restated, the closures that would launch its slot options, place options and preview + again read the table under the composite's spec_id, so the region carries none of them, + nor the facts a part memo hit binds them from (`promote_relaunch`), as a composite answer + carries none (`result.with_composite_answer`). `region`, with any such warning, where no + reading is restated.""" + identities: dict[str, BaseIdentity | None] = {} + renamed: dict[str, Spec] = {} + tables: dict[str, Spec] = {} + + def served_as(spec: Spec) -> BaseIdentity | None: + """The composite identity `spec` is served as, None where it is served as itself.""" + if spec.spec_id not in identities: + identity, _ = bilateral_default_reading(spec.definition.identity, selection, facts) + identities[spec.spec_id] = ( + None + if identity is spec.definition.identity + else elect_latest_window(identity, facts) + ) + return identities[spec.spec_id] + + shown: list[Spec] = [ + *region.specs, + *([region.alternate_match] if region.alternate_match is not None else ()), + *(region.flagged_match or ()), + *(region.nearest_real or ()), + ] + with bound_catalogue(reads.catalogue): + pending = { + spec.spec_id: (spec, identity) + for spec in shown + if (identity := served_as(spec)) is not None + } + withheld = { + spec.spec_id: warning + for spec in shown + if served_as(spec) is None + and ( + warning := bilateral_withheld_warning( + spec.spec_id, spec.definition.identity, selection=selection, facts=facts + ) + ) + is not None + } + if withheld: + region = dataclasses.replace(region, warnings=(*region.warnings, *withheld.values())) + answered = _population_reads(pending, reads=reads) + + def restated[S: Spec](spec: S) -> S | None: + identity = served_as(spec) + if identity is None: + return spec + reading = spec.spec_id + read = answered[reading] + if read is not None: + if read.status != "definite": + return None + # The reading keeps its own class (`NearestRealSpec`) and the facts it carries. + spec = spec.model_copy( + update={"definition": read.spec.definition, "resolved": read.spec.resolved} + ) + composite = restated_reading( + spec, identity, identity_spec_id(identity), facts=facts, reads=reads + ).composite + renamed[reading] = composite + tables[reading] = spec + return cast("S", composite) + + def each[S: Spec](specs: Sequence[S]) -> tuple[S, ...]: + return tuple(kept for spec in specs if (kept := restated(spec)) is not None) + + if region.status == "candidates" and len(region.specs) > 1: + survivors = [ + spec + for spec in region.specs + if (read := answered.get(spec.spec_id)) is None or read.status == "definite" + ] + if len(survivors) == 1: + (survivor,) = survivors + return _collapsed( + region, + survivor, + served_as(survivor), + answered.get(survivor.spec_id), + reads=reads, + selection=selection, + operations=operations, + request_warnings=request_warnings, + ) + specs = each(region.specs) + alternate = region.alternate_match + if alternate is not None: + alternate = restated(alternate) + flagged = region.flagged_match + if flagged is not None: + flagged = each(flagged) or None + nearest = region.nearest_real + if nearest is not None: + nearest = each(nearest) or None + unread = [read for read in answered.values() if read is not None and read.status != "definite"] + if not renamed and not unread: + return region + dropped = frozenset(reading for reading, read in answered.items() if read in unread) + warnings, departure_facts = _without_dropped(region.warnings, region.departure_facts, dropped) + for reading, read in answered.items(): + if read is not None and reading not in dropped: + warnings = _grafted_warnings( + warnings, + read, + reading=reading, + keyed_to=reading, + request_warnings=request_warnings, + own=bool(region.specs) and region.spec.spec_id == reading, + ) + departure_facts.update(read.departure_facts) + for reading, composite in renamed.items(): + own = bool(region.specs) and region.spec.spec_id == reading + warnings = recounted_scope(warnings, tables[reading], composite, own=own) + read_again = dataclasses.replace(region, warnings=warnings, departure_facts=departure_facts) + restated_region = dataclasses.replace( + with_restated_keys(read_again, renamed), + specs=specs, + alternate_match=alternate, + flagged_match=flagged, + nearest_real=nearest, + ) + if region.specs and not specs: + # Every reading the set offered holds no row within its population. + restated_region = dataclasses.replace( + restated_region, status="no_data", no_data_reason=unread[0].no_data_reason + ) + if not region.specs or region.spec.spec_id not in renamed.keys() | answered.keys(): + return restated_region + return dataclasses.replace( + restated_region, + slot_options_relaunch=None, + place_options_relaunch=None, + observations_relaunch=None, + promote_relaunch=None, + ) + + +def _without_dropped( + warnings: Sequence[Warning], + departure_facts: FactsByWarning, + dropped: frozenset[str], +) -> tuple[tuple[Warning, ...], dict[WarningKey, DepartureFact]]: + """`warnings` without those keyed to a reading in `dropped`, and `departure_facts` + without the facts only those warnings disclosed (`departures.warning_key`).""" + kept = tuple(warning for warning in warnings if warning.spec_id not in dropped) + held = set(map(warning_key, kept)) + gone = {warning_key(w) for w in warnings if w.spec_id in dropped} - held + return kept, {key: fact for key, fact in departure_facts.items() if key not in gone} + + +def _population_reads( + pending: Mapping[str, tuple[Spec, BaseIdentity]], *, reads: ComponentReads +) -> dict[str, RegionResult | None]: + """The population read of each reading `pending` maps by spec_id to its table answer and + the composite identity it is served as (`population_table_answer`). Two or more run + concurrently, each in a worker thread over a copy of the caller's context, and every + part's reads together take at most the request's concurrency limit + (`SearchSettings.max_variable_concurrency`) at once (`concurrency.bounded_thread_permits`). + They take no permit of the request semaphore, which the part's own leg holds one of while + it resolves (`fanout`), so a request whose legs held every permit would wait on itself.""" + if len(pending) < 2: + return { + reading: population_table_answer(spec, identity, reads=reads) + for reading, (spec, identity) in pending.items() + } + permits = bounded_thread_permits() + + def read( + context: contextvars.Context, spec: Spec, identity: BaseIdentity + ) -> RegionResult | None: + with permits: + return context.run(lambda: population_table_answer(spec, identity, reads=reads)) + + workers = min(len(pending), settings().max_variable_concurrency) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = { + reading: pool.submit(read, contextvars.copy_context(), spec, identity) + for reading, (spec, identity) in pending.items() + } + return {reading: future.result() for reading, future in futures.items()} + + +def _collapsed( + region: RegionResult, + survivor: Spec, + served: BaseIdentity | None, + read: RegionResult | None, + *, + reads: ComponentReads, + selection: ScopeState, + operations: Sequence[Operation], + request_warnings: Sequence[Warning], +) -> RegionResult: + """`region`, a candidate set left with one reading, `survivor`, once the others' reads + held no row within their populations (`with_bilateral_readings`), answering as a fresh + definite answer of `survivor` does: its scope decided on its own identity under the + part's `selection` and `operations` (`scope_decision.decide_guarded`), and, where that + decision compiles a composite, read within the composite's population + (`with_population_read`) and restated as it (`with_decided_scope`). `read` is the + population read of the composite `served` the survivor was shown as, which serves where + the decision compiles that composite; any other composite is read again. A decision + compiling no composite states its scope on `survivor` itself. The warnings keyed to the + dropped readings go with them.""" + warnings, departure_facts = _without_dropped( + region.warnings, + region.departure_facts, + frozenset(spec.spec_id for spec in region.specs) - {survivor.spec_id}, + ) + definite = dataclasses.replace( + region, + status="definite", + specs=(survivor,), + alternate_match=None, + flagged_match=None, + nearest_real=None, + warnings=warnings, + departure_facts=departure_facts, + ) + decision = decide_guarded( + survivor, selection=selection, operations=operations, coverage=reads.coverage + ) + built = decision.scope + if decision.composite and built is not None: + identity = built.compiled.identity + if served != identity: + read = population_table_answer(survivor, identity, reads=reads) + if read is not None: + definite = with_population_read(definite, read, request_warnings=request_warnings) + if definite.status != "definite": + return definite + decision = dataclasses.replace(decision, spec_id=definite.spec.spec_id) + return with_decided_scope(definite, decision, reads=reads) + + +def without_restated_options(before: RegionResult, after: RegionResult) -> RegionResult: + """`after`, the restatement of `before` (`with_bilateral_readings`), with the trailing + slot options task `before` launched read without the events and warnings of each reading + it restated: they read the table the reading resolved, under a spec_id the set does not + show, and a composite reading carries none, as a composite answer carries none + (`result.with_composite_answer`). The other readings keep theirs. The task is wrapped on + the event loop, so the caller runs this there. `after` itself where it restated no + reading or holds another task.""" + task = after.slot_options_task + if task is None or task is not before.slot_options_task: + return after + withheld = frozenset(spec.spec_id for spec in before.specs) - { + spec.spec_id for spec in after.specs + } + if not withheld: + return after + return dataclasses.replace( + after, slot_options_task=create_owned_task(_without_readings(task, withheld)) + ) + + +async def _without_readings( + task: asyncio.Task[_OptionProbeResult], withheld: frozenset[str] +) -> _OptionProbeResult: + """The events and warnings `task` yields, without those of the spec_ids `withheld`.""" + events, warnings = await task + return ( + tuple(event for event in events if event.spec_id not in withheld), + tuple(warning for warning in warnings if warning.spec_id not in withheld), + ) + + def as_merged_composite( region: RegionResult, group: Sequence[RegionResult], *, reads: ComponentReads ) -> RegionResult: @@ -543,7 +1176,11 @@ def as_merged_composite( members' own preview warnings (`same_shape._with_merged_preview`). The observation task the merge carried over from its first member is left to the combiner's reconciliation (`enrichment._reconcile_after_collapse`), which cancels the members' tasks and launches the composite's where the request streams - observations.""" + observations. A composite binding a population restates the merged table identity read + within it (`population_table_answer`), as a resubmit of the merged composite reads it, + since the merged Spec is rebuilt from its members' slots and plans. Where that read + holds no row or refuses, the merge answers its no_data, as a reading the read drops + does (`with_bilateral_readings`), without the warnings keyed to the merged Spec.""" member = group[0].spec deadlines = [ deadline @@ -557,20 +1194,51 @@ def as_merged_composite( update={"facets": member.definition.identity.facets} ) spec_id = identity_spec_id(identity) - served = composite_served(identity, reads=reads) - table = merged.model_copy( - update={ - "resolved": merged.resolved.model_copy(update={"headline": member.resolved.headline}) - } - ) - with bound_catalogue(reads.catalogue): - spec, _ = assemble_composite( - table, - identity, - spec_id, - served=served, - facts=ScopeFacts(index=get_index(), coverage=reads.coverage), + answered = population_table_answer(merged, identity, reads=reads) + if answered is not None and answered.status != "definite": + # Each member was read within the population, so a merged read holding no row there + # contradicts them; the merge answers it, never the whole table restated. + logger.warning( + "merged population read of %s answered %s (%s)", + merged.spec_id, + answered.status, + answered.no_data_reason, + ) + warnings, departure_facts = _without_dropped( + region.warnings, region.departure_facts, frozenset({merged.spec_id}) ) + return dataclasses.replace( + region, + status="no_data", + specs=(), + no_data_reason=answered.no_data_reason, + warnings=warnings, + departure_facts=departure_facts, + degraded=region.degraded or answered.degraded, + slot_options_task=None, + slot_options_relaunch=None, + place_options_task=None, + place_options_relaunch=None, + observations_task=None, + observations_relaunch=None, + ) + if answered is not None: + table = answered.spec + else: + table = merged.model_copy( + update={ + "resolved": merged.resolved.model_copy( + update={"headline": member.resolved.headline} + ) + } + ) + restated = restated_reading( + table, + identity, + spec_id, + facts=ScopeFacts(index=get_index(), coverage=reads.coverage), + reads=reads, + ) warnings = tuple( warning.model_copy(update={"spec_id": spec_id}) if warning.spec_id == merged.spec_id @@ -579,8 +1247,8 @@ def as_merged_composite( ) return with_composite_answer( dataclasses.replace(region, warnings=warnings, observations_task=None), - spec, - observations_relaunch=CompositeRelaunch(spec=spec, reads=reads, served=served), + restated.composite, + observations_relaunch=restated.relaunch, ) diff --git a/src/search/engine/region/grounding.py b/src/search/engine/region/grounding.py index 8b4d3f7fe..e073c5463 100644 --- a/src/search/engine/region/grounding.py +++ b/src/search/engine/region/grounding.py @@ -293,14 +293,24 @@ def _role_scope_defaulted_base(axis: EntityRoleKey) -> str: return f"{_role_scope_axis_clause(axis)}, so the answer covers every {noun} with data" +def _role_scope_counted_base(axis: EntityRoleKey, member_count: int) -> str: + """The clause of a `role_scope_defaulted` warning on `axis` that states `member_count`.""" + noun = axis_noun(axis) + counted = {1: f"the one {noun}", 2: f"both {noun}s"}.get( + member_count, f"all {member_count} {noun}s" + ) + return f"{_role_scope_axis_clause(axis)}, so the answer covers {counted} with data" + + def role_scope_defaulted_message( - discovered: DiscoveredScope, + axis: EntityRoleKey, *, member_count: int | None, bound_dcids: Sequence[str], label_of: Callable[[Sequence[str]], Sequence[str]], ) -> str: - """Build the defaulted-scope warning with member count and any bound opposite-role entities. + """Build the defaulted-scope warning on open `axis` with member count and any bound + opposite-role entities. `member_count` is how many parties of the open axis the materialised read plan serves (`Materialised.scope_member_dcids`, counted by `enrichment.party_count`), or None for a @@ -310,21 +320,15 @@ def role_scope_defaulted_message( more, report the count and opposite-axis noun without label lookup. `label_of` returns labels in the input order. """ - noun = axis_noun(discovered.axis) if member_count is None: - base = _role_scope_defaulted_base(discovered.axis) + base = _role_scope_defaulted_base(axis) else: - counted = {1: f"the one {noun}", 2: f"both {noun}s"}.get( - member_count, f"all {member_count} {noun}s" - ) - base = ( - f"{_role_scope_axis_clause(discovered.axis)}, so the answer covers {counted} with data" - ) + base = _role_scope_counted_base(axis, member_count) if not bound_dcids: return f"{base}." - preposition = "from" if discovered.axis == "destination" else "to" + preposition = "from" if axis == "destination" else "to" if len(bound_dcids) > _ROLE_SCOPE_MAX_NAMED_LABELS: - opposite_noun = axis_noun("origin" if discovered.axis == "destination" else "destination") + opposite_noun = axis_noun("origin" if axis == "destination" else "destination") named = f"any of the {len(bound_dcids)} {opposite_noun}s" else: labels = label_of(bound_dcids) @@ -337,6 +341,14 @@ def role_scope_defaulted_message( return f"{base} {preposition} {named}." +def role_scope_defaulted_warning(axis: EntityRoleKey, message: str) -> Warning: + """The `role_scope_defaulted` warning on open `axis` stating `message`, carrying its axis + in `params`, by which a composite answer restates its count (`result.recounted_scope`).""" + return Warning( + code=ROLE_SCOPE_DEFAULTED, severity="warn", message=message, params={"axis": axis} + ) + + def role_scope_defaulted_message_for_axis(axis: EntityRoleKey) -> str: """Build the unqualified warning for an open axis. @@ -345,6 +357,31 @@ def role_scope_defaulted_message_for_axis(axis: EntityRoleKey) -> str: return f"{_role_scope_defaulted_base(axis)}." +def stated_role_scope_message(spec: Spec, axis: RoleAxis) -> str | None: + """The `role_scope_defaulted` message `spec` states for its open `axis`, built from its + own facts by the builder a fresh answer's warning is built by + (`role_scope_defaulted_message`): the parties its role axis fact counts there + (`RoleAxisFact.member_count`), and the entities its identity binds on the opposite axis, + in binding order, labelled as `resolved.entities` labels them. None where `spec` counts + no party on `axis`.""" + counted = [ + fact.member_count + for fact in spec.resolved.role_axis_facts or () + if fact.role == axis and fact.member_count is not None + ] + if not counted: + return None + opposite: EntityRoleKey = "origin" if axis == "destination" else "destination" + bound = list(dict.fromkeys(spec.definition.identity.entities.get(opposite, ()))) + labels = {entity.ref: entity.label or entity.ref for entity in spec.resolved.entities} + return role_scope_defaulted_message( + axis, + member_count=counted[0], + bound_dcids=bound, + label_of=lambda dcids: [labels.get(dcid, dcid) for dcid in dcids], + ) + + def memoised_labels( label_of: Callable[[Sequence[str]], Sequence[str]], ) -> Callable[[Sequence[str]], Sequence[str]]: @@ -389,11 +426,10 @@ def role_scope_defaulted_warnings( if discovered is not None: if discovered.axis not in unnamed_wholes: warnings.append( - Warning( - code=ROLE_SCOPE_DEFAULTED, - severity="warn", - message=role_scope_defaulted_message( - discovered, + role_scope_defaulted_warning( + discovered.axis, + role_scope_defaulted_message( + discovered.axis, member_count=_scope_party_count( mat, members if members is not None else mat.scope_member_dcids, @@ -410,20 +446,15 @@ def role_scope_defaulted_warnings( # Both flow axes were unbound. The axis above is the one actually # discovered, and this companion axis defaulted right alongside it. warnings.append( - Warning( - code=ROLE_SCOPE_DEFAULTED, - severity="warn", - message=role_scope_defaulted_message_for_axis(mat.also_defaulted_axis), + role_scope_defaulted_warning( + mat.also_defaulted_axis, + role_scope_defaulted_message_for_axis(mat.also_defaulted_axis), ) ) return warnings if mat.open_origin_dcids is not None and "origin" not in unnamed_wholes: warnings.append( - Warning( - code=ROLE_SCOPE_DEFAULTED, - severity="warn", - message=role_scope_defaulted_message_for_axis("origin"), - ) + role_scope_defaulted_warning("origin", role_scope_defaulted_message_for_axis("origin")) ) return warnings @@ -517,11 +548,7 @@ def carrier_flag_warnings( # whole Materialised and would re-run its discovered-scope arm's label # read for every alternate. This arm needs neither. warnings.append( - Warning( - code=ROLE_SCOPE_DEFAULTED, - severity="warn", - message=role_scope_defaulted_message_for_axis("origin"), - ) + role_scope_defaulted_warning("origin", role_scope_defaulted_message_for_axis("origin")) ) if membership_unverified: warnings.append(membership_unverified_warning()) diff --git a/src/search/engine/region/materialise_step.py b/src/search/engine/region/materialise_step.py index f025fecb3..8c01cba34 100644 --- a/src/search/engine/region/materialise_step.py +++ b/src/search/engine/region/materialise_step.py @@ -68,11 +68,13 @@ def run_materialise( open_axes: frozenset[RoleAxis], coverage: CoverageSnapshot, facet_batch: FacetBatch | None = None, + within: Mapping[RoleAxis, frozenset[str]] | None = None, ) -> Materialised | NoDataDraft | MaterialisedCandidates | UnsupportedDraft: """Materialise the chosen shape with scope discovery enabled. `default_presence` derives a default place scope in place of `place_dcids` (`standard_resolve`), `open_axes` - are the flow axes its places leave open (`place_roles.PlaceRoles.open_axes`), and - `facet_batch` shares the facet reads of a shape whose place sets are settling.""" + are the flow axes its places leave open (`place_roles.PlaceRoles.open_axes`), + `facet_batch` shares the facet reads of a shape whose place sets are settling, and + `within` holds the members an open axis is discovered within (`graph_confirm_resolve`).""" return materialise( shape_draft, bindings, @@ -87,6 +89,7 @@ def run_materialise( open_axes=open_axes, coverage=coverage, facet_batch=facet_batch, + within=within, ) diff --git a/src/search/engine/region/observations/bulk_pair_reads.py b/src/search/engine/region/observations/bulk_pair_reads.py index 0fe3b26b1..80d4c4ab8 100644 --- a/src/search/engine/region/observations/bulk_pair_reads.py +++ b/src/search/engine/region/observations/bulk_pair_reads.py @@ -44,6 +44,7 @@ import pyarrow.compute as pc from dc_sdmx import ( + Among, ContainedIn, EntityFilter, SdmxCapExceededError, @@ -122,13 +123,14 @@ def _period_key(time_period: Literal["LATEST"] | Sequence[str] | None) -> _Perio def _bound_key(entities: Mapping[str, EntityFilter]) -> _Bound | None: - """`entities` as a planned read's `bound`, or None for a containment filter, which no - planned read sends.""" + """`entities` as a planned read's `bound`, a population as its members + (`dc_sdmx.Among.members`), or None for a containment filter, which no planned read + sends.""" bound: list[tuple[str, tuple[str, ...]]] = [] for role, dcids in entities.items(): if isinstance(dcids, ContainedIn): return None - bound.append((role, tuple(dcids))) + bound.append((role, dcids.members if isinstance(dcids, Among) else tuple(dcids))) return tuple(bound) diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index 74f3e326f..ac2a0a2ee 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -288,6 +288,9 @@ def _over_conversion_period( if state.unit_request is None or state.unit_elected_selections is None: if stale is not None: state.depart(*stale) + # The years a trailing request's words asked, which the answer's identity no + # longer holds. + state.from_request(stale[0]) return mat_result, filtered catalogue = current_conversion_catalogue() period = unit_request_period( @@ -314,8 +317,10 @@ def _over_conversion_period( is not None ): state.depart(*passed) + state.from_request(passed[0]) elif stale is not None: state.depart(*stale) + state.from_request(stale[0]) omission = deflator_omission_departure( filtered.plan, state.unit_request, @@ -388,6 +393,8 @@ def _narrowed_to_period( ) warning, fact = conversion_period_departure(period, requested=requested, plan=filtered.plan) state.depart(warning, fact) + # The years asked, which the answer's identity, read over the period, no longer holds. + state.from_request(warning) assert fact.shown is not None return mat_result, filtered, fact.shown @@ -897,7 +904,12 @@ def decide() -> ScopeDecision: ) decision = guarded_scope_decision( - answer.spec_id, decide, operations=state.operations, coverage=state.coverage + answer.spec_id, + decide, + identity=answer.identity, + selection=state.scope_selection, + operations=state.operations, + coverage=state.coverage, ) return _AnswerScope(answer=answer, decision=decision) diff --git a/src/search/engine/region/phase_place_materialise.py b/src/search/engine/region/phase_place_materialise.py index d1c55d1a9..dc4c6af2d 100644 --- a/src/search/engine/region/phase_place_materialise.py +++ b/src/search/engine/region/phase_place_materialise.py @@ -2100,7 +2100,9 @@ async def _phase_materialise(state: ResolveState) -> RegionResult | None: mat_result = await _reelect_zero_observation_sibling(state, mat_result) # Classify terminal outcomes and append the answering result's warnings, after any - # re-election, so they describe the shape in ``state.shape_draft``. + # re-election, so they describe the shape in ``state.shape_draft``. Every warning raised + # so far is the request's: from here on they derive from the rows the read holds. + state.request_warnings = tuple(state.warnings) state.opposite_labels = memoised_labels(state.label_dcids) outcome = interpret_materialised( mat_result, @@ -2201,14 +2203,16 @@ def note_denominator_unverified( named = state.named_denominator if named is not None and check_denominator(shape, named) == "unverified": arc = stated_denominator(shape) - state.warnings.append( - denominator_unverified_warning( - named, - divides=arc is not None, - arc_name=_denominator_name(state, arc) if arc is not None else None, - spec_id=spec_id, - ) + warning = denominator_unverified_warning( + named, + divides=arc is not None, + arc_name=_denominator_name(state, arc) if arc is not None else None, + spec_id=spec_id, ) + state.warnings.append(warning) + # The question's words name the base, and the shape states its own: no row read + # decides it, so it stands on an answer read again within a population. + state.from_request(warning) # The words for the base a schema-term denominator names (`quantity.schema_denominator_kind`). @@ -2428,11 +2432,14 @@ async def _nearest_real_for( if not probe.specs: return None near_specs = probe.specs + before = len(state.warnings) stamp_offered_place_warnings( state.warnings, [(spec.spec_id, probe.reads[spec.spec_id].place) for spec in near_specs], departure_facts=state.departure_facts, ) + # Each suggestion's places are read from the request's words. + state.from_request(*state.warnings[before:]) entity_warnings, previews = await _suggestion_previews(state, near_specs, probe.reads) for warning in entity_warnings: if warning not in state.warnings: diff --git a/src/search/engine/region/phase_shared.py b/src/search/engine/region/phase_shared.py index 7d9b312f1..72a44e716 100644 --- a/src/search/engine/region/phase_shared.py +++ b/src/search/engine/region/phase_shared.py @@ -250,16 +250,18 @@ def withdraw_typed_unit(state: ResolveState, exc: UnitConversionRefusal) -> bool state.unit_elected_selections = None if stated: return True + warning = Warning( + code=QUERY_QUALIFIER_DROPPED, + severity="degraded", + message=f'The requested qualifier "{state.unit_text}" was not applied.', + ) state.depart( - Warning( - code=QUERY_QUALIFIER_DROPPED, - severity="degraded", - message=f'The requested qualifier "{state.unit_text}" was not applied.', - ), + warning, measure_not_applied(asked=state.unit_text, property_dcid=None) if isinstance(exc, NonCurrencyUnitError) else price_basis_not_applied(asked=state.unit_text, shown=None), ) + state.from_request(warning) return True diff --git a/src/search/engine/region/pipeline.py b/src/search/engine/region/pipeline.py index f4669f2ce..966f6a2bf 100644 --- a/src/search/engine/region/pipeline.py +++ b/src/search/engine/region/pipeline.py @@ -12,13 +12,13 @@ from typing import Final import dc_graph +from code_index.composites import binds_composite from code_index.shared import get_index from code_index.types import PartyWord from dc_sdmx import SdmxCapExceededError from entity_resolution import Admission, EntityResolver, Resolution from materialise.member_conversion import request_conversion_reader from origin_catalogue import ( - bound_catalogue, current_catalogue, note_un_unserved, recording_un_blocked_places, @@ -50,9 +50,12 @@ from search.engine.region.codes import ALTERNATES_UNEVALUATED, UN_SYSTEM_UNAVAILABLE from search.engine.region.composite_answer import ( ComponentReads, - CompositeRelaunch, - composite_served, + population_table_answer, + with_bilateral_readings, with_composite_observations, + with_decided_scope, + with_population_read, + without_restated_options, ) from search.engine.region.decisions import PartyReadingWithdrawn from search.engine.region.phase_answer import _phase_outcome @@ -94,7 +97,6 @@ RegionResult, cancel_dropped_tasks, every_member_phrases, - with_composite_answer, ) from search.engine.region.scope_decision import decide_guarded, scope_facts from search.engine.region.source_choice import ( @@ -116,8 +118,8 @@ from search.models import Entry, EntryPath, Operation, PartScope, RoleAxis, Warning from search.scope.basis import identity_basis_record from search.scope.build import build_part_scope -from search.scope.composite import assemble_composite from search.scope.state import ScopeIntake +from search.scope.tables import PartFacts, ScopeFacts from series_coverage.shared import CoverageSnapshot logger = logging.getLogger(__name__) @@ -611,45 +613,92 @@ def _with_scope_choices(result: RegionResult, state: ResolveState) -> RegionResu When the binders settle a composite option (the inclusive provider scope, the OECD headline per year), the scope's base is the composite, and the answer is restated as it - (`composite.assemble_composite`), carrying the closure that reads its preview per - component (`composite_answer.CompositeRelaunch`) for - `_with_composite_preview` to launch on the event loop.""" + (`composite_answer.with_decided_scope`), carrying the closure that reads its preview per + component (`composite_answer.CompositeRelaunch`) for `_with_composite_preview` to launch + on the event loop. A composite whose table read binds a population restates the answer + `_within_population` read within it. A DAC2a answer with no provider axis that reads + total ODA because the bilateral default could not apply carries the decision's + `bilateral_reading_unavailable` warning (`ScopeDecision.warning`). + + A result that is not definite states no scope, and the readings it shows in place of a + definite answer, its candidates, the alternate a refusal offers and its flagged matches, + are read as the definite answer reads DAC2a's total ODA + (`composite_answer.with_bilateral_readings`), under the part's scope facts: the answer + phase's decision's where it decided one, else facts of the part's operations alone.""" if result.status != "definite": + decided = state.scope_decision + return with_bilateral_readings( + result, + reads=_component_reads(state), + selection=state.scope_selection, + facts=decided.facts + if decided is not None + else ScopeFacts( + index=get_index(), + coverage=state.coverage, + part=PartFacts(operations=tuple(state.operations)), + ), + operations=state.operations, + request_warnings=state.request_warnings or (), + ) + decision = state.scope_decision + if decision is None or decision.spec_id != result.spec.spec_id: + decision = decide_guarded( + result.spec, + selection=state.scope_selection, + operations=state.operations, + coverage=state.coverage, + ) + return with_decided_scope(result, decision, reads=_component_reads(state)) + + +async def _within_population(result: RegionResult, state: ResolveState) -> RegionResult: + """`result`, the pass's terminal result, read again within the population its composite + binds, or at the population's total in place of a destination it leaves out, where its + definite answer settles on one (`composite_answer.population_table_answer`): the + answer's open roles were discovered over the whole table before its scope settled on + the composite, so the part answers what a resubmit of the composite's table identity + answers, no_data included, and no_data where the composite refuses the read, carrying + the request's words and the warnings raised before the read + (`composite_answer.with_population_read`). It runs before the pass states requirement + departures, so they are stated on that read; stated-word coverage was judged on the + whole table's read, whose StatVars the population read shares. + The scope decision it reads is the answer phase's, else decided here, and is kept for + `_with_scope_choices` as the decision of the read's answer: the scope is decided once, + on the request's own identity, so a read at the population's total, whose Spec names a + destination the request never named, states the scope and composite identity the + request's answer does. The trailing tasks of the whole-table answer are cancelled. + `result` itself for any other result.""" + if result.status != "definite" or state.request_warnings is None: return result spec = result.spec decision = state.scope_decision if decision is None or decision.spec_id != spec.spec_id: - decision = decide_guarded( + decision = await asyncio.to_thread( + decide_guarded, spec, selection=state.scope_selection, operations=state.operations, coverage=state.coverage, ) - built = decision.scope - if built is None: + state.scope_decision = decision + if not decision.composite or decision.scope is None: return result - if not decision.composite: - return dataclasses.replace(result, scope_choices=built) - reads = _component_reads(state) - # The composite follows its mint, which reads each component under its own pin alone, so - # its years, coverage, headline, download and compiled components read the catalogue its - # preview reads, and its served years are the ones its preview filters by. `facts` - # remembers the years its components hold under the part's source scope, so they are - # read afresh. - served = composite_served(built.compiled.identity, reads=reads) - with bound_catalogue(reads.catalogue): - composite, components = assemble_composite( - spec, - built.compiled.identity, - built.compiled.spec_id, - served=served, - facts=decision.facts, - ) - compiled = built.compiled.model_copy(update={"components": components}) - relaunch = CompositeRelaunch(spec=composite, reads=reads, served=served) + t0 = now_ms() + answered = await asyncio.to_thread( + population_table_answer, + spec, + decision.scope.compiled.identity, + reads=_component_reads(state), + ) + if answered is None: + return result + grafted = with_population_read(result, answered, request_warnings=state.request_warnings) + cancel_dropped_tasks(result, grafted) + if grafted.status == "definite": + state.scope_decision = dataclasses.replace(decision, spec_id=grafted.spec.spec_id) return dataclasses.replace( - with_composite_answer(result, composite, observations_relaunch=relaunch), - scope_choices=built.model_copy(update={"compiled": compiled}), + grafted, timing_by_step={**grafted.timing_by_step, "population_read": now_ms() - t0} ) @@ -693,9 +742,11 @@ def merged_scope_choices( """The scope of `merged`, the region a same-shape merge folded `group` into, built as a resubmit of its identity builds it: over the merged identity, with the union of the members' applied entries and the basis record its identity reads - (`basis.identity_basis_record`). None when no member states a scope, - or the merged identity settles no axis. An entry the merged identity cannot take is - reported in `PartScope.dropped_entries`.""" + (`basis.identity_basis_record`). None when no member states a scope, or the merged + identity settles no axis and compiles to no composite; a scope with no choice whose + compiled identity is a composite stands, as a single part's does + (`scope_decision.decide_scope`). An entry the merged identity cannot take is reported in + `PartScope.dropped_entries`.""" scopes = [region.scope_choices for region in group if region.scope_choices is not None] if not scopes: return None @@ -711,7 +762,8 @@ def merged_scope_choices( refuse=False, ) assert isinstance(built, PartScope) - return built if built.choices else None + composite = binds_composite(built.compiled.identity.model_dump(mode="json", include={"facets"})) + return built if built.choices or composite else None def _union(entries: Iterable[Entry]) -> list[Entry]: @@ -1052,6 +1104,7 @@ async def _run_phases(state: ResolveState) -> RegionResult: if result is None: settled = settled or phase is _phase_settle else: + result = await _within_population(result, state) # Only a vetted election can report qualifier matches: earlier terminals # may hold pre-bind candidates without a forced binding or elected shape. every_member = ( @@ -1093,6 +1146,7 @@ async def _run_phases(state: ResolveState) -> RegionResult: stated = _stating_unpublished_wholes(result, state) answered = await asyncio.to_thread(_with_scope_choices, stated, state) cancel_dropped_tasks(stated, answered) + answered = without_restated_options(stated, answered) return await _with_composite_preview(answered, state) if state.capped_shapes and state.scope is not None and not scoped: reads.enter_context(scoped_reads(state.scope.allowed)) diff --git a/src/search/engine/region/place_options.py b/src/search/engine/region/place_options.py index b311d2800..b16ad7435 100644 --- a/src/search/engine/region/place_options.py +++ b/src/search/engine/region/place_options.py @@ -40,6 +40,7 @@ from dataclasses import dataclass, replace from typing import Literal +from code_index.composites import binds_composite from dc_sdmx import SdmxClient as MaterialiseSdmxClient from dfo_platform.build import build_commit from dfo_platform.executor import submit_with_context @@ -234,6 +235,7 @@ def _resolve_target() -> ResolvedOptionsTarget | _Refused | None: population_memo=population_memo, deadline=deadline, join_inflight=False, + coverage=coverage, ) lost.merge_into(target_lost) return resolved @@ -366,6 +368,11 @@ def place_options_task( ) +def _answers_composite(spec: Spec) -> bool: + """Whether `spec`'s identity binds a declared composite as its source.""" + return binds_composite(spec.definition.identity.model_dump(mode="json", include={"facets"})) + + def replayed_place_options( *, stream: bool, @@ -383,7 +390,9 @@ def replayed_place_options( materialise graph or pool. A fresh run launches one task per definite Spec (`place_options_task`), which the - response's part carries. Parts folded into one Spec share its task. + response's part carries, and none for a Spec that answers a composite, whose place + options would read its table under the composite's spec_id + (`result.with_composite_answer`). Parts folded into one Spec share its task. The tasks read `coverage`, the hit's own snapshot, which the bundle's key holds.""" if not stream or materialise_graph is None or pool is None: return None @@ -395,7 +404,7 @@ def launch(response: ResolveResponse) -> list[asyncio.Task[list[PlaceOptionsEven specs = { part.spec.spec_id: part.spec for part in (response.parts if response.kind == "resolution" else []) - if isinstance(part, DefinitePart) + if isinstance(part, DefinitePart) and not _answers_composite(part.spec) } return [ place_options_task( diff --git a/src/search/engine/region/promote_finalize.py b/src/search/engine/region/promote_finalize.py index ba8bb850e..a6eda536a 100644 --- a/src/search/engine/region/promote_finalize.py +++ b/src/search/engine/region/promote_finalize.py @@ -335,7 +335,9 @@ def _phase_finalize(state: _PromoteState) -> RegionResult: with timed_step(state, "finalize_wholes"): resubmit_warnings = ( *resubmit_warnings, - *whole_total_warnings(spec, coverage=state.coverage.index), + *whole_total_warnings( + spec, coverage=state.coverage.index, populations=state.inp.populations + ), ) relaunch_t0 = now_ms() diff --git a/src/search/engine/region/promote_ground.py b/src/search/engine/region/promote_ground.py index 59be1fa6d..2b904b627 100644 --- a/src/search/engine/region/promote_ground.py +++ b/src/search/engine/region/promote_ground.py @@ -684,7 +684,9 @@ def _flagged_refusal( reason, warnings=( *state.refusal_warnings(), - *whole_total_warnings(spec, coverage=state.coverage.index), + *whole_total_warnings( + spec, coverage=state.coverage.index, populations=state.inp.populations + ), ), flagged_match=(spec,), gate=gate, diff --git a/src/search/engine/region/promote_materialise.py b/src/search/engine/region/promote_materialise.py index acf6eff03..5d7d963a0 100644 --- a/src/search/engine/region/promote_materialise.py +++ b/src/search/engine/region/promote_materialise.py @@ -11,6 +11,7 @@ from search.engine.clients.graph import Facet from search.engine.envelope import now_ms from search.engine.errors import EngineInputError +from search.engine.identity_edit import population_members from search.engine.region.grounding import _build_entity, memoised_labels, role_bindings from search.engine.region.materialise_step import ( NoDataMaterialise, @@ -237,6 +238,7 @@ def materialise() -> MaterialiseResult: default_presence=state.place.presence_for(state.default_presence), open_axes=state.place.roles.open_axes, coverage=state.coverage, + within=population_members(state.inp.populations), ) # The standard materialise probe uses the representative StatVar. diff --git a/src/search/engine/region/resubmit.py b/src/search/engine/region/resubmit.py index 3900f5b0e..1d21dee80 100644 --- a/src/search/engine/region/resubmit.py +++ b/src/search/engine/region/resubmit.py @@ -15,7 +15,7 @@ from search.engine.clients.sdmx import EngineSdmxClient from search.engine.degraded_reads import recording_degraded_reads from search.engine.errors import EngineInputError -from search.engine.identity_edit import check_described_arm, check_not_composite +from search.engine.identity_edit import check_described_arm, check_not_composite, open_within from search.engine.region.resubmit_promote import _resolve_standard_promote from search.engine.region.resubmit_request import pinned_provenance_scope from search.engine.region.result import RegionResult @@ -68,6 +68,11 @@ def resolve_spec_resubmit( member walk's full read of a deep-root family and every phase after it (`_resolve_standard_promote`). + A role `inp.populations` lists is read as the open role the caller posted, within the + population's members (`identity_edit.open_within`): the identity the promote reads opens + it, and its role-scope discovery reads only those members + (`role_scope.discover_role_scope`). + Pinned StatVar identities and composite identities (`identity_edit.check_not_composite`) raise `request_invalid` (400). Missing graph nodes produce `no_data`, except absent observationAbout role refs, which @@ -80,6 +85,9 @@ def resolve_spec_resubmit( check_described_arm(inp.identity.shape, pointer="/input/identity/shape") check_not_composite(inp.identity, pointer="/input/identity/facets/provenance") _options = options if options is not None else EngineOptions() + opened, _ = open_within(inp.identity, inp.populations) + if opened is not inp.identity: + inp = inp.model_copy(update={"identity": opened}) # `_phase_materialise` checks origins after rebuilding the shape. # Role-bearing shapes bind origins on their origin axis. Standard shapes raise diff --git a/src/search/engine/region/result.py b/src/search/engine/region/result.py index ab6877de1..39fdf1cbc 100644 --- a/src/search/engine/region/result.py +++ b/src/search/engine/region/result.py @@ -8,6 +8,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, cast +from code_index.composites import DECLARED_COMPOSITES, composite_ref, destination_population +from code_index.shared import get_index from dfo_platform import Cause from materialise.roles import axis_for_role, axis_noun from search.engine.clients.graph import Facet @@ -21,6 +23,8 @@ from search.engine.region.codes import ( ALTERNATES_UNEVALUATED, QUERY_QUALIFIER_DROPPED, + ROLE_SCOPE_DEFAULTED, + WHOLE_TOTAL_AVAILABLE, _ObsProbeResult, _OptionProbeResult, ) @@ -40,6 +44,7 @@ from search.heading.departure_fact import DepartureFact from search.models import ( Assumption, + BaseIdentity, EntityRoleKey, GraphRef, NearestRealSpec, @@ -576,13 +581,18 @@ def with_composite_answer( so the composite carries none of their tasks and no slot or place options. Its preview is read per component by `observations_relaunch` (`composite_answer.CompositeRelaunch`), which the caller launches on the event loop where the request streams observations - (`enrichment._launch_trailing_tasks`). + (`enrichment._launch_trailing_tasks`). The table answer's own warnings, unpublished-whole + facts and assumption echoes are keyed to the composite (`with_restated_keys`), and its + `role_scope_defaulted` warnings count the composite's open axes (`recounted_scope`). The restatement touches no task, so it runs on any thread. A task `region` held belongs to the request's event loop, and the caller cancels it there (`cancel_dropped_tasks`).""" + recounted = dataclasses.replace( + region, warnings=recounted_scope(region.warnings, region.spec, spec, own=True) + ) return dataclasses.replace( - region, + with_restated_keys(recounted, {region.spec.spec_id: spec}), specs=(spec, *region.specs[1:]), slot_options_task=None, slot_options_relaunch=None, @@ -593,6 +603,93 @@ def with_composite_answer( ) +def recounted_scope( + warnings: Sequence[Warning], table: Spec, composite: Spec, *, own: bool +) -> tuple[Warning, ...]: + """`warnings`, with each `role_scope_defaulted` warning about `table` on an axis + `composite` counts (its `params` axis, `grounding.role_scope_defaulted_warning`) stating + the composite's count in place of `table`'s, built from the composite's own facts by the + builder a fresh answer's warning is built by (`grounding.stated_role_scope_message`): + the composite counts the parties of every component, the table answer those of its one + table (`scope.composite.assemble_composite`). A warning is about `table` when keyed to + it, or keyed to none where `table` is the region's `own` reading. Every other warning + stands.""" + # grounding imports region.facets, which imports this module. + from search.engine.region.grounding import stated_role_scope_message + + restated = { + fact.role: message + for fact in composite.resolved.role_axis_facts or () + if (message := stated_role_scope_message(composite, fact.role)) is not None + } + if not restated: + return tuple(warnings) + + def recounted(warning: Warning) -> Warning: + about = warning.spec_id == table.spec_id or (own and warning.spec_id is None) + axis = (warning.params or {}).get("axis") + if not about or warning.code != ROLE_SCOPE_DEFAULTED or not isinstance(axis, str): + return warning + message = restated.get(axis) + return warning if message is None else warning.model_copy(update={"message": message}) + + return tuple(map(recounted, warnings)) + + +def with_restated_keys(region: RegionResult, renamed: Mapping[str, Spec]) -> RegionResult: + """`region` with what it keys by the spec_id of each reading `renamed` restates keyed to + the restated Spec's: its warnings, without those the restated Spec cannot stand beside + (`_stands_beside`), its unpublished-whole facts and its assumption echoes. A warning keyed + to another reading, or to none, stands as it is.""" + + def warning(held: Warning) -> Warning | None: + spec = renamed.get(held.spec_id) if held.spec_id is not None else None + if spec is None: + return held + if not _stands_beside(held, spec.definition.identity): + return None + return held.model_copy(update={"spec_id": spec.spec_id}) + + def key(spec_id: str) -> str: + spec = renamed.get(spec_id) + return spec_id if spec is None else spec.spec_id + + return dataclasses.replace( + region, + warnings=tuple(kept for held in region.warnings if (kept := warning(held)) is not None), + unpublished_whole_facts={ + key(spec_id): facts for spec_id, facts in region.unpublished_whole_facts.items() + }, + reading_assumption_echoes={ + key(spec_id): echo for spec_id, echo in region.reading_assumption_echoes.items() + }, + ) + + +def _stands_beside(warning: Warning, identity: BaseIdentity) -> bool: + """Whether `warning`, read over a table answer restated as composite `identity`, holds + for the composite: every warning but a `whole_total_available` naming a destination + total that a component of the composite reading one side of its destinations leaves out + (`code_index.composites.destination_population`, `RolePopulation.named`). "All + Recipients, Total" counts the core contributions a bilateral reading leaves out.""" + if warning.code != WHOLE_TOTAL_AVAILABLE: + return True + total = (warning.params or {}).get("total") + ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) + composite = DECLARED_COMPOSITES.get(ref) if ref is not None else None + if composite is None or not isinstance(total, str): + return True + index = get_index() + party = index.parties.get(total) + return not any( + party is not None + and party.publishes("destination", (recipe.dataset,)) + and total not in destination_population(index, recipe.dataset, composite=ref).named + for recipe in composite.components + if recipe.destinations is not None + ) + + def cancel_dropped_tasks(before: RegionResult, after: RegionResult) -> None: """Cancel each trailing task `before` holds that `after`, its restatement, does not (`with_composite_answer`). `Task.cancel` is not thread-safe, so the caller runs this on diff --git a/src/search/engine/region/scope_decision.py b/src/search/engine/region/scope_decision.py index 7a79486c9..1b686607c 100644 --- a/src/search/engine/region/scope_decision.py +++ b/src/search/engine/region/scope_decision.py @@ -19,6 +19,7 @@ from code_index.shared import get_index from search.engine.degraded_reads import LOST_READ_ERRORS, note_degraded_read from search.engine.departures import datasets_served +from search.engine.region.codes import BILATERAL_READING_UNAVAILABLE from search.models import ( BaseIdentity, EntityGroup, @@ -27,10 +28,15 @@ ProvenanceRef, ResolvedFacet, Spec, + Warning, ) from search.scope.basis import basis_dataset, unbound_basis_record from search.scope.build import build_part_scope from search.scope.provider import ( + CONTRIBUTIONS, + bilateral_default_reaches, + bilateral_default_reading, + bilateral_default_withheld, settle_provider, with_summed_provider, without_unoffered_multilateral, @@ -46,12 +52,15 @@ class ScopeDecision: """The scope the answer `spec_id` states, built over `facts`; None when it states none. `composite` when its compiled identity is a declared composite (`composite_ref`), which - the answer is then restated as (`scope.composite.assemble_composite`).""" + the answer is then restated as (`scope.composite.assemble_composite`). `warning` is the + answer's `bilateral_reading_unavailable` warning where it reads total ODA because the + bilateral default could not apply (`bilateral_withheld_warning`).""" spec_id: str facts: ScopeFacts scope: PartScope | None composite: bool + warning: Warning | None = None def answer_scope_facts( @@ -111,12 +120,17 @@ def guarded_scope_decision( spec_id: str, decide: Callable[[], ScopeDecision], *, + identity: BaseIdentity, + selection: ScopeState, operations: Sequence[Operation], coverage: CoverageSnapshot, ) -> ScopeDecision: - """The decision `decide` makes for the answer `spec_id`, or, when it raises, a decision - that states no scope and no composite, so the answer reads as the one table it resolved. - `decide` builds the facts it reads, so an error there is guarded too. + """The decision `decide` makes for the answer `spec_id` of `identity`, or, when it + raises, a decision that states no scope and no composite, so the answer reads as the one + table it resolved. `decide` builds the facts it reads, so an error there is guarded too. + Where the bilateral default reaches `identity`, that answer reads DAC2a's total ODA with + no scope to say so, so the fallback carries the `bilateral_reading_unavailable` warning + with `why` `scope_undecided` (`_undecided_warning`). A composite its components cannot serve (`CompositeNotServable`) or a read the decision lost (`degraded_reads.LOST_READ_ERRORS`) is logged, and the answer can be stored. Any @@ -145,7 +159,32 @@ def guarded_scope_decision( facts = answer_scope_facts( entity_groups={}, facets={}, sources=(), operations=operations, coverage=coverage ) - return ScopeDecision(spec_id=spec_id, facts=facts, scope=None, composite=False) + return ScopeDecision( + spec_id=spec_id, + facts=facts, + scope=None, + composite=False, + warning=_undecided_warning(spec_id, identity, selection=selection, facts=facts), + ) + + +def _undecided_warning( + spec_id: str, identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts +) -> Warning | None: + """The `bilateral_reading_unavailable` warning, `why` `scope_undecided`, for the answer + `spec_id` of `identity` whose scope decision raised, where the bilateral default reaches + `identity` (`provider.bilateral_default_reaches`) and no stated or carried provider entry + asks for total ODA. None otherwise, and where the reach check raises too, which is + logged.""" + stated = selection.entries_on("provider") + if stated and stated[0].option_id == CONTRIBUTIONS: + return None + try: + reaches = bilateral_default_reaches(identity, facts) + except Exception: + logger.exception("bilateral default reach of %s could not be read", spec_id) + return None + return _withheld_warning(spec_id, "scope_undecided") if reaches else None def decide_guarded( @@ -167,6 +206,8 @@ def decide_guarded( operations=operations, coverage=coverage, ), + identity=spec.definition.identity, + selection=selection, operations=operations, coverage=coverage, ) @@ -194,9 +235,22 @@ def decide_scope( none. A part whose binders settled no axis, such as one naming no place, states the scope of its identity where the measure-basis axis applies to it, with the stated or carried basis that identity holds applied (`basis.unbound_basis_record`). A fresh - search does not refuse its scope (`build_part_scope` with `refuse` false).""" - if states_no_scope(identity, selection): - return ScopeDecision(spec_id=spec_id, facts=facts, scope=None, composite=False) + search does not refuse its scope (`build_part_scope` with `refuse` false). + + DAC2a's total ODA is read at its bilateral destinations alone where the part names none + or only bilateral ones (`provider.with_bilateral_default`), before any axis is read, so + the scope's base is that composite, with or without a provider axis. A part with no axis + then states a scope with no choice, whose compiled identity is the composite. A provider + entry asking for total ODA withholds that default (`provider.bilateral_default_reading`). + A part with no provider axis whose default could not apply carries the warning saying + so (`bilateral_withheld_warning`).""" + warning = bilateral_withheld_warning(spec_id, identity, selection=selection, facts=facts) + served, selection = bilateral_default_reading(identity, selection, facts) + if served is identity and states_no_scope(identity, selection): + return ScopeDecision( + spec_id=spec_id, facts=facts, scope=None, composite=False, warning=warning + ) + identity = served records = dict(selection.records) if not records: # No binder settled an axis, as on a part naming no place. The basis axis applies @@ -221,10 +275,60 @@ def decide_scope( refuse=False, ) assert isinstance(built, PartScope) - if not built.choices: - return ScopeDecision(spec_id=spec_id, facts=facts, scope=None, composite=False) composite = ( composite_ref(built.compiled.identity.model_dump(mode="json", include={"facets"})) is not None ) - return ScopeDecision(spec_id=spec_id, facts=facts, scope=built, composite=composite) + if not built.choices and not composite: + return ScopeDecision( + spec_id=spec_id, facts=facts, scope=None, composite=False, warning=warning + ) + return ScopeDecision( + spec_id=spec_id, facts=facts, scope=built, composite=composite, warning=warning + ) + + +_WITHHELD_REASONS: Mapping[str, str] = { + "destination_unclassified": ( + "the data holds a destination no side declaration places, so the bilateral " + "destinations cannot be told apart" + ), + "option_not_covered": "the bilateral destinations do not cover every year of the answer", + "scope_undecided": "the answer's scope could not be decided", +} +"""The message's reason for each `why` with words of its own; any other `why` names the +slot the bilateral composite cannot serve.""" + + +def bilateral_withheld_warning( + spec_id: str, identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts +) -> Warning | None: + """The info warning `bilateral_reading_unavailable` for the fresh reading `spec_id` of + `identity`, stamped with that spec_id, where the bilateral default reaches it on a part + with no provider axis and cannot apply (`provider.bilateral_default_withheld`), so its + DAC2a figures are total ODA, core contributions to multilateral agencies included. Its + params name why: the reason `provider.bilateral_default_withheld` gives. None + otherwise.""" + why = bilateral_default_withheld(identity, selection, facts) + return None if why is None else _withheld_warning(spec_id, why) + + +def _withheld_warning(spec_id: str, why: str) -> Warning: + """The `bilateral_reading_unavailable` warning for the reading `spec_id`, naming `why`.""" + return Warning( + code=BILATERAL_READING_UNAVAILABLE, + severity="info", + message=( + "These figures are total ODA, which includes donors' core contributions to " + f"multilateral agencies: {_withheld_reason(why)}." + ), + spec_id=spec_id, + params={"why": why}, + ) + + +def _withheld_reason(why: str) -> str: + """The message's words for `why` (`_WITHHELD_REASONS`), else the slot it names.""" + return _WITHHELD_REASONS.get( + why, f"the bilateral destinations cannot be read with what the part binds ({why})" + ) diff --git a/src/search/engine/region/state.py b/src/search/engine/region/state.py index f9e674451..aab47d1c7 100644 --- a/src/search/engine/region/state.py +++ b/src/search/engine/region/state.py @@ -894,6 +894,13 @@ class ResolveState: # `spec_id`; a candidates answer sets it in `candidate_collapse._assemble_definite`. None # until then. scope_decision: ScopeDecision | None = None + # The warnings of the request's words, its places, its shapes and its bindings: every + # warning the pass raised before it interpreted the elected shape's read + # (`phase_place_materialise._phase_materialise`), and each one raised later from those + # alone (`from_request`). A definite answer read again within a composite's population + # keeps these and takes the read's own in place of the rest + # (`composite_answer.with_population_read`). None until the read is interpreted. + request_warnings: tuple[Warning, ...] | None = None # Shared warnings list; mutations here affect every stage holding the list. warnings: list[Warning] # The departure facts raised beside warnings, each under its warning's key @@ -1665,6 +1672,14 @@ def family_total_absent_departures(self, specs: Iterable[Spec]) -> list[Departed specs, self.family_total_absent, variable_text=self.variable, label_of=self.label_dcids ) + def from_request(self, *warnings: Warning) -> None: + """Record `warnings`, raised once the read is interpreted from the request's words, + places or election and not from the rows read, among the request's warnings + (`request_warnings`). The caller raises them in `warnings` as before. Before the read + is interpreted every warning is the request's, so nothing is recorded.""" + if self.request_warnings is not None: + self.request_warnings = (*self.request_warnings, *warnings) + def settled_dataset(self, candidate: _Candidate | None = None) -> str | None: """The dataset the part's run has settled on for `candidate`, else for the elected shape: the one the shape names (`ShapeDraft.named_dataset`), which is the dataset diff --git a/src/search/engine/region/whole_totals.py b/src/search/engine/region/whole_totals.py index e463b9274..bed0439f8 100644 --- a/src/search/engine/region/whole_totals.py +++ b/src/search/engine/region/whole_totals.py @@ -29,6 +29,7 @@ from typing import TYPE_CHECKING, cast from code_index.declarations import CONTAINING_TOTALS +from code_index.scope_conventions import RolePopulation from code_index.shared import get_index from code_index.types import ( POPULATION_CLASS, @@ -288,7 +289,12 @@ def population_totals(axes: Collection[RoleAxis]) -> tuple[str, ...]: return tuple(sorted(dcid for axis in set(axes) for dcid in by_axis.get(axis, ()))) -def whole_total_warnings(spec: Spec, *, coverage: CoverageIndex) -> list[Warning]: +def whole_total_warnings( + spec: Spec, + *, + coverage: CoverageIndex, + populations: Mapping[str, RolePopulation] | None = None, +) -> list[Warning]: """`whole_total_available` for each flow side `spec` lists member by member, opened by `open_roles` or defaulted (`ResolvedSpec.role_axis_facts`), where the dataset elected for its identity (`bound_values.identity_dataset`) publishes the total of that side's @@ -300,6 +306,12 @@ def whole_total_warnings(spec: Spec, *, coverage: CoverageIndex) -> list[Warning that sums the members sees the published figure first, and is stamped with `spec`'s id: it holds for that reading alone, in a candidate set as in a definite answer. + A side the answer reads open within a population (`populations`, the roles a composite + binds to one, `ResubmitIdentityPayload.populations`) is offered the population's own + total (`RolePopulation.total`, "Developing Countries, Total" for DAC2a's bilateral side) + in place of the side's whole totals, since those count destinations off the population + ("All Recipients, Total" counts the agencies too) (`_population_reading`). + An answer that reads the other side whole states none: the source publishes no figure between two whole totals ("Official Donors, Total" to "All Recipients, Total"), so no published total stands beside the members' sum.""" @@ -326,18 +338,37 @@ def served(reading: WholeReading) -> bool: ) return held is True + datasets = (dataset,) + within = { + axis: population + for key, population in (populations or {}).items() + if (axis := role_key_axis(key)) is not None + } + + def readings_of(axis: RoleAxis) -> list[WholeReading]: + if (population := within.get(axis)) is not None: + return [_population_reading(axis, population)] + return [ + reading + for dcid in population_totals((axis,)) + if (reading := whole_reading((dcid,), datasets=datasets, years=years)) is not None + ] + readings = [ reading for fact in spec.resolved.role_axis_facts or () - for dcid in population_totals((fact.role,)) - if (reading := whole_reading((dcid,), datasets=(dataset,), years=years)) is not None - and served(reading) + for reading in readings_of(fact.role) + if served(reading) ] return [ Warning( code=WHOLE_TOTAL_AVAILABLE, severity="info", - message=_available_message(reading, dataset=dataset), + message=( + _population_message(reading, within[reading.axis]) + if reading.axis in within + else _available_message(reading, dataset=dataset) + ), spec_id=spec.spec_id, params={ "total": reading.members[0], @@ -349,6 +380,25 @@ def served(reading: WholeReading) -> bool: ] +def _population_reading(axis: RoleAxis, population: RolePopulation) -> WholeReading: + """The total of `population`, the side an answer reads open within it, as the reading a + `whole_total_available` warning names: the dataset's published total of that side + (`RolePopulation.total`), labelled as the code index labels it, covering no class.""" + row = get_index().parties.get(population.total) + label = row.label if row is not None and row.label else population.total + return WholeReading( + axis=axis, members=(population.total,), total=population.total, label=label, whole_of=() + ) + + +def _population_message(reading: WholeReading, population: RolePopulation) -> str: + """The `whole_total_available` message for a side read open within `population`: the + total of the side it names.""" + return ( + f"The source publishes the total of {population.source.rebound_words} as {reading.label}." + ) + + def _identity_window(window: IdentityWindow) -> TimeWindow | None: """The years an identity's window asks for, `None` for every year or the latest.""" if window.kind != "range": diff --git a/src/search/engine/response_cache_keys.py b/src/search/engine/response_cache_keys.py index ef9c6e8d7..3dceb57fb 100644 --- a/src/search/engine/response_cache_keys.py +++ b/src/search/engine/response_cache_keys.py @@ -427,15 +427,19 @@ def part_memo_key( ) -> str: """Key one resubmitted identity's region in the part memo. - The payload is the identity and the named groups posted beside it, and the whole of it - enters, since each field changes the answer. + The payload is the identity, the named groups posted beside it and the populations its + roles bind, and the whole of it enters, since each field changes the answer. A + population enters as its side and the dataset it is drawn from, which with the build + commit and the graph generation fix its members, so a role read open within a population + and one naming the same parties key apart. A payload binding none adds nothing to the + key. `engine_flags` are the four enrichment flags of the call's `EngineOptions`. The build commit covers the code index, and the graph generation, coverage stamp, conversion catalogue revision, origin catalogue build with its UN System setting, and UTC day cover what the graph, coverage and conversion reads return and what the answer admits. Its entries come from regions that make no model call (`region.resolve_spec_resubmit`), so the binder a request was routed to is not in the key.""" - document = { + document: dict[str, object] = { "payload": payload.model_dump(mode="json"), "engine_flags": dict(engine_flags), "build_commit": build_commit, @@ -446,6 +450,11 @@ def part_memo_key( "un_system_enabled": un_system_enabled, "today": today, } + if payload.populations: + document["populations"] = { + key: [population.side, population.source.dataset] + for key, population in sorted(payload.populations.items()) + } return _digest("m", document) diff --git a/src/search/engine/retrieval/bound_values.py b/src/search/engine/retrieval/bound_values.py index 9c6ff17cf..49a3c9170 100644 --- a/src/search/engine/retrieval/bound_values.py +++ b/src/search/engine/retrieval/bound_values.py @@ -44,7 +44,7 @@ from enum import StrEnum from typing import TYPE_CHECKING, Literal, NamedTuple, cast -from code_index.composites import table_identity +from code_index.composites import PopulationUnavailable, table_identity from code_index.labels import PRICE_BASIS_AXIS, is_price_basis, rows_held_codes from code_index.members import ( DatasetFamily, @@ -106,6 +106,7 @@ MembershipTooBroadError, RoleSignatureError, ) +from search.engine.identity_edit import population_refusal from search.engine.reading_coverage import covers from search.engine.retrieval.code_reads import carried_values, reads_code from search.engine.retrieval.facts import ( @@ -1788,8 +1789,13 @@ def slot_expansions(bindings: Sequence[SlotBindingDraft]) -> dict[str, Expansion def _identity_document(identity: BaseIdentity) -> Mapping[str, object]: """Return `identity` as the document a reader of one table's family reads: the document the Datasets mint reads, or for a composite the identity its option was set on - (`code_index.composites.table_identity`).""" - return table_identity(get_index(), identity.model_dump(mode="json", exclude_none=True)) + (`code_index.composites.table_identity`). Refuses a composite whose population is + unavailable as Search's doors refuse it (`identity_edit.population_refusal`).""" + document = identity.model_dump(mode="json", exclude_none=True) + try: + return table_identity(get_index(), document).identity + except PopulationUnavailable as exc: + raise population_refusal(exc, pointer="/input/identity") from exc def identity_query(identity: BaseIdentity) -> FamilyQuery: diff --git a/src/search/engine/retrieval/retrieve.py b/src/search/engine/retrieval/retrieve.py index 3f1e53712..e989ae2c1 100644 --- a/src/search/engine/retrieval/retrieve.py +++ b/src/search/engine/retrieval/retrieve.py @@ -324,6 +324,7 @@ def materialise( open_axes: frozenset[RoleAxis] = frozenset(), coverage: CoverageSnapshot, facet_batch: FacetBatch | None = None, + within: Mapping[RoleAxis, frozenset[str]] | None = None, ) -> Materialised | NoDataDraft | MaterialisedCandidates | UnsupportedDraft: """Resolve a shape through the standard or role-bearing path. @@ -335,7 +336,8 @@ def materialise( destination, or an unbound origin on an origin-only shape, is discovered only when ``discover_scope`` is true. A bound origin with no destination remains ``role_not_resolved``. - ``open_axes`` are the role path's open flow axes (`graph_confirm_resolve`). + ``open_axes`` are the role path's open flow axes, and ``within`` the members an open + axis is discovered within (`graph_confirm_resolve`). Returns a materialised result, surviving candidates, an unsupported-scope refusal, or a named no-data outcome. ``coverage`` is the request's snapshot @@ -371,6 +373,7 @@ def materialise( discover_scope=discover_scope, open_axes=open_axes, coverage_snapshot=coverage, + within=within, ) except RoleEntityUnplaced: return NoDataDraft(reason="role_incompatible") diff --git a/src/search/engine/retrieval/role_resolve.py b/src/search/engine/retrieval/role_resolve.py index 6e2e08455..bfcc46ff0 100644 --- a/src/search/engine/retrieval/role_resolve.py +++ b/src/search/engine/retrieval/role_resolve.py @@ -517,6 +517,7 @@ def _confirm_role_bearing( date_request: DateRequest | None, coverage_snapshot: CoverageSnapshot, facet_label: str, + within: Mapping[RoleAxis, frozenset[str]], ) -> Materialised | NoDataDraft | UnsupportedDraft | _RoleBearingConfirmation: """Confirm members against coverage's fixed recipient/donor role labels. @@ -548,6 +549,7 @@ def _confirm_role_bearing( date_request=date_request, coverage_snapshot=coverage_snapshot, facet_label=facet_label, + within=within.get("origin"), ) if subject_dcids or not origin_dcids: return NoDataDraft(reason="role_incompatible") @@ -567,6 +569,7 @@ def _confirm_role_bearing( date_request=date_request, coverage_snapshot=coverage_snapshot, facet_label=facet_label, + within=within.get("destination"), ) if companion_open_axis is not None and isinstance(destination_result, Materialised): destination_result.also_defaulted_axis = companion_open_axis @@ -972,6 +975,7 @@ def graph_confirm_resolve( discover_scope: bool = False, open_axes: frozenset[RoleAxis] = frozenset(), coverage_snapshot: CoverageSnapshot, + within: Mapping[RoleAxis, frozenset[str]] | None = None, ) -> Materialised | NoDataDraft | UnsupportedDraft: """Filter a role-bearing shape's members by bound values, then confirm observations. @@ -996,7 +1000,9 @@ def graph_confirm_resolve( answers with the twin, and one whose members all lack the axis elects as it would without it (`statvar_members.electing_role_keys`). The result's ``open_axes`` holds the identity's open roles (`_AdmittedPopulation.open_axes`), as does a no-data - outcome over the elected members (`NoDataDraft.open_axes`). + outcome over the elected members (`NoDataDraft.open_axes`). ``within`` holds, by flow + axis, the members an open axis is discovered within (`role_scope.discover_role_scope`): + the population a composite binds the role to, which the caller posted open. Returns ``Materialised``, ``NoDataDraft``, or ``UnsupportedDraft``. A bound destination with no observations returns an unavailable ``Materialised`` @@ -1092,6 +1098,7 @@ def graph_confirm_resolve( date_request=date_request, coverage_snapshot=coverage_snapshot, facet_label=facet_label, + within=within or {}, ) if not isinstance(confirmation, _RoleBearingConfirmation): # The elected members declare the open axes whether or not they hold data, so a diff --git a/src/search/engine/retrieval/role_scope.py b/src/search/engine/retrieval/role_scope.py index 489c2ebbc..27748cf15 100644 --- a/src/search/engine/retrieval/role_scope.py +++ b/src/search/engine/retrieval/role_scope.py @@ -772,6 +772,7 @@ def discover_role_scope( date_request: DateRequest | None, coverage_snapshot: CoverageSnapshot, facet_label: str = DEFAULT_FACET_LABEL, + within: frozenset[str] | None = None, ) -> Materialised | NoDataDraft | UnsupportedDraft: """Discover members on an open role axis and resolve them from coverage. @@ -797,6 +798,11 @@ def discover_role_scope( lookup confirms only members holding a point in those years, so the plan always serves one. + ``within`` narrows the open axis to those parties: the population a composite binds the + role to (`code_index.composites.ComponentIdentity.populations`), which the caller posted + open. It joins the one bound every read below takes, so the members, the plan, the + coverage and the headline all count the population alone. + Opposite-role and time filters narrow the lookup. Bound facet filters are recorded for echo only because coverage has no per-row facet metadata. Candidates come from real ``pair_rows`` with facet metadata; each confirmed @@ -827,9 +833,10 @@ def discover_role_scope( coverage_index = coverage_snapshot.index discover_coverage_role = _COVERAGE_ROLE_FOR_AXIS[open_flow_axis] opposite_coverage_role = _COVERAGE_ROLE_FOR_AXIS[_opposite_flow_axis(open_flow_axis)] - bound: BoundRoles = ( - {opposite_coverage_role: frozenset(opposite_dcids)} if opposite_dcids else {} - ) + bound: BoundRoles = { + **({opposite_coverage_role: frozenset(opposite_dcids)} if opposite_dcids else {}), + **({discover_coverage_role: within} if within is not None else {}), + } check_minted_role_members( coverage_index, minted=surviving_svs, member_facts=member_facts, bound=bound ) diff --git a/src/search/engine/same_shape.py b/src/search/engine/same_shape.py index 59e0d1082..8e1e5cd55 100644 --- a/src/search/engine/same_shape.py +++ b/src/search/engine/same_shape.py @@ -18,7 +18,7 @@ from contextlib import nullcontext from typing import TYPE_CHECKING, NamedTuple -from code_index.composites import composite_ref +from code_index.composites import binds_composite from code_index.shared import get_index from dfo_platform.request_deadline import bounded_deadline, request_deadline from search.engine.config import settings @@ -715,9 +715,9 @@ def _collapse_group( def _binds_composite(spec: Spec) -> bool: - """Whether `spec`'s identity binds a declared composite provenance (`composite_ref`).""" - facets = spec.definition.identity.model_dump(mode="json", include={"facets"}) - return composite_ref(facets) is not None + """Whether `spec`'s identity binds a declared composite provenance + (`code_index.composites.binds_composite`).""" + return binds_composite(spec.definition.identity.model_dump(mode="json", include={"facets"})) def _check_merge_gates(group: list[RegionResult], base_spec: Spec) -> None: diff --git a/src/search/heading/departure_fact.py b/src/search/heading/departure_fact.py index 8254c17b1..5fd2a2f42 100644 --- a/src/search/heading/departure_fact.py +++ b/src/search/heading/departure_fact.py @@ -20,6 +20,11 @@ another source the query names, which lists it as a party; its words resolving to no single source; or none of its own data leading the recall of the source reading.""" +type ContributionRecipient = Literal["party", "total", "residual"] +"""What the multilateral recipient of a `core_contribution_read` answer is +(`code_index.types.PartyKind`): a multilateral organisation, the DAC's total over +multilateral agencies, or the DAC's line for the agencies it does not name.""" + type UndirectedRule = Literal["complement", "operation", "declaration", "coverage"] """Why a place named without a direction is read on one flow axis (`region.undirected.settle_undirected_axis`): the part names its other places on the other @@ -165,6 +170,8 @@ class DepartureFact: value"). - `left_out` (`measure_not_applied`): `asked` is the request's words for something it asked to leave out ("excluding humanitarian aid"), which the answer does not. + - `recipient` (`core_contribution_read`): what the multilateral recipient `shown` names + is, which the text words its figures by. """ code: DepartureCode @@ -215,6 +222,7 @@ class DepartureFact: left_out: bool = False read_by: UndirectedRule | None = None listed_by: str | None = None + recipient: ContributionRecipient | None = None def ordered_facts(facts: Iterable[DepartureFact]) -> tuple[DepartureFact, ...]: diff --git a/src/search/heading/pins.py b/src/search/heading/pins.py index 4280d3fe2..068d9160d 100644 --- a/src/search/heading/pins.py +++ b/src/search/heading/pins.py @@ -9,7 +9,7 @@ from __future__ import annotations -from code_index.composites import table_identity +from code_index.composites import PopulationUnavailable, table_identity from code_index.members import elected_pins, identity_family_query from code_index.shared import get_index from search.models import BaseIdentity, IdentityShapeSchema @@ -18,10 +18,20 @@ def identity_pinned_properties(identity: BaseIdentity) -> frozenset[str]: """The properties the dataset elected for `identity`'s family pins, or for a composite those of the identity its option was set on (`code_index.composites.table_identity`). - Empty for an identity that names its one StatVar.""" + Refuses a composite whose population is unavailable as Search's doors refuse it + (`identity_edit.population_refusal`). Empty for an identity that names its one + StatVar.""" if not isinstance(identity.shape, IdentityShapeSchema): return frozenset() index = get_index() - document = table_identity(index, identity.model_dump(mode="json", exclude_none=True)) + try: + document = table_identity( + index, identity.model_dump(mode="json", exclude_none=True) + ).identity + except PopulationUnavailable as exc: + # Importing search.engine runs its package init, which imports this module. + from search.engine.identity_edit import population_refusal + + raise population_refusal(exc, pointer="/input/identity") from exc query = identity_family_query(index, document) return frozenset(elected_pins(index, query)) diff --git a/src/search/models/departure.py b/src/search/models/departure.py index 80d204e2f..8b1a20baf 100644 --- a/src/search/models/departure.py +++ b/src/search/models/departure.py @@ -97,6 +97,9 @@ class Departure(StrictModel): "because its words name no single source, or because none of the data it " "publishes measures what the request asked; asked is the request's words for it, " "and the text says which reason. " + "'core_contribution_read': the answer names a multilateral organisation as the " + "recipient, so its figures are the donor's core contributions to it, which total " + "ODA counts and a bilateral reading leaves out; shown is the organisation's name. " "'scope_limited': a no_data answer's check covered " "only values the engine chose, which the request did not name; its text " "names only what the check covered ('This check covered only ODA.')" diff --git a/src/search/models/envelope.py b/src/search/models/envelope.py index 057f8d299..f8a950db4 100644 --- a/src/search/models/envelope.py +++ b/src/search/models/envelope.py @@ -36,8 +36,8 @@ class Warning(StrictModel): """A run-health signal. Note: this name shadows the Python builtin Warning. Only `observations_series_truncated`, `elected_shape_no_data`, `un_system_unavailable`, - `whole_total_available`, `currency_unmapped_omitted`, `deflator_missing_omitted` and - `deflator_projected` set `params`. A `None` `params` is left out of every dump, so other codes' warnings stay + `whole_total_available`, `bilateral_reading_unavailable`, `currency_unmapped_omitted`, + `deflator_missing_omitted`, `deflator_projected` and `role_scope_defaulted` set `params`. A `None` `params` is left out of every dump, so other codes' warnings stay exactly `{code, severity, message}` plus `spec_id` where one applies. """ @@ -74,7 +74,11 @@ class Warning(StrictModel): "unavailable. whole_total_available: {total, total_label, whole_of}, the dcid " "and label of the total the source publishes for the side the answer lists " "member by member, and the classes of party it covers, comma-joined (official, " - "private, residual). currency_unmapped_omitted: {count, entities, entity_labels}, " + "private, residual). bilateral_reading_unavailable: {why}, why a DAC2a total " + "ODA answer could not read its bilateral destinations alone: " + "destination_unclassified, option_not_covered, scope_undecided, or the slot " + "the bilateral composite cannot serve. " + "currency_unmapped_omitted: {count, entities, entity_labels}, " "how many entities the conversion left out, the dcids of the first ten, sorted, " "and each one's name, in entities order, '' where the conversion tables hold " "none, which is usual here, since an entity no currency names, such as a " @@ -84,8 +88,9 @@ class Warning(StrictModel): "in entities order; a warning without params says the loaded tables cannot tell " "which entities. deflator_projected: {count, examples}, how many rows took a " "projected rate or deflator, and at most ten distinct entity@year keys among them, " - "sorted; count counts rows, not keys, and examples may leave keys out. Omitted " - "for every other code" + "sorted; count counts rows, not keys, and examples may leave keys out. " + "role_scope_defaulted: {axis}, the open role axis the answer covers in full, " + "origin or destination. Omitted for every other code" ), ) diff --git a/src/search/models/request.py b/src/search/models/request.py index 9da255a4b..b25077ba4 100644 --- a/src/search/models/request.py +++ b/src/search/models/request.py @@ -6,11 +6,14 @@ from pydantic import ( Field, + InstanceOf, ValidationInfo, field_validator, model_validator, ) +from code_index.scope_conventions import RolePopulation + from .assumption import Assumption from .base import StrictModel from .entity import EntityGroup @@ -614,6 +617,13 @@ class ResubmitIdentityPayload(StrictModel): `entity_groups` names the group a role's entities were asked for as. The engine keeps a group the graph confirms for the identity's entities on that role (`resubmit_request.resubmit_named_groups`). + + `populations` maps each `entities` key of `identity` that binds a population in place + of names a caller posted (`code_index.composites.TableIdentity.populations`) to that + population. The resolver reads such a role as open within the population's members + (`region.resubmit.resolve_spec_resubmit`). Only the engine sets it, from a composite's + table or component identity, so it stays out of every dump; the part memo keys it by + the dataset whose population it is (`response_cache_keys.part_memo_key`). """ identity: BaseIdentity = Field( @@ -624,6 +634,14 @@ class ResubmitIdentityPayload(StrictModel): ) ) entity_groups: ResubmitEntityGroups + populations: dict[str, InstanceOf[RolePopulation]] = Field( + default_factory=dict, + exclude=True, + description=( + "the identity's entities keys that bind a population, each read as open within " + "the population's members; set by the engine alone" + ), + ) # --------------------------------------------------------------------------- diff --git a/src/search/models/response.py b/src/search/models/response.py index f4d66acfa..415458b38 100644 --- a/src/search/models/response.py +++ b/src/search/models/response.py @@ -173,7 +173,8 @@ class _PartBase(StrictModel): "entity_unavailable, member_unavailable, year_unavailable, modelled_values, " "group_read_as_members, whole_total_read, whole_total_unpublished, " "direction_discarded, organisation_read_as_source, " - "direction_inferred, organisation_read_as_party, scope_limited. [] when the answer is what was asked. A " + "direction_inferred, organisation_read_as_party, core_contribution_read, " + "scope_limited. [] when the answer is what was asked. A " "client that renders its own sentence reads them here; each one's text " "also closes the sentence that carries this part (see Departure.text). " "A warning on the same part reports run health and stays separate" diff --git a/src/search/models/scope.py b/src/search/models/scope.py index 4cc574909..3f20afe96 100644 --- a/src/search/models/scope.py +++ b/src/search/models/scope.py @@ -157,9 +157,10 @@ class ProviderBinding(StrictModel): provenance: str | None = Field( default=None, description=( - "the composite provenance the inclusive scope binds as the identity's source, " - "which stacks the part's bilateral flows on the imputed multilateral line of its " - "own table; None for every other scope" + "the composite provenance the option binds as the identity's source: for the " + "inclusive scope, the part's bilateral flows stacked on the imputed multilateral " + "line of its own table; for the bilateral scope on DAC2a's total ODA, its " + "bilateral destinations alone; None for every other option" ), ) @@ -185,8 +186,9 @@ class OptionAvailability(StrictModel): default_factory=dict, description=( "the code's parameters: `why` for provider_not_offered, `dataset` for " - "basis_not_published and contributions_counted, " - "`entry` and `cause` for conflicts_with" + "basis_not_published and contributions_counted, `destination` for " + "multilateral_destination, the multilateral agency the part names as recipient, " + "whose flows are core contributions, `entry` and `cause` for conflicts_with" ), ) diff --git a/src/search/models/spec.py b/src/search/models/spec.py index ec7512368..53f833aec 100644 --- a/src/search/models/spec.py +++ b/src/search/models/spec.py @@ -974,9 +974,11 @@ class ResolvedSpec(StrictModel): "entries applied, and the identity they compile to (see PartScope). To switch " "an option, post base.identity as identity on a dataset_resubmit identity " "entry, with entries as scope_entries, the entry on the switched axis " - "replaced by the switch: one entry per axis. None when no axis was settled " - "and no scope entries were " - "posted. Never a member of BaseIdentity, never hashed." + "replaced by the switch: one entry per axis. Empty choices when no axis was " + "settled but the part is served from a composite, which compiled.identity and " + "compiled.components name (a DAC2a ranking read to its bilateral destinations). " + "None when no axis was settled, no composite serves the part and no scope " + "entries were posted. Never a member of BaseIdentity, never hashed." ), ) # The named entities the answer's read plan serves no point at, each with the role it diff --git a/src/search/models/vocab.py b/src/search/models/vocab.py index 8953a7df5..971d99d08 100644 --- a/src/search/models/vocab.py +++ b/src/search/models/vocab.py @@ -130,6 +130,7 @@ def _require_schema_version_int(value: object) -> object: "organisation_read_as_source", "direction_inferred", "organisation_read_as_party", + "core_contribution_read", "scope_limited", ] """Every way an answer can differ from what its request asked (see @@ -162,6 +163,7 @@ def _require_schema_version_int(value: object) -> object: "stated_choice_not_applied", "option_not_covered", "contributions_counted", + "multilateral_destination", ] """Every reason a scope option is partial or unavailable, or a carried or stated entry is not applied (see `models.scope.OptionAvailability`, `models.scope.DroppedEntry`). A closed vocabulary diff --git a/src/search/render/departures.py b/src/search/render/departures.py index ec8fd54f2..24c06e106 100644 --- a/src/search/render/departures.py +++ b/src/search/render/departures.py @@ -674,6 +674,34 @@ def _organisation_read_as_party( raise ValueError("organisation_read_as_party needs the reason for the reading") +def _core_contribution_read( + fact: DepartureFact, _family: FlowFamily | None, shows_data: bool +) -> str: + """The multilateral recipient the answer reads (`shown`), whose figures are the donor's + core contributions to it, worded by what it is (`recipient`): a multilateral + organisation, the DAC's total over multilateral agencies, or the DAC's line for the + agencies it does not name.""" + if fact.shown is None: + raise ValueError("core_contribution_read needs the recipient the answer reads") + figures = "these figures are" if shows_data else "its figures would be" + match fact.recipient: + case "party" | None: + return ( + f"{fact.shown} is a multilateral organisation, so {figures} core " + "contributions to it, counted in total ODA." + ) + case "total": + return ( + f"{fact.shown} is the DAC's total over multilateral agencies, so {figures} " + "core contributions to them, counted in total ODA." + ) + case "residual": + return ( + f"{fact.shown} is the DAC's line for multilateral agencies it does not name, " + f"so {figures} core contributions to them, counted in total ODA." + ) + + def _scope_limited(fact: DepartureFact, _family: FlowFamily | None, _shows_data: bool) -> str: return no_data_limit_text(fact.limits) @@ -702,6 +730,7 @@ def _asked(fact: DepartureFact) -> str: "organisation_read_as_source": _organisation_read_as_source, "direction_inferred": _direction_inferred, "organisation_read_as_party": _organisation_read_as_party, + "core_contribution_read": _core_contribution_read, "scope_limited": _scope_limited, } ) diff --git a/src/search/scope/composite.py b/src/search/scope/composite.py index 5d81dd8cb..afa0d5001 100644 --- a/src/search/scope/composite.py +++ b/src/search/scope/composite.py @@ -22,6 +22,7 @@ from __future__ import annotations import json +import logging from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from typing import cast @@ -37,7 +38,9 @@ from code_index.members import SourceNotInFamily, identity_family_query, index_family from code_index.query import CodeIndex from code_index.types import CompositeDeclaration +from dfo_platform.roles import role_key_axis from materialise.composite import spans, stacked_years, table_years +from materialise.enrichment import party_count, party_kind_of from search.engine.retrieval.bound_values import served_members from search.models import ( BaseIdentity, @@ -53,8 +56,11 @@ ObservationsOmissionReason, PreviewOrder, ProvenanceRef, + ResolvedEntity, ResolvedFacet, + ResolvedSpec, ResolvedStatVars, + RoleAxis, SlotValue, Spec, SpecDefinition, @@ -66,7 +72,11 @@ from search.render.labels import join_labels from search.scope.tables import ScopeFacts from series_coverage.coverage_types import BoundRoles, Role +from series_coverage.coverage_types import Headline as CoverageHeadline from series_coverage.index import CoverageIndex, elected_year +from series_coverage.schema import ABSENT_ENTITY + +logger = logging.getLogger(__name__) _ROLES: Mapping[str, Role] = {"origin": "donor", "destination": "recipient"} """The coverage role each flow role of an identity's entities reads.""" @@ -188,13 +198,17 @@ def _years_asked(window: Mapping[str, object], coverage: CoverageIndex) -> tuple def _bound(identity: Mapping[str, object]) -> BoundRoles: - """The places `identity` binds on each coverage role.""" + """The places `identity` binds on each coverage role, every entities key read by the + flow axis it names (`role_key_axis`), as `code_index.composites` binds a component's + destinations.""" entities = cast("Mapping[str, Sequence[str]]", identity.get("entities") or {}) - return { - _ROLES[role]: frozenset(dcids) - for role, dcids in entities.items() - if role in _ROLES and dcids - } + bound: dict[Role, frozenset[str]] = {} + for key, dcids in entities.items(): + axis = role_key_axis(key) + if axis is not None and dcids: + role = _ROLES[axis] + bound[role] = bound.get(role, frozenset()) | frozenset(dcids) + return bound def table_document(identity: BaseIdentity, index: CodeIndex) -> dict[str, object] | None: @@ -202,7 +216,9 @@ def table_document(identity: BaseIdentity, index: CodeIndex) -> dict[str, object (`code_index.composites.table_identity`): itself, or the identity a composite's option was set on. None for a composite whose components cannot be derived.""" try: - return dict(table_identity(index, identity.model_dump(mode="json", exclude_none=True))) + return dict( + table_identity(index, identity.model_dump(mode="json", exclude_none=True)).identity + ) except CompositeNotServable: return None @@ -413,6 +429,9 @@ def assemble_composite( component tables give; its sources are its component tables, whose labels its `published_tables` joins, and its measure label reads the declaration's `measure_noun` (`_measure_label`), so its sentence cites both tables and names the composite's measure; + its open role axes (`ResolvedSpec.scope`, `role_axis_facts`) list and count the parties + of every component's series in the years it serves (`_open_axes`), as the mint's frame + holds them, where the table identity's answer read the first component's table alone; its concept label stays the table identity's answer's, which names no basis; its download facts are the composite file's (`composite_download`); and it carries no observation preview, since the table identity's reads one table. The engine reads the composite's @@ -443,10 +462,9 @@ def assemble_composite( ), "facets": facets, "observations": None, - "headline": _headline(served, document, spec.resolved.headline, facts=facts), - "stat_vars": _stat_vars( - served, spec.resolved.stat_vars, document, declaration, facts=facts - ), + "headline": _headline(served, spec.resolved.headline, facts=facts), + **_open_axes(spec.resolved, served, declaration, facts=facts), + "stat_vars": _stat_vars(served, spec.resolved.stat_vars, declaration, facts=facts), "slots": [ slot.model_copy(update={"binding": _window_binding(window)}) if slot.key.axis == "when" and window is not None @@ -469,6 +487,117 @@ def assemble_composite( return restated, _components(served) +def _open_axes( + stated: ResolvedSpec, + served: Sequence[ServedComponent], + declaration: CompositeDeclaration, + *, + facts: ScopeFacts, +) -> dict[str, object]: + """`stated`'s open role axes, the table identity's answer's, restated over every + component of the composite `declaration`: its scope lists, and each role axis fact + counts (`enrichment.party_count`, in the components' datasets), the parties on the + axis's coverage role of the series every component writes in the years it serves, each + read under its own places (`_served_parties`). A stacked composite's imputed line can + hold recipients its bilateral line lacks, which the table identity's answer, reading + the bilateral table alone, leaves out. A scope member the answer listed keeps its label + and type, and one only another component holds is labelled from the code index. Empty + where the answer leaves no axis open, or where the snapshot cannot list the parties, + which is logged at WARNING with the components it cannot list, since the answer then + counts the table identity's parties alone.""" + if stated.scope is None and not stated.role_axis_facts: + return {} + coverage = facts.coverage.index + unlisted = [item for item in served if item.served and not _lists_parties(item, coverage)] + if unlisted: + logger.warning( + "composite %s keeps the open axes of its table read: the snapshot cannot list " + "the parties of %s", + declaration.provenance, + ", ".join(item.component.recipe.dataset for item in unlisted), + ) + return {} + held = _served_parties(served, coverage) + index = facts.index + datasets = [recipe.dataset for recipe in declaration.components] + restated: dict[str, object] = {} + if stated.role_axis_facts: + restated["role_axis_facts"] = [ + fact.model_copy( + update={ + "member_count": party_count( + held[_ROLES[fact.role]], + axis=fact.role, + parties=index.parties, + datasets=datasets, + ) + } + ) + if fact.member_count is not None + else fact + for fact in stated.role_axis_facts + ] + scope = stated.scope + if scope is not None and scope.axis != "observationAbout": + role = _ROLES[scope.axis] + listed = {member.ref: member for member in scope.members} + restated["scope"] = scope.model_copy( + update={ + "members": [ + listed.get(dcid) + or ResolvedEntity( + ref=dcid, + label=row.label if (row := index.parties.get(dcid)) else None, + role=scope.axis, + ) + for dcid in _parties( + held[role], axis=scope.axis, index=index, datasets=datasets + ) + ] + } + ) + return restated + + +def _parties( + dcids: Iterable[str], *, axis: RoleAxis, index: CodeIndex, datasets: Sequence[str] +) -> list[str]: + """`dcids`, members of a role on `axis`, without the totals `party_count` leaves out of + its count, in order: those the code index declares a `total` on `axis` in `datasets` + (`party_kind_of`), unless every member is one. The scope lists the parties its role axis + fact counts.""" + ordered = sorted(dcids) + kept = [ + dcid + for dcid in ordered + if party_kind_of(dcid, axis=axis, parties=index.parties, datasets=datasets) != "total" + ] + return kept or ordered + + +def _lists_parties(item: ServedComponent, coverage: CoverageIndex) -> bool: + """Whether the snapshot lists the parties of the series `item` writes: it settles its + members and holds a row of each (`CoverageIndex.covers`), as `composite_download` + counts rows only there.""" + return item.members is not None and all(coverage.covers(sv) for sv in item.members) + + +def _served_parties( + served: Sequence[ServedComponent], coverage: CoverageIndex +) -> dict[Role, frozenset[str]]: + """The donors and recipients of the series each of `served` writes in the years it + serves (`_served_series`), as the mint's frame holds them, for components whose parties + the snapshot lists (`_lists_parties`).""" + parties: dict[Role, set[str]] = {"recipient": set(), "donor": set()} + for item in served: + if not item.served: + continue + for (_variable, recipient, donor), _held in _served_series(item, coverage): + parties["recipient"].add(recipient) + parties["donor"].add(donor) + return {role: frozenset(held - {ABSENT_ENTITY}) for role, held in parties.items()} + + def _measure_label(declaration: CompositeDeclaration, measure: str | None) -> str | None: """The measure label a composite answer's sentence names: a stacked composite's `measure_noun` after `measure`, the label of the measure its components share, and a @@ -481,11 +610,15 @@ def _measure_label(declaration: CompositeDeclaration, measure: str | None) -> st def _component_sources(index: CodeIndex, declaration: CompositeDeclaration) -> list[ProvenanceRef]: """The tables a composite is served from, in component order, once each, as the - provenance refs a sentence's byline and a citation name (`observations.provenance_refs`).""" + provenance refs a sentence's byline and a citation name (`observations.provenance_refs`), + each under its declared label, as a plain answer names its table + (`spec_build.elected_sources`).""" datasets = [recipe.dataset for recipe in declaration.components] - return provenance_refs( - datasets, {dataset: index.datasets[dataset].label for dataset in datasets} - ) + labels = {dataset: index.datasets[dataset].label for dataset in datasets} + return [ + ref.model_copy(update={"label": labels[ref.ref]}) + for ref in provenance_refs(datasets, labels) + ] def _window_binding(window: TimeWindow) -> BindingValue: @@ -536,22 +669,22 @@ def _with_data( def _stat_vars( served: Sequence[ServedComponent], stated: ResolvedStatVars, - identity: Mapping[str, object], declaration: CompositeDeclaration, *, facts: ScopeFacts, ) -> ResolvedStatVars: """`stated`, the table identity's StatVar counts, restated over every component of the - composite `declaration`. The member count is the StatVars holding a series in the years - their component serves, the served count sums the StatVars a mint of each component - serves (None where the index does not settle some component's members), the served date + composite `declaration`. The member count is the StatVars holding a series under their + component's own places (`_bound`) in the years it serves, the served count sums the + StatVars a mint of each component serves (None where the index does not settle some + component's members), the served date range spans the years the components serve, and `sums_components` states whether a groupby sum adds the components (`code_index.composites.components_sum`), the rule the mint sums by.""" years = sorted({year for item in served for year in item.served}) - bound = _bound(identity) confirmed = sum( - len(_with_data(item, bound, facts.coverage.index, item.served)) for item in served + len(_with_data(item, _bound(item.component.identity), facts.coverage.index, item.served)) + for item in served ) counts = [len(item.members) if item.members is not None else None for item in served] return stated.model_copy( @@ -675,50 +808,29 @@ def _shared_licence(served: Sequence[ServedComponent]) -> DownloadLicence | None def _headline( served: Sequence[ServedComponent], - identity: Mapping[str, object], stated: Headline | None, *, facts: ScopeFacts, ) -> Headline | None: """The composite's headline: the latest year any component serves, counted over the - rows of every component serving it (`CoverageIndex.headline` over their StatVars, one - year at a time), with the same counts for it and the two years before it among those - the composite serves. The named window is `stated`'s, the request's. None when no - component serves a year or settles its members.""" + rows of every component serving it, each under its own places (`_headline_year`), with + the same counts for it and the two years before it among those the composite serves. A + component reads its own places, so one that binds its destinations to a population + (`ComponentIdentity.populations`) counts that population alone. The named window is + `stated`'s, the request's. None when no component serves a year or settles its + members.""" coverage = facts.coverage.index - bound = _bound(identity) years = sorted({year for item in served for year in item.served}, reverse=True)[:3] counted: list[HeadlineYear] = [] for year in years: - variables = sorted( - { - variable - for item in served - if year in item.served and item.members is not None - for variable in item.members - } - ) - found = ( - coverage.headline(variables=variables, bound=bound, first_year=year, last_year=year) - if variables - else None - ) - if found is None or found.year != year: - continue - names = coverage.role_names.get(variables[0]) - role_dcids = {} if names is None else {"donor": names.donor, "recipient": names.recipient} - counted.append( - HeadlineYear( - year=year, - observations=found.observations, - series=found.series, - role_counts={ - dcid: count - for role, count in found.role_counts.items() - if (dcid := role_dcids.get(role)) is not None - }, - ) - ) + reads: dict[tuple[tuple[Role, frozenset[str]], ...], set[str]] = {} + for item in served: + if year in item.served and item.members is not None: + bound = _bound(item.component.identity) + reads.setdefault(tuple(sorted(bound.items())), set()).update(item.members) + found = _headline_year(coverage, reads, year) + if found is not None: + counted.append(found) if not counted: return None head = counted[0] @@ -731,3 +843,61 @@ def _headline( role_counts=head.role_counts, recent_years=counted, ) + + +def _headline_year( + coverage: CoverageIndex, + reads: Mapping[tuple[tuple[Role, frozenset[str]], ...], set[str]], + year: int, +) -> HeadlineYear | None: + """The counts of `year` over `reads`, the StatVars of the components serving it grouped + by the places they bind (`_bound`, as sorted items): `CoverageIndex.headline` of one + group, or across several the summed observations and series of each group's, whose + StatVars differ, and the distinct parties on each role across all of their series in + the year. None when no row holds `year`.""" + found: list[tuple[CoverageHeadline, BoundRoles, list[str]]] = [] + for key, members in reads.items(): + variables = sorted(members) + bound: BoundRoles = dict(key) + held = coverage.headline(variables=variables, bound=bound, first_year=year, last_year=year) + if held is not None and held.year == year: + found.append((held, bound, variables)) + if not found: + return None + names = coverage.role_names.get(found[0][2][0]) + role_dcids = {} if names is None else {"donor": names.donor, "recipient": names.recipient} + if len(found) == 1: + role_counts = found[0][0].role_counts + else: + role_counts = _distinct_parties(coverage, [(b, v) for _, b, v in found], year) + return HeadlineYear( + year=year, + observations=sum(held.observations for held, _, _ in found), + series=sum(held.series for held, _, _ in found), + role_counts={ + dcid: count + for role, count in role_counts.items() + if (dcid := role_dcids.get(role)) is not None + }, + ) + + +def _distinct_parties( + coverage: CoverageIndex, reads: Sequence[tuple[BoundRoles, list[str]]], year: int +) -> dict[Role, int]: + """The distinct donors and recipients of the series `reads` hold in `year`, each read + under its own bound, by coverage role; a role no series carries a party on is left + out.""" + parties: dict[Role, set[str]] = {"recipient": set(), "donor": set()} + for bound, variables in reads: + found = coverage.series_years( + variables=variables, bound=bound, first_year=year, last_year=year + ) + if found is None: + continue + series = found.keys() + for _variable, recipient, donor in series: + parties["recipient"].add(recipient) + parties["donor"].add(donor) + counts = {role: len(held - {ABSENT_ENTITY}) for role, held in parties.items()} + return {role: count for role, count in counts.items() if count} diff --git a/src/search/scope/edits.py b/src/search/scope/edits.py index 2e0b9a03b..cc59b0e7d 100644 --- a/src/search/scope/edits.py +++ b/src/search/scope/edits.py @@ -17,7 +17,9 @@ from code_index.composites import DECLARED_COMPOSITES, provenance_datasets from code_index.declarations import COMPOSITES from code_index.measure_basis import edit_basis +from code_index.scope_conventions import BILATERAL_SOURCES, BilateralSource from code_index.types import CompositeKind +from dfo_platform.roles import role_key_axis from search.models import ( BaseIdentity, EntitiesBinding, @@ -132,22 +134,66 @@ def _edit_shape( def _edit_provider( identity: BaseIdentity, binding: OptionBinding, siblings: Sequence[OptionBinding] ) -> BaseIdentity: - """Bind the stacked composite of bilateral and imputed multilateral spending the option - names (`ProviderBinding.provenance`) as the identity's source for the inclusive scope, - with the option's window when it sets one (`ProviderBinding.window`), or remove a stacked - composite for the bilateral scope, leaving the window as it is. The two are inverses on an - identity that binds no other provenance. The contributions and multilateral scopes are - held only by the published donor totals that count them, so they leave the identity as it - is. Raises `ScopeTableError` for an inclusive option that names no composite.""" + """Bind the stacked composite the option names (`ProviderBinding.provenance`) as the + identity's source, with the option's window when it sets one (`ProviderBinding.window`): + the bilateral and imputed multilateral spending of the inclusive scope, or the bilateral + destinations alone of the bilateral scope on a published total that counts core + contributions (`code_index.scope_conventions.BILATERAL_SOURCES`). Such a composite reads + a destination total of both sides as the total of the bilateral side + (`rebound_destinations`). An option naming no composite removes a stacked composite, + leaving the window as it is: the bilateral scope on a part the CRS serves, and the + contributions and multilateral scopes, held by the published donor totals that count + them. Binding and removing are inverses on an identity that binds no other provenance + and names no destination total of both sides. Raises `ScopeTableError` for an inclusive + option that names no composite.""" assert isinstance(binding, ProviderBinding) - if binding.scope == "bilateral": - return _with_source(identity, None, kind="stacked") - if binding.scope != "inclusive": - return identity if binding.provenance is None: - raise ScopeTableError("the inclusive option names no composite source") + if binding.scope == "inclusive": + raise ScopeTableError("the inclusive option names no composite source") + return _with_source(identity, None, kind="stacked") edited = _with_source(identity, binding.provenance, kind="stacked") - return _with_window(edited, binding.window) + return _with_window(rebound_destinations(edited, binding.provenance), binding.window) + + +def rebound_destinations(identity: BaseIdentity, provenance: str) -> BaseIdentity: + """`identity`, whose source is the composite `provenance`, naming the total of the + bilateral side in place of the total of both sides (`BilateralSource.rebound` for + `BilateralSource.spanning`) where a component of `provenance` reads the bilateral + destinations of the dataset the row declares (`ComponentRecipe.destinations`), whose + population holds no row of the total of both sides. `identity` itself otherwise.""" + composite = DECLARED_COMPOSITES.get(provenance) + if composite is None: + return identity + bound = { + recipe.dataset for recipe in composite.components if recipe.destinations == "bilateral" + } + rebound = {row.spanning: row.rebound for row in BILATERAL_SOURCES if row.dataset in bound} + entities = dict(identity.entities) + for key, held in identity.entities.items(): + if role_key_axis(key) == "destination" and rebound.keys() & set(held): + entities[key] = list(dict.fromkeys(rebound.get(dcid, dcid) for dcid in held)) + if entities == identity.entities: + return identity + return identity.model_copy(update={"entities": entities}) + + +def counted_bilateral_source( + identity: BaseIdentity, *, dataset_of: Callable[[BaseIdentity], str | None] +) -> BilateralSource | None: + """The bilateral source (`BILATERAL_SOURCES`) whose published total `identity`'s bound + slots count: the row whose code a slot binds, declared for the dataset `dataset_of` + elects for `identity`. `dataset_of` runs only where a slot binds a row's code, since + electing a dataset reads the code index. None where no row is.""" + bound = { + (slot.property, slot.binding.ref) + for slot in identity.slots + if isinstance(slot.binding, IdentityBindingValue) + } + sources = [row for row in BILATERAL_SOURCES if (row.code_system, row.code) in bound] + if not sources: + return None + dataset = dataset_of(identity) + return next((row for row in sources if row.dataset == dataset), None) def _with_window(identity: BaseIdentity, window: YearSpan | None) -> BaseIdentity: @@ -202,7 +248,9 @@ def _with_source( "slot": Edit(members=frozenset({"slots", "requested_window"}), apply=_edit_slot), "entities": Edit(members=frozenset({"entities"}), apply=_edit_entities), "shape": Edit(members=frozenset({"shape", "slots", "facets"}), apply=_edit_shape), - "provider": Edit(members=frozenset({"facets", "requested_window"}), apply=_edit_provider), + "provider": Edit( + members=frozenset({"facets", "entities", "requested_window"}), apply=_edit_provider + ), } """The edit of each binding kind.""" @@ -210,9 +258,9 @@ def _with_source( def applies_in_pass(binding: OptionBinding) -> bool: """Whether a fresh search applies an option binding `binding` to the identity it resolved: it binds a declared composite as the identity's source, the OECD headline - basis (`ShapeBinding.provenance`) or the inclusive provider scope, or it sets a provider - scope on the identity's source, as the bilateral scope removes the inclusive composite. - Any other option reads another identity, which a switch or resubmit serves.""" + basis (`ShapeBinding.provenance`) or a provider scope's composite, or it sets a provider + scope on the identity's source, as the bilateral scope on the CRS removes the inclusive + composite. Any other option reads another identity, which a switch or resubmit serves.""" return isinstance(binding, ProviderBinding) or ( isinstance(binding, ShapeBinding) and binding.provenance is not None ) diff --git a/src/search/scope/provider.py b/src/search/scope/provider.py index 395452d5d..1fbd5eedb 100644 --- a/src/search/scope/provider.py +++ b/src/search/scope/provider.py @@ -3,35 +3,46 @@ the core contributions alone. The inclusive scope binds a stacked composite of the part's bilateral flows and an imputed -multilateral line as the identity's source; the bilateral scope removes it. Each table that -offers the scope declares its composite and the measure bases it adds on +multilateral line as the identity's source; on the CRS the bilateral scope removes it. Each +table that offers the scope declares its composite and the measure bases it adds on (`code_index.scope_conventions.INCLUSIVE_SOURCES`): the CRS with ONE's imputed multilateral table on gross disbursements, and DAC2a's total ODA with its own imputed multilateral ODA on gross and net flows. The axis is present on a part the CRS serves and on a donor's published total (`provider_rule`). A published total holds the scope its declared row names -(`code_index.scope_conventions.PUBLISHED_PROVIDER_SCOPES`): DAC1's total ODA the -contributions scope, its bilateral ODA the bilateral scope and its multilateral ODA the -multilateral scope, and DAC2a's total ODA the bilateral scope. Each DAC1 total offers the +(`code_index.scope_conventions.PUBLISHED_PROVIDER_SCOPES`): DAC1's total ODA the contributions +scope, its bilateral ODA the bilateral scope and its multilateral ODA the multilateral scope, +and DAC2a's total ODA the contributions scope, since it lists the multilateral agencies among +its recipients and counts the donor's core contributions to them. Each DAC1 total offers the others' scopes as options binding their aid types. Such an option serves the years every donor -holding its own total in a year also holds the other in, or, for a part asking for the -latest year, the one year the pool's read elects (`_published_table`). Where a table declares -an inclusive source, the inclusive option serves the years both lines cover for the part -(`inclusive_reach`): available when they cover every year the bilateral answer holds, partial -with a shorter window of its own when they cover fewer, and, for a part asking for the latest -year, available over the year a `latest` read elects among those both cover. It is not -offered (`provider_not_offered`) where the imputed line cannot serve what the part binds, or -cannot count the providers it names (`_providers_not_offered`): a part naming no provider, -a provider the line carries no rows for, or a pool naming a multilateral whose outflows -the line attributes to its funders. On any other basis the source does not add on it -conflicts with the measure-basis option (`AxisTable.conflicts`). +holding its own total in a year also holds the other in, or, for a part asking for the latest +year, the one year the pool's read elects (`_published_table`). DAC2a's bilateral scope binds a +composite of its total ODA that reads the bilateral destinations alone +(`code_index.scope_conventions.BILATERAL_SOURCES`): countries, regions and the DAC's +unallocated lines. One rule reads the destinations a part names (`destination_rule`): on a part +naming a multilateral agency the bilateral and inclusive scopes are unavailable +(`multilateral_destination`, naming the agency), and on a part naming all recipients they read +the DAC's total of developing countries in its place, as their labels say (`_rebound_label`). +Over destinations on the bilateral side alone the total holds no core contribution, so a +composite reading offers no switch to it. Where a table declares an inclusive source, the +inclusive option serves the years both lines cover for the part (`inclusive_reach`): available +when they cover every year the bilateral answer holds, partial with a shorter window of its own +when they cover fewer, and, for a part asking for the latest year, available over the year a +`latest` read elects among those both cover. It is not offered (`provider_not_offered`) where +the imputed line cannot serve what the part binds, or cannot count the providers it names +(`_providers_not_offered`): a part naming no provider, a provider the line carries no rows for, +or a pool naming a multilateral whose outflows the line attributes to its funders. On any other +basis the source does not add on it conflicts with the measure-basis option +(`AxisTable.conflicts`). The default is declared (`code_index.scope_conventions.PROVIDER_CONVENTIONS`): a named donor's total of a CRS sector or recipient part counts both, when they cover every year asked for. `settle_provider` applies it, or the part's explicit or carried entry, on the parsed door, -where the answer can serve it (`_served_in_pass`). A choice names the convention's rule only -where the request matched the convention and the part holds its scope. Otherwise it names -the rule of the option it holds, its own scope's (`AxisTable.rules`), as every resubmit -does. +where the answer can serve it (`_served_in_pass`). A fresh DAC2a part counting total ODA +reads its bilateral destinations before the axis is read (`bilateral_default_reading`), where +it names no destination or only bilateral ones, whatever donors it names. A choice names the +convention's rule only where the request matched the convention and the part holds its scope. +Otherwise it names the rule of the option it holds, its own scope's (`AxisTable.rules`), as +every resubmit does. """ from __future__ import annotations @@ -44,8 +55,10 @@ from code_index.composites import ( CompositeNotServable, + PopulationUnavailable, component_identities, composite_ref, + population_check, table_identity, without_composite, ) @@ -53,6 +66,7 @@ from code_index.query import CodeIndex from code_index.scope_conventions import ( BASIS_MEASURES, + BILATERAL_SOURCES, DONOR_ORGANISATIONS, INCLUSIVE_SOURCES, OPTION_LABELS, @@ -60,10 +74,13 @@ PROVIDER_SCOPE_RULES, PUBLISHED_PROVIDER_SCOPES, REDISTRIBUTED_PROVIDERS, + BilateralSource, InclusiveSource, ProviderConvention, PublishedProviderScope, + destination_side, ) +from dfo_platform.roles import role_key_axis from materialise.composite import spans from search.catalogue.counts import COUNTRY_PREFIX from search.engine.spec_id import identity_spec_id @@ -89,8 +106,18 @@ YearSpan, ) from search.scope.basis import held_measure, identity_part_shape -from search.scope.composite import component_years, donor_years, served_components -from search.scope.edits import applies_in_pass, apply_binding +from search.scope.composite import ( + ServedComponent, + component_years, + donor_years, + served_components, +) +from search.scope.edits import ( + applies_in_pass, + apply_binding, + counted_bilateral_source, + rebound_destinations, +) from search.scope.region import policy_members, policy_members_between from search.scope.state import AxisRecord, ScopeState from search.scope.tables import AxisTable, Conflict, ScopeFacts @@ -117,6 +144,13 @@ _SOURCES: Mapping[str, InclusiveSource] = {row.dataset: row for row in INCLUSIVE_SOURCES} """The inclusive source of each table that declares one, by the table's provenance.""" _SOURCE_COMPOSITES = frozenset(row.composite for row in INCLUSIVE_SOURCES) +_BILATERAL_SOURCES: Mapping[tuple[str, str, str], BilateralSource] = { + (row.dataset, row.code_system, row.code): row for row in BILATERAL_SOURCES +} +"""The bilateral source of each published total that declares one, by the total's dataset, +code system and code.""" +_PROVIDER_COMPOSITES = _SOURCE_COMPOSITES | frozenset(row.composite for row in BILATERAL_SOURCES) +"""Every composite a provider option binds as the identity's source.""" _PUBLISHED_DATASETS = frozenset(row.dataset for row in PUBLISHED_PROVIDER_SCOPES) """The tables whose provider axis their published rows declare, on those totals alone.""" _COUNTS_CONTRIBUTIONS: frozenset[ProviderScope] = frozenset({"contributions", "multilateral"}) @@ -162,13 +196,22 @@ def inclusive_reach(identity: BaseIdentity, facts: ScopeFacts, *, composite: str bilateral = next(c for c in served if c.component.role == "bilateral") covered = frozenset[int].intersection(*(c.served for c in served)) if document["requested_window"]["kind"] == "latest": - counts = {year: sum(c.years.counts[year] for c in served) for year in covered} return InclusiveReach( - asked=bilateral.years.held, covered=covered, latest=elected_year(counts) + asked=bilateral.years.held, covered=covered, latest=elected_year(_served_counts(served)) ) return InclusiveReach(asked=bilateral.years.held, covered=covered) +def _served_counts(served: Sequence[ServedComponent]) -> dict[int, int]: + """The series per year a composite serves, summed over its `served` components + (`composite.served_components`), in the years every component serves, so a `latest` read + of the composite and an option electing its year count alike.""" + if not served: + return {} + covered = frozenset[int].intersection(*(c.served for c in served)) + return {year: sum(c.years.counts.get(year, 0) for c in served) for year in covered} + + def _with_composite(document: Mapping[str, object], composite: str) -> dict[str, object]: """`document` binding `composite` as its source.""" facets = cast("Mapping[str, object]", document.get("facets") or {}) @@ -212,8 +255,8 @@ def _table(base: BaseIdentity, facts: ScopeFacts) -> AxisTable | None: return None if ref is not None and ref not in _SOURCE_COMPOSITES: return None - bilateral = _without_inclusive(base, ref) - dataset = _dataset(bilateral, facts) + bilateral = _without_provider_composite(base, ref) + dataset = _dataset(bilateral, facts.index) source = _SOURCES.get(dataset) if dataset not in _PUBLISHED_DATASETS else None if source is None or ref not in (None, source.composite): return None @@ -243,11 +286,11 @@ def _basis_conflicts(source: InclusiveSource) -> tuple[Conflict, ...]: ) -def _without_inclusive(identity: BaseIdentity, ref: str | None) -> BaseIdentity: - """`identity`, whose composite is `ref`, without an inclusive source's composite: the - identity its table serves its bilateral flows on. `identity` itself when `ref` is another - one.""" - if ref not in _SOURCE_COMPOSITES: +def _without_provider_composite(identity: BaseIdentity, ref: str | None) -> BaseIdentity: + """`identity`, whose composite is `ref`, without a composite a provider option binds (an + inclusive source's, or a bilateral source's on a published total): the identity of the + table it reads. `identity` itself when `ref` is another one.""" + if ref not in _PROVIDER_COMPOSITES: return identity return BaseIdentity.model_validate( without_composite(identity.model_dump(mode="json", exclude_none=True)) @@ -260,30 +303,47 @@ def _published_table( """The provider table of a part counting the published total `row` declares, whose identity is `identity` (`_published_row`). - The part holds `row`'s scope, or the inclusive one where `identity` binds its dataset's - inclusive source (`INCLUSIVE_SOURCES`). Every other row of its dataset and code system - (`PUBLISHED_PROVIDER_SCOPES`) is an option binding that row's code, so a switch to it - reads the total that counts its scope: on DAC1's total ODA, bilateral flows bind + The part holds `row`'s scope, the inclusive one where `identity` binds its dataset's + inclusive source (`INCLUSIVE_SOURCES`), or the bilateral one where it binds the bilateral + source of `row`'s total (`BILATERAL_SOURCES`). Every other row of its dataset and code + system (`PUBLISHED_PROVIDER_SCOPES`) is an option binding that row's code, so a switch to + it reads the total that counts its scope: on DAC1's total ODA, bilateral flows bind bilateral ODA and core contributions alone multilateral ODA, and on bilateral ODA, core contributions bind total ODA. Such an option serves the years every donor holding the part's total in a year also holds the sibling in (`_sibling_option`), or, on a part asking for the latest year, the one year the pool's read elects (`_latest_sibling_option`). - Where the dataset declares an inclusive source, the inclusive option is offered as on a - part the CRS serves (`_inclusive_option`), with its basis conflicts. Otherwise it is - unavailable with the total's dataset: `contributions_counted` on a total counting core - contributions, since it counts them in place of an imputed share of the multilaterals' - spending, and `provider_not_offered` (why `dataset`) on any other. Each option names its - row's rule, and the inclusive one its scope's. No preference but the held scope leads, so - no default moves the part to another total.""" + Where `row`'s total declares a bilateral source, the bilateral option binds its composite + (`_bilateral_option`), and the destinations the part names are read by one rule + (`destination_rule`). Where the dataset declares an inclusive source, the inclusive option + is offered as on a part the CRS serves (`_inclusive_option`), with its basis conflicts, + and under the same rule (`_published_inclusive`). Otherwise it is unavailable with the + total's dataset: `contributions_counted` on a total counting core contributions, since it + counts them in place of an imputed share of the multilaterals' spending, and + `provider_not_offered` (why `dataset`) on any other. Each option names its row's rule, + the bilateral source's its rule, and the inclusive one its scope's. A total that declares + a bilateral source names its own option by `BilateralSource.total_label`, and on a part + holding a composite over destinations on the bilateral side alone that option is not + offered (`provider_not_offered`, why `destination_bilateral`): those destinations hold no + core contribution, so it reads the composite's figures. No preference but the held scope + leads, so no default moves the part to another total: a fresh part reads the bilateral + source before its table is read (`bilateral_default_reading`).""" rows = { sibling.scope: sibling for sibling in PUBLISHED_PROVIDER_SCOPES if (sibling.dataset, sibling.code_system) == (row.dataset, row.code_system) } source = _SOURCES.get(row.dataset) + split = _BILATERAL_SOURCES.get((row.dataset, row.code_system, row.code)) ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) inclusive_held = source is not None and ref == source.composite - bilateral = _without_inclusive(identity, ref) + bilateral_held = split is not None and ref == split.composite + bilateral = _without_provider_composite(identity, ref) + rule = destination_rule(bilateral, split, facts) if split is not None else None + # Over bilateral-side destinations alone the total holds no core contribution, so a + # switch to it from a composite reads the same figures under another label. + total_alike = ( + rule is not None and rule.named and rule.bilateral and (bilateral_held or inclusive_held) + ) options: list[Option] = [] rules: dict[str, str] = {} own = _donor_years(bilateral, facts) @@ -294,15 +354,29 @@ def _published_table( ) for scope in get_args(ProviderScope): sibling = rows.get(scope) + option_id = provider_option_id(scope) + if scope == "bilateral" and rule is not None: + rules[option_id] = rule.source.rule + options.append(_bilateral_option(bilateral, rule, facts)) + continue if sibling is None and scope != "inclusive": continue - option_id = provider_option_id(scope) rules[option_id] = sibling.rule if sibling is not None else _SCOPE_RULES[scope] if sibling is None: - options.append(_published_inclusive(row, source, bilateral, facts, held=inclusive_held)) + options.append( + _published_inclusive(row, source, bilateral, facts, held=inclusive_held, rule=rule) + ) continue if sibling is row: - options.append(_option(scope, binding=ProviderBinding(scope=scope))) + label = split.total_label if split is not None else None + availability = ( + _unavailable("provider_not_offered", {"why": "destination_bilateral"}) + if total_alike + else None + ) + options.append( + _option(scope, availability, binding=ProviderBinding(scope=scope), label=label) + ) continue binding = SlotBinding(slot=sibling.code_system, state="value", refs=[sibling.code]) switched = apply_binding(bilateral, binding, []) @@ -314,7 +388,13 @@ def _published_table( scope, binding, own, sibling_years, _year_counts(switched, facts), own_year ) ) - held = INCLUSIVE if inclusive_held else provider_option_id(row.scope) + held = ( + INCLUSIVE + if inclusive_held + else BILATERAL + if bilateral_held + else provider_option_id(row.scope) + ) return AxisTable( axis="provider", options=tuple(options), @@ -326,6 +406,191 @@ def _published_table( ) +@dataclass(frozen=True, slots=True) +class DestinationRule: + """How the composite of `source` (`BilateralSource`), which reads the bilateral + destinations of `source`'s total alone, reads the destinations a part names, each placed + by `code_index.scope_conventions.destination_side` in `source`'s dataset: `multilateral`, + the first one on the multilateral side, None when none is, where the composite holds no + row and the part reads the total itself; `spanning`, `source`'s total + of both sides, None when the part does not name it, which the composite reads at the + total of the bilateral side in its place (`read`); `bilateral`, whether + the part names no destination or only ones on the bilateral side; and `named`, whether it + names any. A destination the index holds no row for, or that it cannot place, is on + neither side. + + One rule for a fresh part's provider options and its default (`_published_table`, + `with_bilateral_default`), read once per scope build (`destination_rule`). A posted + identity that binds the composite to a destination off its side is refused at every + Search door that reads a posted identity's table, the resubmit, candidate and options + doors (`identity_edit.composite_table_identity`).""" + + source: BilateralSource + multilateral: str | None + spanning: str | None + bilateral: bool + named: bool + + def read(self, identity: BaseIdentity) -> BaseIdentity: + """`identity` naming the total of the bilateral side in place of the total of both + sides where the part names it (`edits.rebound_destinations`), `identity` itself + otherwise.""" + if self.spanning is None: + return identity + return rebound_destinations(identity, self.source.composite) + + +def destination_rule( + identity: BaseIdentity, source: BilateralSource, facts: ScopeFacts +) -> DestinationRule: + """The `DestinationRule` of the destinations `identity` names against `source`, read in + `facts`' code index once per scope build for `identity` and `source`.""" + key = f"destination_rule:{source.composite}:" + identity.model_dump_json(exclude_none=True) + return facts.remembered(key, lambda: _destination_rule(identity, source, facts.index)) + + +def _destination_rule( + identity: BaseIdentity, source: BilateralSource, index: CodeIndex +) -> DestinationRule: + named = [ + dcid + for key, dcids in identity.entities.items() + if role_key_axis(key) == "destination" + for dcid in dcids + ] + sides = { + dcid: destination_side(party, datasets=(source.dataset,)) + if (party := index.parties.get(dcid)) is not None + else None + for dcid in named + } + return DestinationRule( + source=source, + multilateral=next((d for d, side in sides.items() if side == "multilateral"), None), + spanning=source.spanning if source.spanning in sides else None, + bilateral=all(side == "bilateral" for side in sides.values()), + named=bool(sides), + ) + + +def _population_unavailable(composite: str, facts: ScopeFacts) -> bool: + """Whether a component of `composite` reads a population of destinations that is + unavailable in `facts`' indexes (`code_index.composites.population_check`): a destination + the code index or the coverage snapshot holds that no side declaration places, which the + mint refuses to read.""" + return population_check(facts.index, facts.coverage.index).refusal(composite) is not None + + +def _bilateral_option(identity: BaseIdentity, rule: DestinationRule, facts: ScopeFacts) -> Option: + """The bilateral option binding the composite of `rule`'s source on a part whose identity + without a provider composite is `identity`, labelled by `BilateralSource.label`, read + once per scope build for `identity` and the source. It is unavailable + (`multilateral_destination`, naming the destination) where the part names a multilateral + destination, whose flows are core contributions the composite leaves out, and not + offered (`provider_not_offered`) where the composite's population is unavailable (why + `destination_unclassified`, `_population_unavailable`) or the composite refuses what + `identity` binds (`code_index.composites.CompositeNotServable`; why the slot at fault, as + `_inclusive_option` names it). Otherwise it serves the years its one component serves + (`_composite_years`), judged as a sibling total is (`_sibling_option`, + `_latest_sibling_option`, each `option_not_covered`): available when they cover every + year the part's answer holds, partial over their latest run when they cover fewer, and + unavailable when they cover none, as for a donor whose total ODA holds no breakdown by + recipient. Where the part names the total of both sides, the composite reads the total + of the bilateral side in its place, and its label says so (`_rebound_label`).""" + key = f"bilateral_option:{rule.source.composite}:" + identity.model_dump_json(exclude_none=True) + return facts.remembered(key, lambda: _read_bilateral_option(identity, rule, facts)) + + +def _read_bilateral_option( + identity: BaseIdentity, rule: DestinationRule, facts: ScopeFacts +) -> Option: + source = rule.source + binding = ProviderBinding(scope="bilateral", provenance=source.composite) + if rule.multilateral is not None: + return _option( + "bilateral", + _unavailable("multilateral_destination", {"destination": rule.multilateral}), + binding=binding, + label=source.label, + ) + if _population_unavailable(source.composite, facts): + return _option( + "bilateral", + _unavailable("provider_not_offered", {"why": "destination_unclassified"}), + binding=binding, + label=source.label, + ) + try: + years = _composite_years(identity, rule.read(identity), source.composite, facts) + except CompositeNotServable as exc: + why = ( + "destination_unclassified" + if isinstance(exc, PopulationUnavailable) + else _SLOT_WHY.get(prop := _slot_at(identity, exc.pointer), prop) + ) + return _option( + "bilateral", + _unavailable("provider_not_offered", {"why": why}), + binding=binding, + label=source.label, + ) + if identity.requested_window.kind == "latest": + own_year = elected_year(_year_counts(identity, facts)) + option = _latest_sibling_option( + "bilateral", binding, years.own, years.held, years.counts, own_year, label=source.label + ) + else: + option = _sibling_option("bilateral", binding, years.own, years.held, label=source.label) + if rule.spanning is None: + return option + return option.model_copy(update={"label": _rebound_label(option.label, source)}) + + +@dataclass(frozen=True, slots=True) +class _CompositeYears: + """What the bilateral option of a part reads (`_composite_years`), for each donor the + part names, in order, or for every donor together where it names none: `own`, the years + it holds a series of the total the part counts, and `held`, the years the composite + serves it (`composite.served_components`). `counts` are the composite's series per year + it serves (`_served_counts`), which `elect_latest_window` elects by too.""" + + own: list[frozenset[int]] + held: list[frozenset[int]] + counts: dict[int, int] + + +def _composite_years( + identity: BaseIdentity, read: BaseIdentity, composite: str, facts: ScopeFacts +) -> _CompositeYears: + """The years the part whose identity is `identity` holds of its total, and those + `composite` over `read` serves it, from the components it binds + (`composite.served_components`): a population's members on an open destination, so a + donor whose rows sit at the DAC's totals alone holds none. Raises + `code_index.composites.CompositeNotServable` as `composite.served_components` does.""" + document = _with_composite(read.model_dump(mode="json", exclude_none=True), composite) + served = served_components(document, facts=facts) + covered = frozenset[int].intersection(*(c.served for c in served)) if served else frozenset() + counts = _served_counts(served) + donors = identity.entities.get("origin") or [] + if not donors: + own = frozenset(y for y, n in _year_counts(identity, facts).items() if n > 0) + return _CompositeYears(own=[own], held=[covered], counts=counts) + by_component = [donor_years(c.component.identity, facts=facts) for c in served] + held = [ + covered & frozenset[int]().union(*(years.get(donor, frozenset()) for years in by_component)) + for donor in donors + ] + return _CompositeYears(own=_donor_years(identity, facts), held=held, counts=counts) + + +def _rebound_label(label: str, source: BilateralSource) -> str: + """`label` read at `source`'s total of the bilateral side in place of its total of both + sides (`OPTION_LABELS.provider_rebound`).""" + return OPTION_LABELS.provider_rebound.format( + label=label, rebound=source.rebound_words, spanning=source.spanning_words + ) + + def _published_inclusive( row: PublishedProviderScope, source: InclusiveSource | None, @@ -333,24 +598,170 @@ def _published_inclusive( facts: ScopeFacts, *, held: bool, + rule: DestinationRule | None, ) -> Option: - """The inclusive option of the published total `row` whose identity without the - inclusive source is `bilateral` (`_published_table`).""" - if source is not None: + """The inclusive option of the published total `row` whose identity without a provider + composite is `bilateral` (`_published_table`), whose destinations read by `rule` against + the bilateral source of `row`'s total, None where it declares none. The inclusive + composite's bilateral line reads the bilateral destinations alone, so the option is + unavailable (`multilateral_destination`) on a part naming a multilateral destination, not + offered (`provider_not_offered`, why `destination_unclassified`) where its population is + unavailable, as the bilateral option is, and on a part naming the total of both sides it + reads the total of the bilateral side in its place, as its label says + (`_rebound_label`).""" + if source is None: + if row.scope in _COUNTS_CONTRIBUTIONS: + return _option( + "inclusive", _unavailable("contributions_counted", {"dataset": row.dataset}) + ) + return _option("inclusive", _unavailable("provider_not_offered", {"why": "dataset"})) + if rule is None: return _inclusive_option(bilateral, source, facts, held=held) - if row.scope in _COUNTS_CONTRIBUTIONS: - return _option("inclusive", _unavailable("contributions_counted", {"dataset": row.dataset})) - return _option("inclusive", _unavailable("provider_not_offered", {"why": "dataset"})) + if rule.multilateral is not None: + return _option( + "inclusive", + _unavailable("multilateral_destination", {"destination": rule.multilateral}), + source=source.composite, + ) + if _population_unavailable(source.composite, facts): + return _option( + "inclusive", + _unavailable("provider_not_offered", {"why": "destination_unclassified"}), + source=source.composite, + ) + option = _inclusive_option(rule.read(bilateral), source, facts, held=held) + if rule.spanning is None: + return option + return option.model_copy(update={"label": _rebound_label(option.label, rule.source)}) + + +def with_bilateral_default(identity: BaseIdentity, facts: ScopeFacts) -> BaseIdentity: + """`identity` bound to the composite of the bilateral source (`BILATERAL_SOURCES`, rule + `BilateralSource.rule`) of the published total it counts, where a fresh search reads it: + `identity` binds the total's code as a value on a described shape, binds no source but a + pin to the total's dataset, the index elects that dataset for it, it names no + destination or only destinations on the bilateral side (`destination_rule`), and its + bilateral option is available (`_bilateral_option`), so the composite serves every year + its answer holds. A donor whose total ODA holds no breakdown by recipient in those years + keeps total ODA, and so does a part whose population is unavailable. The donors it names + do not matter otherwise, so the rule holds whether or not the part has a provider axis, + as for a ranking over every donor. `identity` itself otherwise. + + The composite then is the answer's own reading and the scope's base: its provider table + holds the bilateral option, a switch of another axis posted over the base keeps it, and a + switch to the total itself or to the inclusive scope reads another source. The composite + replaces a provenance pin to the total's dataset (`edits._with_source`), so a part + pinned to DAC2a and switched to total ODA from the composite reads the unpinned total, + whose figures are the pinned one's under another spec_id. A resubmit settles nothing, + so a posted identity reads as it binds: a bare total holds the contributions scope + (`PUBLISHED_PROVIDER_SCOPES`).""" + defaulted = _default_bilateral_option(identity, facts) + if defaulted is None or defaulted[1].availability.status != "available": + return identity + source, _ = defaulted + return apply_binding( + identity, ProviderBinding(scope="bilateral", provenance=source.composite), [] + ) + + +def _default_bilateral_option( + identity: BaseIdentity, facts: ScopeFacts +) -> tuple[BilateralSource, Option] | None: + """The bilateral source of the published total `identity` counts and its bilateral + option (`_bilateral_option`), where the default reaches `identity` + (`with_bilateral_default`) whether or not the option is available. None otherwise.""" + reached = _default_rule(identity, facts) + if reached is None: + return None + return reached.source, _bilateral_option(identity, reached, facts) + + +def bilateral_default_reaches(identity: BaseIdentity, facts: ScopeFacts) -> bool: + """Whether the bilateral default reaches `identity` (`with_bilateral_default`), whether + or not its option is available. It reads the code index alone, never coverage.""" + return _default_rule(identity, facts) is not None + + +def _default_rule(identity: BaseIdentity, facts: ScopeFacts) -> DestinationRule | None: + """The destination rule of the published total `identity` counts against its bilateral + source, where the default reaches `identity`: it binds a source's code, reads its + dataset under no other pin, and names no destination off the bilateral side. None + otherwise.""" + if not isinstance(identity.shape, IdentityShapeSchema): + return None + source = counted_bilateral_source(identity, dataset_of=lambda held: _dataset(held, facts.index)) + if source is None: + return None + pin = (identity.facets or {}).get("provenance") + if pin is not None and pin != IdentityBindingValue(ref=source.dataset): + return None + rule = destination_rule(identity, source, facts) + return rule if rule.bilateral else None + + +def bilateral_default_withheld( + identity: BaseIdentity, selection: ScopeState, facts: ScopeFacts +) -> str | None: + """Why a fresh reading of `identity` that the bilateral default reaches + (`with_bilateral_default`) reads total ODA, where no provider axis states it + (`provider_rule`) and no stated or carried provider entry asks for total ODA: the + unavailable bilateral option's `why` (`_bilateral_option`), as `destination_unclassified` + for an unavailable population or the slot at fault where the composite refuses what + `identity` binds, else its reason code, `option_not_covered` where it does not cover + every year the answer holds. A part with a provider axis states the reason on its + bilateral option instead. None where the default applies or does not reach `identity`.""" + defaulted = _default_bilateral_option(identity, facts) + if defaulted is None: + return None + availability = defaulted[1].availability + if availability.status == "available": + return None + stated = selection.entries_on("provider") + if stated and stated[0].option_id == CONTRIBUTIONS: + return None + if provider_rule(identity, None, facts) is not None: + return None + return availability.detail.get("why") or availability.code + + +def bilateral_default_reading( + identity: BaseIdentity, selection: ScopeState, facts: ScopeFacts +) -> tuple[BaseIdentity, ScopeState]: + """The identity a fresh reading of `identity` is served as, and the selection its scope + reads: one rule for a fresh search's definite answer (`scope_decision.decide_scope`) and + every other reading it shows (`composite_answer.with_bilateral_readings`). + + DAC2a's total ODA reads its bilateral destinations (`with_bilateral_default`) unless the + part's first provider entry, explicit before carried (`ScopeState.entries_on`), asks for + total ODA (`provider:contributions`), which keeps `identity`. Where the part has a + provider axis (`provider_rule`), its table settles the entry as it settles any + (`settle_provider`). Where it has none, as on a ranking over every donor, no table can + apply the entry, so the default serves it here: a `provider:contributions` entry keeps + total ODA and a `provider:bilateral` one reads the composite, and the entry leaves the + selection, so the scope does not report it as dropped. Any other entry, or one on a part + the default does not reach, stays in the selection as it was posted.""" + default = with_bilateral_default(identity, facts) + stated = selection.entries_on("provider") + if default is identity or not stated or stated[0].option_id not in (CONTRIBUTIONS, BILATERAL): + return default, selection + entry = stated[0] + served = identity if entry.option_id == CONTRIBUTIONS else default + if provider_rule(served, None, facts) is not None: + return served, selection + entries = tuple(each for each in selection.entries if each is not entry) + return served, dataclasses.replace(selection, entries=entries) def _sibling_option( scope: ProviderScope, - binding: SlotBinding, + binding: SlotBinding | ProviderBinding, own: Sequence[frozenset[int]], held: Sequence[frozenset[int]], + *, + label: str | None = None, ) -> Option: - """The option of `scope` binding the sibling total `binding` names, on a part asking - for a range or every year. `own` and `held` name, for each donor of the part in order, + """The option of `scope` binding the sibling total `binding` names, labelled `label` or + else its scope's label, on a part asking for a range or every year. `own` and `held` name, for each donor of the part in order, the years it holds the part's own total and the sibling in (`_donor_years`). The answer holds the years any donor holds its own total in; one of them is covered when every donor holding its own total in it holds the sibling in it, so a donor holding no own @@ -364,9 +775,9 @@ def _sibling_option( asked = frozenset[int]().union(*own) if not asked: return ( - _option(scope, binding=binding) + _option(scope, binding=binding, label=label) if all(held) - else _option(scope, _unavailable("option_not_covered"), binding=binding) + else _option(scope, _unavailable("option_not_covered"), binding=binding, label=label) ) covered = frozenset( year @@ -374,28 +785,31 @@ def _sibling_option( if all(year in theirs for ours, theirs in zip(own, held, strict=True) if year in ours) ) if covered >= asked: - return _option(scope, binding=binding) + return _option(scope, binding=binding, label=label) if not covered: - return _option(scope, _unavailable("option_not_covered"), binding=binding) + return _option(scope, _unavailable("option_not_covered"), binding=binding, label=label) runs = spans(covered) return _option( scope, OptionAvailability(status="partial", years=runs, code="option_not_covered"), binding=binding.model_copy(update={"window": runs[-1]}), window=runs[-1], + label=label, ) def _latest_sibling_option( scope: ProviderScope, - binding: SlotBinding, + binding: SlotBinding | ProviderBinding, own: Sequence[frozenset[int]], held: Sequence[frozenset[int]], counts: Mapping[int, int], own_year: int | None, + *, + label: str | None = None, ) -> Option: - """The option of `scope` binding the sibling total `binding` names, on a part asking - for the latest year, whose read elects `own_year` for the pool. `own` and `held` are as + """The option of `scope` binding the sibling total `binding` names, labelled `label` or + else its scope's label, on a part asking for the latest year, whose read elects `own_year` for the pool. `own` and `held` are as `_sibling_option` takes them, and `counts` the sibling's series per year (`_year_counts`). A `latest` read elects one year for the whole pool, so the option elects the sibling's year as `elected_year` does among the years every donor holds it @@ -413,16 +827,17 @@ def _latest_sibling_option( shared = frozenset[int].intersection(*demanded) if demanded else frozenset[int]() year = elected_year({y: n for y, n in counts.items() if y in shared}) if year is None: - return _option(scope, _unavailable("option_not_covered"), binding=binding) + return _option(scope, _unavailable("option_not_covered"), binding=binding, label=label) window = (year, year) pinned = binding.model_copy(update={"window": window}) if year == own_year: - return _option(scope, binding=pinned) + return _option(scope, binding=pinned, label=label) return _option( scope, OptionAvailability(status="partial", years=[window], code="option_not_covered"), binding=pinned, window=window, + label=label, ) @@ -521,10 +936,12 @@ def _published_row(identity: BaseIdentity, facts: ScopeFacts) -> PublishedProvid if not rows or not _names_donors_alone(identity, facts.index): return None try: - document = table_identity(facts.index, identity.model_dump(mode="json", exclude_none=True)) + document = table_identity( + facts.index, identity.model_dump(mode="json", exclude_none=True) + ).identity except CompositeNotServable: return None - dataset = _dataset(BaseIdentity.model_validate(document), facts) + dataset = _dataset(BaseIdentity.model_validate(document), facts.index) return next((row for row in rows if row.dataset == dataset), None) @@ -568,17 +985,19 @@ def without_uncounted_inclusive( can take the imputed share (`_inclusive_option`, and on a published total `_published_row`). Where it does not, as for a part naming no provider, or a provider whose total already counts the multilaterals' outflows, the identity is served its - bilateral line (`_without_inclusive`), and `provider:inclusive` joins its entries as a - carried entry from the posted identity's provider choice, unless they hold a provider - entry, so the scope drops it as it drops any carried entry the part cannot take - (`compile`), and says so. `identity` and `entries` themselves otherwise.""" + bilateral line: the table the composite stacks on (`_without_provider_composite`), read + as a fresh search reads it (`with_bilateral_default`), so DAC2a's total ODA reads the + bilateral destinations alone. `provider:inclusive` joins its entries as a carried entry + from the posted identity's provider choice, unless they hold a provider entry, so the + scope drops it as it drops any carried entry the part cannot take (`compile`), and says + so. `identity` and `entries` themselves otherwise.""" try: ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) except CompositeNotServable: return identity, entries if ref not in _SOURCE_COMPOSITES: return identity, entries - bilateral = _without_inclusive(identity, ref) + bilateral = _without_provider_composite(identity, ref) table = provider_rule(bilateral, None, facts) option = table.option(INCLUSIVE) if table is not None else None if option is not None and option.availability.code != "provider_not_offered": @@ -593,7 +1012,7 @@ def without_uncounted_inclusive( origin=ChoiceOrigin(choice_id=f"{identity_spec_id(identity)}:provider"), ) ) - return bilateral, held + return with_bilateral_default(bilateral, facts), held def _counts_as_donor(party: str, index: CodeIndex) -> bool: @@ -661,9 +1080,8 @@ def _pools_redistributed(providers: Collection[str]) -> bool: return len(providers) > 1 and not _REDISTRIBUTED.isdisjoint(providers) -def _dataset(identity: BaseIdentity, facts: ScopeFacts) -> str | None: +def _dataset(identity: BaseIdentity, index: CodeIndex) -> str | None: """The dataset the index elects for `identity`'s family, None when it decides none.""" - index = facts.index try: family = index_family( index, identity_family_query(index, identity.model_dump(mode="json", exclude_none=True)) @@ -727,25 +1145,27 @@ def option( def elect_latest_window(identity: BaseIdentity, facts: ScopeFacts) -> BaseIdentity: - """`identity`, served as it reads: a provider-inclusive composite asking for the latest - year is read over the year `inclusive_reach` elects among those both its lines cover, a + """`identity`, served as it reads: a composite a provider option binds asking for the + latest year is read over the year `inclusive_reach` elects among those its lines cover, a one-year range, since its lines can elect different latest years and the Datasets mint - refuses a composite's `latest` window. Any other identity, or one whose tables elect no year, is - `identity` itself. + refuses a composite's `latest` window. The bilateral destinations of DAC2a's total ODA + (`BILATERAL_SOURCES`) are one line, read over the latest year it serves. Any other + identity, or one whose tables elect no year, is `identity` itself. - A part's base keeps its `latest` window, so the bilateral option, which removes the - composite, compiles to the bilateral answer's own latest year; the inclusive compiles - here to the one year both cover.""" + A part's base keeps its `latest` window, so an option that removes the composite + compiles to its table's own latest year; a composite compiles here to one year.""" if identity.requested_window.kind != "latest": return identity try: ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) except CompositeNotServable: return identity - if ref not in _SOURCE_COMPOSITES: + if ref not in _PROVIDER_COMPOSITES: return identity try: - year = inclusive_reach(_without_inclusive(identity, ref), facts, composite=ref).latest + year = inclusive_reach( + _without_provider_composite(identity, ref), facts, composite=ref + ).latest except CompositeNotServable: return identity if year is None: @@ -852,7 +1272,7 @@ def _convention( """The convention `provider_convention` reads.""" shape = identity_part_shape(identity, facts.index) names_donor = bool(identity.entities.get("origin")) - dataset = _dataset(identity, facts) + dataset = _dataset(identity, facts.index) return next( ( row @@ -872,15 +1292,23 @@ def with_summed_provider( """`state` with its words summing the bilateral and multilateral lines (`ScopeState.sums_providers`, "bilateral plus multilateral ODA") read as the option summing them on the dataset of the part whose answer is `identity`. Where the part's - provider table offers the contributions scope (`provider_rule`, a DAC1 total, whose total - ODA counts bilateral flows plus core contributions to multilaterals), the explicit - inclusive entry read from those words asks for that scope. Elsewhere, on the CRS and - DAC2a, the inclusive composite is the sum, and `state` is itself.""" + provider table offers the contributions scope (`provider_rule`) on a dataset that + declares no inclusive source (`INCLUSIVE_SOURCES`), DAC1's total ODA, which counts + bilateral flows plus core contributions to multilaterals, the explicit inclusive entry + read from those words asks for that scope. Where the dataset declares one, on the CRS + and DAC2a, the inclusive composite is the sum, whatever destination the part names, and + `state` is itself.""" if not state.sums_providers: return state table = provider_rule(identity, None, facts) if table is None or table.option(CONTRIBUTIONS) is None: return state + try: + ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) + except CompositeNotServable: + return state + if _dataset(_without_provider_composite(identity, ref), facts.index) in _SOURCES: + return state entries = tuple( entry.model_copy(update={"option_id": CONTRIBUTIONS}) if entry.option_id == INCLUSIVE @@ -942,7 +1370,7 @@ def settle_provider( if table.published: return _settled(table, state, None) ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) - bilateral = _without_inclusive(identity, ref) + bilateral = _without_provider_composite(identity, ref) convention = provider_convention(bilateral, facts, asks_total=state.total) return _settled(table, state, convention) diff --git a/tests/assembler/test_export_component.py b/tests/assembler/test_export_component.py index 82af9ff38..c5e204193 100644 --- a/tests/assembler/test_export_component.py +++ b/tests/assembler/test_export_component.py @@ -33,6 +33,8 @@ ) from assembler.ports import FrameRef, ResolvedFrame from assembler.testing import FIXTURE_RETRIEVED_ON, build_resolved_frame +from code_index.composites import DECLARED_COMPOSITES +from code_index.declarations import DAC2A_BILATERAL from dfo_analysis_engine.place_type import UNTYPED from materialise.bundle_naming import card_slug from tests.presentation_store.test_frame_store import ( @@ -1517,3 +1519,67 @@ def test_a_table_row_of_a_continued_series_names_its_own_method( *(("Ethiopia", str(year), "Estimated") for year in range(2022, 2024)), *(("Ghana", str(year), "Reported") for year in range(2019, 2024)), ] + + +def test_a_card_over_a_composite_frame_cites_its_tables_and_names_the_composite( + frame_store, revision_store, release_registry +): + """A composite's rows carry its own provenance, which its citation does not list: the + citation names its component tables. The bundle cites those tables by their own label, + names the composite by its label and measure noun in the codes table, the README, + SOURCES.md and the datapackage description, so the two read apart, and states there what + it reads and the destinations it keeps.""" + composite = DECLARED_COMPOSITES[DAC2A_BILATERAL] + (table,) = {recipe.dataset for recipe in composite.components} + rows = [ + _row(recipient=recipient, period=str(year), value=float(year), provenance=DAC2A_BILATERAL) + for recipient in ("country/ETH", "country/KEN") + for year in range(2019, 2024) + ] + resolved = build_resolved_frame( + frame_ref=_FRAME_REF, + role_keys=("recipient",), + rows=rows, + labels={"country/ETH": "Ethiopia", "country/KEN": "Kenya"}, + manifest={ + "schema_version": 1, + "citation": { + "text": "exemplar", + "sources": [{"ref": table, "label": "OECD DAC2a", "license": "OECD terms"}], + }, + }, + ) + frame_store.put(resolved_frame=resolved) + intent = { + **_INTENTS["line_chart"], + "select": { + "stat_var": _STAT_VAR, + "recipient": ["country/ETH", "country/KEN"], + "provenance": DAC2A_BILATERAL, + }, + "frame_ref": _frame_ref_body(_FRAME_REF), + } + created = write_component( + intent=intent, + frame_store=frame_store, + revision_store=revision_store, + release_registry=release_registry, + ) + + files = _unzip( + _export(created, frame_store=frame_store, revision_store=revision_store, bundle=True).data + ) + + descriptor = json.loads(files["datapackage.json"]) + assert [source["title"] for source in descriptor["sources"]] == ["OECD DAC2a"] + codes = {row["code"]: row["label"] for row in _csv_rows(files["data/codes.csv"])} + assert codes[DAC2A_BILATERAL] == "OECD DAC2a (bilateral flows)" + cut = "counting bilateral flows to countries and regions, without core contributions" + for text in ( + files["README.md"].decode("utf-8"), + files["SOURCES.md"].decode("utf-8"), + descriptor["description"], + ): + assert "OECD DAC2a (bilateral flows)" in text + assert DAC2A_BILATERAL in text + assert cut in text diff --git a/tests/code_index/test_composites.py b/tests/code_index/test_composites.py index 822e61dcb..802562f5c 100644 --- a/tests/code_index/test_composites.py +++ b/tests/code_index/test_composites.py @@ -13,6 +13,7 @@ from code_index.composites import ( CompositeNotServable, CompositeProvenanceRead, + binds_composite, component_identities, component_of, composite_digest, @@ -33,7 +34,7 @@ ) from tests.code_index.support import INDEX_FIXTURE -_STACKED, _DAC2A_STACKED, _SPLICED = COMPOSITES +_STACKED, _DAC2A_STACKED, _DAC2A_BILATERAL, _SPLICED = COMPOSITES def _payload() -> dict: @@ -321,9 +322,28 @@ def _first_component(composite, **changes): ], ) def test_the_composite_digest_moves_with_what_a_run_composes_by(composite, changed): - assert composite_digest(changed(composite)) != composite_digest(composite) - assert composite_digest(composite) == declared_composite_digest(composite.provenance) + digest = composite_digest(composite, index=_INDEX) + assert composite_digest(changed(composite), index=_INDEX) != digest + assert digest == declared_composite_digest(composite.provenance, index=_INDEX) def test_a_provenance_no_composite_declares_has_no_declared_digest(): - assert declared_composite_digest("provenance/NotAComposite") is None + assert declared_composite_digest("provenance/NotAComposite", index=_INDEX) is None + + +@pytest.mark.parametrize( + ("provenance", "binds"), + [ + pytest.param(None, False, id="unbound"), + pytest.param({"ref": "OECD_DAC2A"}, False, id="table"), + pytest.param({"ref": _DAC2A_BILATERAL.provenance}, True, id="composite"), + pytest.param( + {"refs": ["OECD_DAC2A", _DAC2A_BILATERAL.provenance]}, True, id="set-naming-one" + ), + ], +) +def test_binds_composite_reads_a_set_naming_one_as_a_composite( + provenance: dict[str, object] | None, binds: bool +) -> None: + facets = {} if provenance is None else {"provenance": provenance} + assert binds_composite({"facets": facets}) is binds diff --git a/tests/code_index/test_destination_population.py b/tests/code_index/test_destination_population.py new file mode 100644 index 000000000..715d9f593 --- /dev/null +++ b/tests/code_index/test_destination_population.py @@ -0,0 +1,319 @@ +"""A composite component that binds its destinations to the bilateral side reads DAC2a's +bilateral population (`code_index.composites.destination_population`): an open destination +binds the population's members and is tagged as a population, a named one passes only on the +bilateral side, and an absent one binds the population's total. A destination no side +declaration places, in the code index or in coverage, makes the population unavailable.""" + +from __future__ import annotations + +import gzip +import json +import weakref + +import pytest + +import code_index.composites as composites_module +from code_index.composites import ( + CompositeNotServable, + PopulationUnavailable, + UnclassifiedDestinations, + component_identities, + composite_digest, + destination_population, + population_check, + table_identity, +) +from code_index.declarations import ( + COMPOSITES, + CRS_INCLUSIVE, + DAC2A, + DAC2A_BILATERAL, + DAC2A_INCLUSIVE, +) +from code_index.loader import parse_index +from code_index.query import CodeIndex +from code_index.scope_conventions import ( + BILATERAL_SOURCES, + DAC_DEVELOPING_COUNTRIES, + destination_side, +) +from series_coverage.schema import FacetMetadata, RoleNames +from tests.code_index.aid_identity import AID_TYPE +from tests.code_index.support import INDEX_FIXTURE +from tests.engine._fake_coverage import build_coverage_index + +_DAC2A_INCLUSIVE, _BILATERAL = DAC2A_INCLUSIVE, DAC2A_BILATERAL +_KENYA = "country/KEN" +_FRANCE = "country/FRA" + + +def _parse() -> CodeIndex: + with gzip.open(INDEX_FIXTURE) as handle: + return parse_index(json.load(handle), source="fixture", composites=COMPOSITES) + + +_INDEX = _parse() + + +def _identity(provenance: str, **entities: object) -> dict: + """France's net DAC2a total ODA over every year, binding the composite `provenance`, with + `entities` merged over France as origin.""" + identity: dict = { + "identity_version": "spec1", + "shape": { + "kind": "statvar_schema", + "population_type": "FinancialFlow", + "measured_property": "amount", + "stat_type": "measuredValue", + "measurement_qualifier": "Nominal", + }, + "slots": [ + {"property": AID_TYPE, "binding": {"state": "value", "ref": "DAC2aAidType206"}}, + {"property": "financialFlowType", "binding": {"state": "value", "ref": "Disbursement"}}, + {"property": "nettingBasis", "binding": {"state": "value", "ref": "Net"}}, + ], + "entities": {"origin": [_FRANCE]}, + "requested_window": {"kind": "all"}, + "facets": {"provenance": {"state": "value", "ref": provenance}}, + } + open_roles = entities.pop("open_roles", None) + identity["entities"].update(entities) + if open_roles is not None: + identity["open_roles"] = open_roles + return identity + + +def test_the_population_holds_bilateral_parties_and_residuals_and_names_its_totals(): + population = destination_population(_INDEX, DAC2A) + + assert population.side == "bilateral" + assert population.total == DAC_DEVELOPING_COUNTRIES + assert {_KENYA, "DACBilateralUnallocated", "EastAfricanCommunity"} <= set(population.members) + assert not {"EuropeanCommission", "DACOtherMultilateralRecipient", "DACAllRecipients"} & ( + population.named + ) + assert not {DAC_DEVELOPING_COUNTRIES, "DACAfrica"} & set(population.members) + assert {DAC_DEVELOPING_COUNTRIES, "DACAfrica"} <= population.named + assert set(population.members) <= population.named + assert list(population.members) == sorted(population.members) + + +@pytest.mark.parametrize("provenance", [_BILATERAL, _DAC2A_INCLUSIVE]) +def test_an_open_destination_binds_the_population_and_tags_its_key(provenance: str): + population = destination_population(_INDEX, DAC2A) + + components = component_identities(_INDEX, _identity(provenance, open_roles=["destination"])) + + assert components is not None + for component in components: + assert component.identity["entities"] == { + "destination": list(population.members), + "origin": [_FRANCE], + } + assert "open_roles" not in component.identity + assert component.populations == {"destination": population} + + +@pytest.mark.parametrize( + "named", [[_KENYA], [DAC_DEVELOPING_COUNTRIES], [_KENYA, "DACAfrica"], ["country/ZZZ"]] +) +def test_a_named_destination_on_the_bilateral_side_passes_untagged(named: list[str]): + """A ref DAC2a does not publish as a destination, such as `country/ZZZ`, passes and reads + no rows, as on the plain table.""" + components = component_identities(_INDEX, _identity(_BILATERAL, destination=named)) + + assert components is not None + (component,) = components + assert component.identity["entities"]["destination"] == named + assert component.populations == {} + + +@pytest.mark.parametrize( + "named", + [ + ["EuropeanCommission"], + ["DACAllRecipients"], + ["DACAllMultilateralRecipients"], + [_KENYA, "DACOtherMultilateralRecipient"], + ], +) +@pytest.mark.parametrize("provenance", [_BILATERAL, _DAC2A_INCLUSIVE]) +def test_a_named_destination_off_the_bilateral_side_is_refused(provenance: str, named: list[str]): + with pytest.raises(CompositeNotServable) as refused: + component_identities(_INDEX, _identity(provenance, destination=named)) + + assert refused.value.ref == provenance + assert refused.value.pointer == "/entities/destination" + assert refused.value.refs == tuple(ref for ref in named if ref != _KENYA) + assert refused.value.key == "destination" + + +def test_an_absent_destination_binds_the_population_s_total_untagged(): + components = component_identities(_INDEX, _identity(_BILATERAL)) + + assert components is not None + (component,) = components + assert component.identity["entities"]["destination"] == [DAC_DEVELOPING_COUNTRIES] + assert component.populations == {} + + +def test_the_table_identity_carries_the_first_component_s_population(): + population = destination_population(_INDEX, DAC2A) + + read = table_identity(_INDEX, _identity(_DAC2A_INCLUSIVE, open_roles=["destination"])) + + assert read.identity["entities"]["destination"] == list(population.members) + assert "facets" not in read.identity + assert read.populations == {"destination": population} + + +def test_the_table_identity_of_an_identity_binding_no_composite_is_itself(): + identity = _identity(DAC2A, open_roles=["destination"]) + + read = table_identity(_INDEX, identity) + + assert read.identity is identity + assert read.populations == {} + + +def _placing_kenya_as(monkeypatch: pytest.MonkeyPatch, side: str | None) -> CodeIndex: + """A fresh index, so the population memo builds again, whose classifier places Kenya on + `side`.""" + + def placed(row, *, datasets): + return side if row.dcid == _KENYA else destination_side(row, datasets=datasets) + + monkeypatch.setattr(composites_module, "destination_side", placed) + return _parse() + + +@pytest.mark.parametrize("provenance", [_BILATERAL, _DAC2A_INCLUSIVE]) +def test_a_destination_the_code_index_cannot_place_refuses_the_composite_reading_it( + monkeypatch: pytest.MonkeyPatch, provenance: str +): + """The refusal is a `CompositeNotServable`, so every handler of one catches it, and names + the composite asked for and the drifted destination.""" + index = _placing_kenya_as(monkeypatch, None) + expected = UnclassifiedDestinations( + dataset=DAC2A, destinations=(_KENYA,), found_in="code_index" + ) + + with pytest.raises(CompositeNotServable) as drifted: + component_identities(index, _identity(provenance, open_roles=["destination"])) + + assert isinstance(drifted.value, PopulationUnavailable) + assert drifted.value.unclassified == expected + assert (drifted.value.ref, drifted.value.pointer) == (provenance, "/facets/provenance") + coverage = _coverage([_KENYA]) + assert population_check(index, coverage).refusal(provenance) == expected + + +def test_the_composite_digest_moves_with_the_population(monkeypatch: pytest.MonkeyPatch): + composite = _INDEX.composite(_BILATERAL) + assert composite is not None + before = composite_digest(composite, index=_INDEX) + + after = composite_digest(composite, index=_placing_kenya_as(monkeypatch, "multilateral")) + + assert after != before + + +# The variables DAC2a's population reads in the fixture index: the 206 and imputed 106 +# totals a component binding its destinations binds. +_POPULATION_VARIABLES = composites_module._population_variables(_INDEX, DAC2A) + + +def _coverage(recipients: list[str], *, metadata: FacetMetadata | None = None): + """Coverage holding `recipients` for France under each population variable, in one facet + whose metadata is `metadata`, DAC2a's by default.""" + return build_coverage_index( + [ + (variable, "dac2a", _FRANCE, recipient, [2021]) + for variable in _POPULATION_VARIABLES + for recipient in recipients + ], + role_names={ + variable: RoleNames(donor="financingProvider", recipient="financingRecipient") + for variable in _POPULATION_VARIABLES + }, + facet_metadata={"dac2a": metadata or FacetMetadata(provenance_id=DAC2A, unit="USD")}, + ) + + +def test_coverage_whose_recipients_the_population_places_leaves_it_available(): + check = population_check(_INDEX, _coverage([_KENYA, "EuropeanCommission", "DACAllRecipients"])) + + assert check.unclassified == {} + assert check.refusal(_BILATERAL) is None + + +@pytest.mark.parametrize("provenance", [_BILATERAL, _DAC2A_INCLUSIVE]) +def test_a_covered_recipient_the_code_index_does_not_publish_makes_the_population_unavailable( + provenance: str, +): + check = population_check(_INDEX, _coverage([_KENYA, "SomeNewAgency"])) + + assert check.refusal(provenance) == UnclassifiedDestinations( + dataset=DAC2A, destinations=("SomeNewAgency",), found_in="coverage" + ) + assert check.refusal(CRS_INCLUSIVE) is None + + +@pytest.mark.parametrize( + "metadata", + [ + pytest.param(FacetMetadata(unit="USD"), id="no_provenance"), + pytest.param( + FacetMetadata(provenance_id="provenance/OECDCRS", unit="USD"), id="another_provenance" + ), + ], +) +def test_coverage_rows_not_attributed_to_the_dataset_make_the_population_unavailable( + metadata: FacetMetadata, +): + """The check fails closed: a destination in rows coverage cannot attribute to the dataset + could sit off every side unseen. The reason names the variables.""" + check = population_check(_INDEX, _coverage([_KENYA], metadata=metadata)) + + assert check.refusal(_BILATERAL) == UnclassifiedDestinations( + dataset=DAC2A, destinations=(), found_in="coverage", unlisted=_POPULATION_VARIABLES + ) + assert _POPULATION_VARIABLES[0] in check.unclassified[DAC2A].message() + + +def test_every_declared_component_binding_destinations_has_a_declared_source(): + """A component binding the destinations of a dataset no `BILATERAL_SOURCES` row declares + would fail at its first read.""" + bound = { + recipe.dataset + for composite in COMPOSITES + for recipe in composite.components + if recipe.destinations is not None + } + assert bound == {source.dataset for source in BILATERAL_SOURCES} == {DAC2A} + + +def test_the_population_check_memo_holds_one_index_finalizer_however_many_coverages( + monkeypatch: pytest.MonkeyPatch, +): + """A code index outlives many coverage snapshots. Each check is dropped with its own + coverage, and a snapshot after the first adds no finalizer to the index.""" + index = _parse() + finalized: list[object] = [] + register = weakref.finalize + + def recording(obj, *args, **kwargs): + finalized.append(obj) + return register(obj, *args, **kwargs) + + monkeypatch.setattr(weakref, "finalize", recording) + coverages = [_coverage([_KENYA]) for _ in range(3)] + first = population_check(index, coverages[0]) + on_index = sum(obj is index for obj in finalized) + + for coverage in coverages[1:]: + population_check(index, coverage) + + assert population_check(index, coverages[0]) is first + assert sum(obj is index for obj in finalized) == on_index + assert [obj for obj in finalized if obj is not index] == coverages diff --git a/tests/code_index/test_destination_side.py b/tests/code_index/test_destination_side.py new file mode 100644 index 000000000..121f5fabe --- /dev/null +++ b/tests/code_index/test_destination_side.py @@ -0,0 +1,124 @@ +"""`destination_side` places a DAC2a destination on the bilateral or multilateral side of a +donor's aid, with the classifier Search uses for donors (`code_index.multilaterals`).""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + +from code_index.declarations import DAC2A +from code_index.multilaterals import organisation_dcids +from code_index.scope_conventions import DestinationSide, destination_side +from code_index.types import PartyKind, PartyRow +from entity_resolution import pack +from tests.code_index.support import INDEX_FIXTURE, load_index + +_DATASETS = (DAC2A,) + + +def _row(dcid: str, kind: PartyKind) -> PartyRow: + return PartyRow( + dcid=dcid, + kind=kind, + party_class={"party": "official", "residual": "residual", "total": None}[kind], + published=((DAC2A, "destination"),), + ) + + +@pytest.mark.parametrize( + ("dcid", "kind", "side"), + [ + ("EuropeanCommission", "party", "multilateral"), + ("InternationalDevelopmentAssociation", "party", "multilateral"), + ("DACOtherMultilateralRecipient", "residual", "multilateral"), + ("DACOtherUnitedNations", "residual", "multilateral"), + ("DACOtherRegionalDevelopmentBanks", "residual", "multilateral"), + ("country/SEN", "party", "bilateral"), + ("DACAfricaUnallocated", "residual", "bilateral"), + ("DACBilateralUnallocated", "residual", "bilateral"), + ("DACMekongDeltaProject", "residual", "bilateral"), + ("EastAfricanCommunity", "party", "bilateral"), + ("IndusBasinDevelopmentFund", "party", "bilateral"), + ("DACAllRecipients", "total", "spanning"), + ("DACAllMultilateralRecipients", "total", "multilateral"), + ("DACDevelopingCountries", "total", "bilateral"), + ("DACAfrica", "total", "bilateral"), + ], +) +def test_a_dac2a_destination_is_placed_on_its_side( + dcid: str, kind: PartyKind, side: DestinationSide +): + assert destination_side(_row(dcid, kind), datasets=_DATASETS) == side + + +def test_an_unknown_party_that_is_no_country_is_unclassified(): + """A party no organisation record or country prefix names is a drift the mint refuses, + not a bilateral recipient.""" + assert destination_side(_row("SomeNewAgency", "party"), datasets=_DATASETS) is None + + +@pytest.mark.parametrize( + ("dcid", "kind"), + [ + pytest.param("DACSomeNewGroup", "total", id="an-undeclared-total"), + pytest.param("DACSomeNewUnallocated", "residual", id="an-undeclared-residual"), + ], +) +def test_a_total_or_residual_no_declaration_places_is_unclassified(dcid: str, kind: PartyKind): + """A total or residual could count multilateral agencies, so the classifier places one only + where it is declared on a side, and the mint refuses any other.""" + assert destination_side(_row(dcid, kind), datasets=_DATASETS) is None + + +def test_a_party_no_dataset_publishes_as_a_recipient_is_unclassified(): + row = PartyRow(dcid="country/SEN", kind="party", published=((DAC2A, "origin"),)) + + assert destination_side(row, datasets=_DATASETS) is None + + +def test_every_dac2a_destination_of_the_recorded_index_is_classified(): + parties = load_index(INDEX_FIXTURE).parties.values() + destinations = [row for row in parties if row.publishes("destination", _DATASETS)] + + assert destinations + unclassified = [ + row.dcid for row in destinations if destination_side(row, datasets=_DATASETS) is None + ] + assert unclassified == [] + assert len(destinations) == 279 + + +def test_the_organisation_records_read_from_a_zipped_package_without_running_its_init( + tmp_path: Path, +): + """`organisation_dcids` reads the entity resolver's records through `importlib.resources`, + so the read works when the package is served from a zip, and never runs the package's + `__init__`, which loads resolvekit. The zipped `__init__` raises if it runs.""" + records = Path(importlib.util.find_spec("entity_resolution").origin).parent / "data" + archive = tmp_path / "packages.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("entity_resolution/__init__.py", "raise ImportError('init ran')\n") + zipped.write(records / "graph_entities.json", "entity_resolution/data/graph_entities.json") + script = ( + "import sys\n" + f"sys.path.insert(0, {str(archive)!r})\n" + "from code_index.multilaterals import organisation_dcids\n" + "print(len(organisation_dcids()), 'entity_resolution' in sys.modules," + " any(name.startswith('resolvekit') for name in sys.modules))\n" + ) + + ran = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, check=False, timeout=60 + ) + + assert ran.returncode == 0, ran.stderr + assert ran.stdout.split() == [str(len(organisation_dcids())), "False", "False"] + + +def test_the_entity_resolver_pack_reads_the_organisations_through_the_one_reader(): + assert pack.organisation_dcids is organisation_dcids diff --git a/tests/code_index/test_family_label.py b/tests/code_index/test_family_label.py index b42b02643..27b2bbee3 100644 --- a/tests/code_index/test_family_label.py +++ b/tests/code_index/test_family_label.py @@ -93,6 +93,33 @@ def test_a_composite_is_named_by_its_declaration(index, identity, plain): assert label == ", ".join([*lead, *([slots] if slots else [])]) +_DAC2A_IDENTITY = { + **_HEADLINE_IDENTITY, + "slots": [ + {"property": "dac2aAidType", "binding": {"state": "value", "ref": "DAC2aAidType206"}}, + {"property": "financialFlowType", "binding": {"state": "value", "ref": "Disbursement"}}, + {"property": "nettingBasis", "binding": {"state": "value", "ref": "Net"}}, + ], +} + + +def test_a_stacked_composite_of_one_table_names_its_cut(index): + """DAC2a's bilateral destinations are a cut of DAC2a's own rows, so the reading is named + as DAC2a's total ODA is, with the cut its sentence names after the measure.""" + measures = get_measure_index() + cut = { + **_DAC2A_IDENTITY, + "facets": {"provenance": {"state": "value", "ref": "provenance/ONEDAC2aBilateral"}}, + } + plain = family_label(_DAC2A_IDENTITY, index=index, measure_index=measures) + assert plain is not None + measure, *slots = plain.split(", ") + + label = family_label(cut, index=index, measure_index=measures) + + assert label == ", ".join([f"{measure} (bilateral flows)", *slots]) + + def test_the_imputed_label_leaves_out_every_slot_its_five_tuple_pins(index): identity = imputed_education_aid_identity() pinned = index.dataset_pins[IMPUTED_PROVENANCE, five_tuple_from_identity(identity["shape"])] diff --git a/tests/datasets_api/test_composite_download.py b/tests/datasets_api/test_composite_download.py index 35293c077..405c58a18 100644 --- a/tests/datasets_api/test_composite_download.py +++ b/tests/datasets_api/test_composite_download.py @@ -42,7 +42,7 @@ _mint, ) -_HEADLINE = COMPOSITES[2].provenance +_HEADLINE = COMPOSITES[3].provenance _DAC1 = "provenance/OECDDAC1" _DAC1_FACET = "facet_dac1" _NET = "Amount_FinancialFlow_Net_Disbursement_DAC1AidType1010_Nominal" diff --git a/tests/datasets_api/test_derived.py b/tests/datasets_api/test_derived.py index f273704bd..d5a4d9b7d 100644 --- a/tests/datasets_api/test_derived.py +++ b/tests/datasets_api/test_derived.py @@ -19,6 +19,8 @@ import datasets_api.doors.analysis as analysis_door from artifact_store import Tier +from code_index.composites import DECLARED_COMPOSITES +from code_index.declarations import DAC2A_BILATERAL from dataset_store.snapshot import ManifestFacts, StoredVersion from datasets_api.analysis import AnalysisRequest, run_analysis from datasets_api.config import DatasetsBuildSettings @@ -48,8 +50,9 @@ from .conftest import FakeDatasetStore -def _aid() -> Any: - """ODA to three recipients over 2021 and 2022, each recipient a party.""" +def _aid(provenance: str = "dcid:dc/base/OECD_CRS") -> Any: + """ODA to three recipients over 2021 and 2022, each recipient a party, read from + `provenance`.""" return build_frame( role_keys=["financingRecipient", "financingRecipient_party_kind"], rows=[ @@ -60,7 +63,7 @@ def _aid() -> Any: "time_period": year, "value": 10.0, "unit": "USDollar", - "provenance": "dcid:dc/base/OECD_CRS", + "provenance": provenance, } for recipient in ("country/GHA", "country/KEN", "country/NGA") for year in ("2021", "2022") @@ -107,18 +110,21 @@ def _population() -> Any: ) -def _roots(stores: Stores) -> FakeRootSource: +def _roots(stores: Stores, *, aid: Any = None) -> FakeRootSource: return FakeRootSource( - frames={(AID, AID_V1): _aid(), (POPULATION, POPULATION_V1): _population()}, + frames={ + (AID, AID_V1): _aid() if aid is None else aid, + (POPULATION, POPULATION_V1): _population(), + }, newest={AID: AID_V1, POPULATION: POPULATION_V1}, recipes=stores.recipe_store, ) -def _stored(stores: Stores) -> dict[str, Any]: +def _stored(stores: Stores, *, aid: Any = None, tree: Any = _CHAIN) -> dict[str, Any]: result = run_analysis( - AnalysisRequest.model_validate({"tree": _CHAIN}), - source=_roots(stores), + AnalysisRequest.model_validate({"tree": tree}), + source=_roots(stores, aid=aid), stores=stores.analysis(), ) return result.ref()["members"][0] @@ -234,6 +240,42 @@ def test_the_file_route_writes_the_frame_in_each_format( assert any(name.startswith("data/") and name.endswith(".csv") for name in names) +def test_a_bundle_of_a_frame_over_a_composite_cites_its_tables_and_names_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A derived frame keeps its root's composite provenance. Its bundle cites the composite's + component table, as every other bundle does, and names the composite with what it reads + and the destinations it keeps.""" + stores = Stores() + # Aid's change from 2021 to 2022, a step on one root, which keeps its provenance. + growth = step( + "growth", + dataset(AID, AID_V1), + window_start="2021", + window_end="2022", + entity_by=["financingRecipient"], + change="absolute", + ) + member = _stored(stores, aid=_aid(DAC2A_BILATERAL), tree=growth) + composite = DECLARED_COMPOSITES[DAC2A_BILATERAL] + (table,) = {recipe.dataset for recipe in composite.components} + + with _client(stores, monkeypatch) as client: + response = client.get(_path(member, "/file")) + + assert response.status_code == 200, response.text + with zipfile.ZipFile(io.BytesIO(response.content)) as bundle: + files = {name: bundle.read(name).decode() for name in bundle.namelist()} + descriptor = json.loads(files["datapackage.json"]) + # The roots here carry no stored citation, so the table is named by its ref. + assert [source["title"] for source in descriptor["sources"]] == [table] + cut = "counting bilateral flows to countries and regions, without core contributions" + for text in (files["README.md"], files["SOURCES.md"], descriptor["description"]): + assert "OECD DAC2a (bilateral flows)" in text + assert DAC2A_BILATERAL in text + assert cut in text + + def _locations(stores: Stores, member: dict[str, Any]) -> Any: return stores.snapshot_reader.read_artifact_locations( member["spec"]["spec_id"], member["content_version"] diff --git a/tests/datasets_api/test_family_sum_matches_summable.py b/tests/datasets_api/test_family_sum_matches_summable.py index d1d8146ff..f492f8975 100644 --- a/tests/datasets_api/test_family_sum_matches_summable.py +++ b/tests/datasets_api/test_family_sum_matches_summable.py @@ -16,7 +16,7 @@ import search.engine.retrieval.bound_values as bound_values_module from code_index import CodeIndex -from code_index.declarations import COMPOSITES +from code_index.declarations import COMPOSITES, DAC2A_BILATERAL from code_index.shared import get_index from datasets_api.adapter import resolve_definition_members from datasets_api.config import DatasetsBuildSettings @@ -227,6 +227,13 @@ def _composite_case(identity: dict, populated: download._Populated): _COMPOSITE_CASES = [ _composite_case(_identity(), download._stacked_crs()), _composite_case(_dac2a_oda_identity("Net"), download._stacked_dac2a()), + _composite_case( + { + **_dac2a_oda_identity("Net"), + "facets": {"provenance": {"state": "value", "ref": DAC2A_BILATERAL}}, + }, + download._stacked_dac2a(), + ), _composite_case(download._headline_identity(), download._spliced_headline()), ] @@ -269,7 +276,6 @@ def recording_init(self, *args, axis: str, precondition: str) -> None: served = _stat_vars( served_components(identity, facts=facts), ResolvedStatVars(member_count=0), - identity, declaration, facts=facts, ) diff --git a/tests/datasets_api/test_mint_dac2a_destinations.py b/tests/datasets_api/test_mint_dac2a_destinations.py new file mode 100644 index 000000000..fd93d6563 --- /dev/null +++ b/tests/datasets_api/test_mint_dac2a_destinations.py @@ -0,0 +1,439 @@ +"""An open destination under a DAC2a composite that counts bilateral flows reads bilateral-side +destinations alone. DAC2a aid type 206 also holds a row for each multilateral agency and a +`DACAllRecipients` row spanning both sides, so a mint that reads every destination counts core +contributions beside the countries' rows. + +The SDMX double holds one agency, one spanning and one country row of 206 and one country row +of the imputed 106, and honours the recipient filter a request sends, as the endpoint does. +""" + +from __future__ import annotations + +import gzip +import json +from collections.abc import Sequence + +import httpx +import pytest + +import code_index.composites as composites_module +from code_index.composites import destination_population +from code_index.declarations import COMPOSITES, DAC2A_BILATERAL, DAC2A_INCLUSIVE +from code_index.loader import parse_index +from code_index.scope_conventions import BILATERAL_SOURCES, destination_side +from code_index.shared import get_index +from datasets_api.adapter import composite_components +from dc_sdmx import SdmxClient +from materialise.errors import DestinationUnclassifiedError, FacetNotAvailableError +from series_coverage.schema import FacetMetadata, RoleNames +from spec_registry import recompute_identity +from spec_registry.reader import LiveSpecReader +from tests.code_index.support import INDEX_FIXTURE +from tests.engine._fake_coverage import FakeCoverageService, build_coverage_index + +from ._artifact_doubles import FakeArtifactStore +from ._graph_doubles import FakeMintGraphReader +from .conftest import FakeDatasetStore +from .test_definition_door import _client +from .test_mint_composite import ( + _CSV_HEADER, + _DAC2A, + _DAC2A_FACET, + _DAC2A_IMPUTED, + _IMPUTED_YEARS, + _PROVIDER, + _RECIPIENT, + _composite_client, + _csv_rows, + _dac2a_oda_identity, + _identity, + _mint, +) + +_BILATERAL = "provenance/ONEDAC2aBilateral" +_TOTAL_ODA = "Amount_FinancialFlow_Net_Disbursement_DAC2aAidType206_Nominal" +_AGENCY = "EuropeanCommission" +_SPANNING = "DACAllRecipients" +# What each (stat var, recipient) holds. +_HELD = { + (_TOTAL_ODA, _AGENCY): 200, + (_TOTAL_ODA, _SPANNING): 500, + (_TOTAL_ODA, _RECIPIENT): 100, + (_DAC2A_IMPUTED, _RECIPIENT): 7, +} + + +def _open_destination(provenance: str) -> dict: + identity = {**_dac2a_oda_identity("Net"), "facets": {"provenance": _ref(provenance)}} + identity["entities"] = {"origin": [_PROVIDER]} + identity["open_roles"] = ["destination"] + return identity + + +def _ref(provenance: str) -> dict: + return {"state": "value", "ref": provenance} + + +_REQUESTS: list[httpx.Request] = [] + + +def _respond(request: httpx.Request) -> httpx.Response: + """The held rows of the named variables, cut to the recipients the request names.""" + _REQUESTS.append(request) + named = str(request.url.params.get("c[variableMeasured]", "")).split(",") + recipients = request.url.params.get("c[financingRecipient]") + wanted = None if recipients is None else str(recipients).split(",") + body = _CSV_HEADER + "".join( + f"dataflow,DC:DF_OBS(1.0.0),I,{dcid},{_PROVIDER},{recipient},USD,GrantEquivalent," + f"NotApplicable,{_DAC2A},{year},{value},,{_DAC2A_FACET}\r\n" + for (dcid, recipient), value in _HELD.items() + if dcid in named and (wanted is None or recipient in wanted) + for year in _IMPUTED_YEARS + ) + return httpx.Response( + 200, text=body, headers={"content-type": "application/vnd.sdmx.data+csv;version=2.0.0"} + ) + + +def _open_client( + *, + artifacts: FakeArtifactStore | None = None, + store: FakeDatasetStore | None = None, + held_elsewhere: Sequence[str] = (), +): + """A client over `_HELD`, whose coverage also holds total ODA at each recipient of + `held_elsewhere`.""" + _REQUESTS.clear() + held = [*_HELD, *((_TOTAL_ODA, recipient) for recipient in held_elsewhere)] + coverage = FakeCoverageService( + build_coverage_index( + [ + (dcid, _DAC2A_FACET, _PROVIDER, recipient, list(_IMPUTED_YEARS)) + for dcid, recipient in held + ], + role_names={ + dcid: RoleNames(donor="financingProvider", recipient="financingRecipient") + for dcid, _recipient in held + }, + facet_metadata={ + _DAC2A_FACET: FacetMetadata( + provenance_id=_DAC2A, measurement_method="GrantEquivalent", unit="USD" + ) + }, + ) + ) + return _client( + graph=FakeMintGraphReader( + labels={ + _PROVIDER: "France", + _RECIPIENT: "Kenya", + _AGENCY: "European Commission", + _SPANNING: "Developing countries and multilateral agencies", + } + ), + sdmx_transport=httpx.MockTransport(_respond), + coverage=coverage, + artifact_store=artifacts, + store=store, + ) + + +def _mint_open_manifest(provenance: str, **options: object) -> tuple[dict, list[dict]]: + """The manifest of a mint of `_open_destination(provenance)` and its CSV rows.""" + artifacts = FakeArtifactStore() + client = _open_client(artifacts=artifacts) + minted = _mint(client, identity=_open_destination(provenance), options=options) + assert minted.status_code == 200, minted.text + return minted.json(), _csv_rows(artifacts, minted.json()) + + +def _mint_open(provenance: str, **options: object) -> list[dict]: + return _mint_open_manifest(provenance, **options)[1] + + +@pytest.mark.parametrize("provenance", [_BILATERAL, DAC2A_INCLUSIVE]) +def test_an_open_destination_mints_no_agency_or_spanning_row(provenance: str) -> None: + rows = _mint_open(provenance) + + assert {row["financingRecipient"] for row in rows} == {_RECIPIENT} + + +@pytest.mark.parametrize( + ("provenance", "total"), + [(_BILATERAL, 100), (DAC2A_INCLUSIVE, 100 + 7)], +) +def test_an_open_destination_sums_the_country_row_alone_and_inclusive_adds_106( + provenance: str, total: int +) -> None: + rows = _mint_open(provenance) + + sums: dict[str, float] = {} + for row in rows: + sums[row["time_period"]] = sums.get(row["time_period"], 0.0) + float(row["value"]) + assert sums == {"2020": total, "2021": total} + + +def test_the_bound_recipient_list_fits_one_request_under_the_url_budget() -> None: + """A bilateral read sends the population's recipients as one filter, 207 dcids on the + recorded index, about 3,800 bytes of the client's 7,800-byte budget, so no request splits + the population across requests.""" + _mint_open(_BILATERAL) + + members = set(destination_population(get_index(), _DAC2A).members) + budget = SdmxClient(base_url="https://example.org/core/api").url_budget_bytes + carried = [request for request in _REQUESTS if "c[financingRecipient]" in request.url.params] + assert carried, "no request carried the recipient filter" + for request in carried: + assert len(request.url.query) <= budget + assert set(str(request.url.params["c[financingRecipient]"]).split(",")) == members + + +@pytest.mark.parametrize("provenance", [_BILATERAL, DAC2A_INCLUSIVE]) +def test_an_open_destination_states_no_population_member_as_unserved(provenance: str) -> None: + """The population an open destination binds is no list the caller named, so the mint + counts none of its members it holds no rows at below the scope. Kenya is served in every + year, so nothing else falls below it.""" + manifest, _rows = _mint_open_manifest(provenance) + + below = [w for w in manifest["warnings"] if w["code"] == "coverage_below_scope"] + assert below == [] + + +@pytest.mark.parametrize("provenance", [_BILATERAL, DAC2A_INCLUSIVE]) +def test_a_composite_whose_population_coverage_cannot_place_refuses_before_any_read( + provenance: str, +) -> None: + """Coverage holding total ODA at a recipient the code index does not place leaves the + bilateral population unavailable, since its members would leave that recipient out + without a word: the mint refuses as a server fault naming it, and reads no rows.""" + client = _open_client(artifacts=FakeArtifactStore(), held_elsewhere=["SomeNewAgency"]) + + refused = _mint(client, identity=_open_destination(provenance)) + + assert refused.status_code == 500, refused.text + assert refused.json()["code"] == "internal_error" + assert "SomeNewAgency" in refused.json()["detail"] + assert _REQUESTS == [] + + +@pytest.mark.parametrize("provenance", [_BILATERAL, DAC2A_INCLUSIVE]) +def test_a_composite_whose_code_index_cannot_place_a_destination_is_a_server_fault( + monkeypatch: pytest.MonkeyPatch, provenance: str +) -> None: + """A code index publishing a destination no side declaration places leaves the population + unavailable: the components refuse as a server fault naming it, and not as the caller's + unservable composite.""" + + def unplaced(row, *, datasets): + return None if row.dcid == _RECIPIENT else destination_side(row, datasets=datasets) + + monkeypatch.setattr(composites_module, "destination_side", unplaced) + with gzip.open(INDEX_FIXTURE) as handle: + drifted = parse_index(json.load(handle), source="fixture", composites=COMPOSITES) + + with pytest.raises(DestinationUnclassifiedError) as refused: + composite_components(_open_destination(provenance), code_index=drifted) + + assert (refused.value.dataset, refused.value.destinations) == (_DAC2A, (_RECIPIENT,)) + + +_SOURCE = next(source for source in BILATERAL_SOURCES if source.dataset == _DAC2A) +_OFF_SIDE = [ + pytest.param( + DAC2A_INCLUSIVE, + _SOURCE.spanning, + "destination", + _SOURCE.rebound, + id="inclusive-spanning-total", + ), + pytest.param( + DAC2A_BILATERAL, + _SOURCE.spanning, + "destination", + _SOURCE.rebound, + id="bilateral-spanning-total", + ), + pytest.param(DAC2A_BILATERAL, _AGENCY, "provenance", _DAC2A, id="bilateral-agency"), + pytest.param( + DAC2A_INCLUSIVE, + "DACAllMultilateralRecipients", + "provenance", + _DAC2A, + id="inclusive-multilateral-total", + ), +] + + +def _off_side_identity(provenance: str, destination: str) -> dict: + """France's DAC2a total ODA to `destination` read through the composite `provenance`.""" + identity = {**_dac2a_oda_identity("Net"), "facets": {"provenance": _ref(provenance)}} + identity["entities"] = {"origin": [_PROVIDER], "destination": [destination]} + return identity + + +def _assert_off_side_refusal( + refused: httpx.Response, *, destination: str, axis: str, fix: str +) -> None: + """`refused` is a 422 `facet_not_available` whose one issue names `destination` and the + `fix` to bind on `axis` in its place.""" + assert refused.status_code == 422, refused.text + problem = refused.json() + assert problem["code"] == "facet_not_available" + [issue] = problem["issues"] + assert issue["member"] == destination + assert issue["axis"] == axis + assert [alternative["ref"] for alternative in issue["alternatives"]] == [fix] + + +@pytest.mark.parametrize(("provenance", "destination", "axis", "fix"), _OFF_SIDE) +def test_the_mint_door_refuses_a_composite_bound_to_a_destination_off_its_side( + provenance: str, destination: str, axis: str, fix: str +) -> None: + """A definition binding a DAC2a composite that reads bilateral destinations to one off + that side is the client's fault: the mint door refuses it 422 before any read and names + the fix, the total of developing countries for the total of all recipients and the plain + DAC2a table for an agency or the total of them. Nothing is registered.""" + store = FakeDatasetStore() + client = _open_client(artifacts=FakeArtifactStore(), store=store) + identity = _off_side_identity(provenance, destination) + + refused = _mint(client, identity=identity) + + _assert_off_side_refusal(refused, destination=destination, axis=axis, fix=fix) + assert _REQUESTS == [] + assert LiveSpecReader(store=store).read_definition(recompute_identity(identity).spec_id) is None + + +@pytest.mark.parametrize(("provenance", "destination", "axis", "fix"), _OFF_SIDE[1:3]) +def test_the_definition_door_refuses_a_composite_bound_to_a_destination_off_its_side( + provenance: str, destination: str, axis: str, fix: str +) -> None: + """The options route refuses such a definition 422 as the mint does, before it registers + the definition.""" + store = FakeDatasetStore() + client = _open_client(artifacts=FakeArtifactStore(), store=store) + identity = _off_side_identity(provenance, destination) + spec_id = recompute_identity(identity).spec_id + + refused = client.post( + f"/api/datasets/{spec_id}/options", + json={"definition": {"schema_version": 1, "spec_id": spec_id, "identity": identity}}, + ) + + _assert_off_side_refusal(refused, destination=destination, axis=axis, fix=fix) + assert LiveSpecReader(store=store).read_definition(spec_id) is None + + +@pytest.mark.parametrize(("provenance", "destination", "axis", "fix"), _OFF_SIDE) +def test_the_off_side_refusal_points_where_its_fix_is( + provenance: str, destination: str, axis: str, fix: str +) -> None: + """The total spanning both sides is fixed on the destination role, so its refusal points + there; an agency or the total of them is fixed on the provenance facet.""" + with pytest.raises(FacetNotAvailableError) as refused: + composite_components(_off_side_identity(provenance, destination), code_index=get_index()) + + assert refused.value.facet_pointer == ( + "/entities/destination" if axis == "destination" else "/facets/provenance" + ) + + +def _mint_scoped(client, *, identity: dict, scope: dict, sidecar: bool) -> httpx.Response: + """A mint of `identity`'s spec_id under `scope`, posting the sidecar when `sidecar`.""" + spec_id = recompute_identity(identity).spec_id + definition = {"schema_version": 1, "spec_id": spec_id, "identity": identity} + return client.post( + "/api/datasets", + json={ + "schema_version": 1, + "input": { + "kind": "spec_id", + "spec_id": spec_id, + **({"definition": definition} if sidecar else {}), + "scope": scope, + }, + "options": {"bundle": False}, + }, + ) + + +def _sent_recipients() -> set[str]: + """The recipients the SDMX requests made so far name.""" + return { + recipient + for request in _REQUESTS + for recipient in str(request.url.params.get("c[financingRecipient]", "")).split(",") + if recipient + } + + +@pytest.mark.parametrize("registered", [False, True], ids=["sidecar", "registered"]) +@pytest.mark.parametrize("provenance", [_BILATERAL, DAC2A_INCLUSIVE]) +def test_a_scope_binding_the_destination_reads_that_destination_alone( + provenance: str, registered: bool +) -> None: + """A scope's entities replace the definition's, so a composite with an open destination + minted under a scope binding Kenya reads Kenya alone, whether the definition is posted + beside the mint or registered before it.""" + artifacts = FakeArtifactStore() + client = _open_client(artifacts=artifacts) + identity = _open_destination(provenance) + if registered: + assert _mint(client, identity=identity).status_code == 200 + _REQUESTS.clear() + scope = {"entities": {"origin": [_PROVIDER], "destination": [_RECIPIENT]}} + + minted = _mint_scoped(client, identity=identity, scope=scope, sidecar=not registered) + + assert minted.status_code == 200, minted.text + assert _sent_recipients() == {_RECIPIENT} + assert {row["financingRecipient"] for row in _csv_rows(artifacts, minted.json())} == { + _RECIPIENT + } + + +@pytest.mark.parametrize("registered", [False, True], ids=["sidecar", "registered"]) +def test_a_scope_binding_an_agency_refuses_before_any_read(registered: bool) -> None: + """A scope binding the bilateral composite's destination to an agency is refused 422 with + the plain table to read in its place, before any read, as the definition itself would + be.""" + store = FakeDatasetStore() + client = _open_client(artifacts=FakeArtifactStore(), store=store) + identity = _open_destination(_BILATERAL) + if registered: + assert _mint(client, identity=identity).status_code == 200 + _REQUESTS.clear() + scope = {"entities": {"origin": [_PROVIDER], "destination": [_AGENCY]}} + + refused = _mint_scoped(client, identity=identity, scope=scope, sidecar=not registered) + + _assert_off_side_refusal(refused, destination=_AGENCY, axis="provenance", fix=_DAC2A) + assert _REQUESTS == [] + + +_ONE_YEAR = {"window": {"kind": "range", "start_year": 2021, "end_year": 2021}} + + +@pytest.mark.parametrize("registered", [False, True], ids=["sidecar", "registered"]) +@pytest.mark.parametrize("source", ["crs-inclusive", "dac2a-bilateral", "dac2a-inclusive"]) +def test_a_scope_narrowing_the_window_reads_that_year_alone(source: str, registered: bool) -> None: + """A scope's window replaces the definition's, so a composite minted under a scope of one + year serves that year alone, whether the definition is posted beside the mint or + registered before it.""" + if source == "crs-inclusive": + artifacts, client = _composite_client() + identity = _identity() + else: + artifacts = FakeArtifactStore() + client = _open_client(artifacts=artifacts) + identity = _open_destination(_BILATERAL if source == "dac2a-bilateral" else DAC2A_INCLUSIVE) + assert artifacts is not None + if registered: + assert _mint(client, identity=identity).status_code == 200 + + minted = _mint_scoped(client, identity=identity, scope=_ONE_YEAR, sidecar=not registered) + + assert minted.status_code == 200, minted.text + rows = _csv_rows(artifacts, minted.json()) + assert rows + assert {row["time_period"] for row in rows} == {"2021"} diff --git a/tests/datasets_api/test_mint_error_classification.py b/tests/datasets_api/test_mint_error_classification.py index 5e555aa31..7bb3f6a1b 100644 --- a/tests/datasets_api/test_mint_error_classification.py +++ b/tests/datasets_api/test_mint_error_classification.py @@ -41,6 +41,7 @@ from materialise.errors import ( DatasetDisagreementError, DefaultFacetUndefinedError, + DestinationUnclassifiedError, FacetNotAvailableError, MembersNotHeldError, UpstreamResponseError, @@ -701,6 +702,19 @@ def test_a_dataset_disagreement_is_a_server_fault_whose_detail_names_the_dataset assert "provenance/Elected" in spec.detail assert [issue.member for issue in spec.issues or ()] == ["dcid:Member"] + def test_an_unclassified_destination_is_a_server_fault_whose_detail_names_it(self): + exc = DestinationUnclassifiedError( + destinations=["SomeNewAgency"], dataset="provenance/OECDDAC2A" + ) + + spec = classify_mint_exception(exc) + + assert spec is not None + assert spec.status == 500 + assert spec.code == "internal_error" + assert spec.retryable is False + assert "SomeNewAgency" in spec.detail + def test_a_frame_the_grammar_refuses_is_a_typed_server_fault(self): exc = FrameError( "unknown_column", "row carries unknown column(s): ['stat_var_exampleAxis']" diff --git a/tests/datasets_api/test_render_bundle_documents.py b/tests/datasets_api/test_render_bundle_documents.py index c8854527c..a73cd773e 100644 --- a/tests/datasets_api/test_render_bundle_documents.py +++ b/tests/datasets_api/test_render_bundle_documents.py @@ -15,6 +15,7 @@ import pytest from code_index import CodeIndex, CodeRow, CodeSystemDeclaration, IndexStamp +from code_index.declarations import CRS_INCLUSIVE from code_index.shared import get_index from code_index.types import StatVarRow from datasets_api.manifest_models import ( @@ -31,6 +32,7 @@ from dfo_platform.config import PublicSiteSettings from materialise.bundle import ReproducingCall from materialise.enrichment import EnrichmentLookup +from materialise.errors import MaterialisationUsageError from ._artifact_doubles import FakeArtifactStore from ._frame_chunks import frame_of_rows @@ -154,14 +156,21 @@ def node_arcs_many(self, dcids, *, properties, individual_probe_budget=None, dea raise GraphReadError("graph read past its deadline") -def _render_crs(*, graph: FakeMintGraphReader | None = None) -> dict[str, bytes]: +def _render_crs( + *, + graph: FakeMintGraphReader | None = None, + provenance: str = _CRS, + with_index: bool = True, +) -> dict[str, bytes]: + """The bundle of `_CRS_TABLE`, its rows read as carrying `provenance`, rendered over the + bound release's code index, or none without `with_index`.""" store = FakeArtifactStore() options = MintOptions(format="csv", bundle=True) rendered = render_and_store( frame_chunks=lambda: iter((_CRS_TABLE,)), options=options, spec=_CRS_SPEC, - provenance_refs=[_CRS], + provenance_refs=[provenance], citations={_CRS: _CRS_CITATION}, reproducing_call=ReproducingCall( path=f"/api/datasets/{_CRS_SPEC_ID}", options={"format": "csv", "bundle": "true"} @@ -174,7 +183,7 @@ def _render_crs(*, graph: FakeMintGraphReader | None = None) -> dict[str, bytes] enrichment=EnrichmentLookup(graph or FakeMintGraphReader(arcs=_GRAPH_ARCS)), pivot_max_output_cells=5_000_000, groupby_fold_max_rows=2_000_000, - code_index=get_index(), + code_index=get_index() if with_index else None, resolved_stat_var_refs=(), ) return _unzip(store.fetch(rendered.artifact.url)) @@ -300,6 +309,13 @@ def test_a_crs_bundle_validates_and_names_every_code_column_and_link(tmp_path): _SITE_LINK = f"https://site.example/data?spec={_CRS_SPEC_ID}&v={_CRS_VERSION}" +def test_a_composite_render_without_a_code_index_refuses(): + """A composite is named and described from its declaration over the code index, so a + render that has none refuses rather than write a bundle with no word of it.""" + with pytest.raises(MaterialisationUsageError, match="provenance/ONEBilateral"): + _render_crs(provenance=CRS_INCLUSIVE, with_index=False) + + @pytest.mark.parametrize( ("spec", "site", "printed"), [ diff --git a/tests/datasets_api/test_sync_estimate.py b/tests/datasets_api/test_sync_estimate.py index 0195fbdfb..c0999354f 100644 --- a/tests/datasets_api/test_sync_estimate.py +++ b/tests/datasets_api/test_sync_estimate.py @@ -25,7 +25,7 @@ from datasets_api.current import DefinitionDeps, plan_stored_render from datasets_api.manifest_models import Manifest from datasets_api.mint_models import MintOptions -from dc_sdmx import ContainedIn +from dc_sdmx import Among, ContainedIn from materialise.errors import MaterialisationError, OverCeilingError from materialise.facets import ElectedDataset, FacetBinding from materialise.formats import EXCEL_MAX_ROWS @@ -66,7 +66,7 @@ def _request( *, stat_var: str, donor: str | ContainedIn | None = None, - recipient: str | ContainedIn | None = None, + recipient: str | ContainedIn | Among | None = None, declared_roles: tuple[str, ...] = ("financingProvider", "financingRecipient"), window: RequestedWindow = _ALL_WINDOW, ) -> MaterialisationRequest: @@ -75,7 +75,7 @@ def _request( role_bindings[declared_roles[0]] = donor if isinstance(donor, ContainedIn) else [donor] if recipient is not None: role_bindings[declared_roles[1]] = ( - recipient if isinstance(recipient, ContainedIn) else [recipient] + recipient if isinstance(recipient, ContainedIn | Among) else [recipient] ) return MaterialisationRequest( stat_var=f"dcid:{stat_var}", @@ -332,6 +332,20 @@ def test_contained_in_bound_role_is_treated_as_unbound_a_safe_over_estimate(): assert contained_estimate.rows == 4 +def test_a_population_bound_role_prices_only_its_members_rows(): + """A role bound to a population (`Among`) is bound to its members: Uganda's row, outside + the population, is not charged.""" + request = _request( + stat_var="Amount_Crs_A", + donor="country/DEU", + recipient=Among(members=("country/KEN",), side="bilateral"), + ) + estimate = sync_estimate( + (request,), index=_INDEX, member_concurrency=_SETTINGS.mint_member_max_concurrency + ) + assert estimate.rows == 3 + + def test_within_sync_budget_at_and_over_the_boundary(): assert ( within_sync_budget( diff --git a/tests/datasets_api/test_warning_backfill.py b/tests/datasets_api/test_warning_backfill.py index 87e06f561..75e677f97 100644 --- a/tests/datasets_api/test_warning_backfill.py +++ b/tests/datasets_api/test_warning_backfill.py @@ -29,6 +29,7 @@ from dc_sdmx import SdmxHttpError, SdmxUnsupportedFilterError from materialise import facets as facets_module from materialise.codes import MATERIALISE_CODE_CATALOGUE +from materialise.errors import DestinationUnclassifiedError, FacetNotAvailableError from materialise.finish import COUNTRIES_FALLBACK_SENTENCE, member_sentences from materialise.graph import Facet, GraphReadError from origin_catalogue import record_un_read_failure @@ -893,16 +894,38 @@ def observation_facets_batch_many(self, **kwargs): raise ValueError("a defect, not a failed read") +def _raising(error: Exception) -> Callable[..., object]: + def composite_components(*args: object, **kwargs: object) -> object: + raise error + + return composite_components + + @pytest.mark.parametrize( ("fault", "reason"), - [("ceiling", "probe_refused"), ("malformed-identity", "identity_refused")], + [ + ("ceiling", "probe_refused"), + ("malformed-identity", "identity_refused"), + ("off-side-destination", "facet_not_available"), + ("population-unavailable", "destination_unclassified"), + ], ) def test_a_deterministic_failure_is_refused_for_good(fault, reason, monkeypatch, capsys): + from datasets_api import warning_backfill + row = _row(_warning([_IN]), zero=[_IN]) if fault == "ceiling": monkeypatch.setattr(facets_module, "_MAX_COLLECTION_PROBE_PAIRS", 0) - else: + elif fault == "malformed-identity": row["definition"]["identity"]["facets"] = {"unit": {"ref": "USD", "request": {}}} + else: + # The composite derivation's refusals (`adapter.composite_components`). + error = ( + FacetNotAvailableError("no rows there", facet_pointer="/entities/destination") + if fault == "off-side-destination" + else DestinationUnclassifiedError(destinations=["dcid:x"], dataset="OECD_DAC2A") + ) + monkeypatch.setattr(warning_backfill, "composite_components", _raising(error)) store = _Store([row]) assert _run(store) == 0 diff --git a/tests/engine/test_composite_answer.py b/tests/engine/test_composite_answer.py index 95e3cb10c..2a0ba151f 100644 --- a/tests/engine/test_composite_answer.py +++ b/tests/engine/test_composite_answer.py @@ -8,14 +8,17 @@ import dataclasses import threading from collections.abc import Sequence +from types import SimpleNamespace from typing import Any, cast import pytest import dc_graph +from code_index.shared import get_index from origin_catalogue import current_catalogue, scoped_reads from search.engine import response_cache_flow, same_shape from search.engine.concurrency import scoped_concurrency +from search.engine.enrichment import _launch_trailing_tasks from search.engine.entry import dataset_resubmit from search.engine.errors import ( EngineInfraError, @@ -23,15 +26,26 @@ GraphDeadlineFailure, GraphInfraError, ) -from search.engine.events import DoneEvent, Event, ObservationsEvent, ResponseEvent -from search.engine.region import composite_answer, pipeline -from search.engine.region.codes import MEMBERS_ABSENT_FROM_GRAPH, _ObsProbeResult +from search.engine.events import ( + DoneEvent, + Event, + FacetOptionsEvent, + ObservationsEvent, + ResponseEvent, +) +from search.engine.region import composite_answer, pipeline, place_options, result +from search.engine.region.codes import ( + MEMBERS_ABSENT_FROM_GRAPH, + _ObsProbeResult, + _OptionProbeResult, +) from search.engine.region.composite_answer import ( ComponentReads, CompositeRelaunch, composite_served, read_composite_preview, ) +from search.engine.region.promote_relaunch import bind_relaunches, without_relaunches from search.engine.region.result import RegionResult from search.engine.region.state import EngineOptions, ResolveState from search.engine.response_cache import ResponseCache @@ -41,6 +55,7 @@ CoverageExact, DatasetResubmitInput, DefinitePart, + Entry, IdentityBindingSet, IdentityBindingValue, ObservationPoint, @@ -61,11 +76,14 @@ ) from search.scope import composite from search.scope.composite import ServedComponent, compose_observations +from search.scope.state import ScopeState +from search.scope.tables import ScopeFacts from series_coverage.shared import CoverageSnapshot from tests.engine._fake_coverage import FakeCoverageService, patch_coverage from tests.engine._fake_sdmx import FakeSdmxClient from tests.engine._harness import offline_events, offline_resolve from tests.scope import test_provider as provider +from tests.scope._axes import carried, explicit from tests.scope.test_basis import _graph, _PickingLLM _WINDOW = range(2015, 2021) @@ -535,13 +553,13 @@ def capture(result: RegionResult, state: ResolveState) -> RegionResult: previewed = _dates(_observed(_stream(_fresh())).observations) (result, state), *_ = kept scopes: list[frozenset[str] | None] = [] - assemble = pipeline.assemble_composite + assemble = composite_answer.assemble_composite def record(*args: Any, **kwargs: Any) -> Any: scopes.append(current_catalogue().scope) return assemble(*args, **kwargs) - monkeypatch.setattr(pipeline, "assemble_composite", record) + monkeypatch.setattr(composite_answer, "assemble_composite", record) held = composite._component_years def narrowed(identity: Any, *, facts: Any) -> composite.YearCounts: @@ -986,3 +1004,412 @@ async def run() -> _ObsProbeResult: [provider._KEN, _UGA] ] * 2 assert _dates(observations) == _BOTH + + +def _dac2a_and_crs_snapshot(monkeypatch: pytest.MonkeyPatch) -> CoverageSnapshot: + """A snapshot of the CRS rows `provider._coverage_rows` patches in and DAC2a's rows from + the United States to Kenya in every year.""" + rows, extra = provider._coverage_rows(provider._ALL_YEARS) + dac2a_rows = [ + (dcid, "dac2a", provider._USA, provider._KEN, list(provider._ALL_YEARS)) + for dcid in provider.basis._DAC2A_MEMBERS + ] + snapshot = FakeCoverageService( + patch_coverage( + monkeypatch, + [*rows, *dac2a_rows], + role_names={ + **extra["role_names"], + **dict.fromkeys(provider.basis._DAC2A_MEMBERS, provider._FLOW), + }, + facet_metadata={ + **extra["facet_metadata"], + "dac2a": provider.FacetMetadata(provenance_id=provider._DAC2A, unit="USD"), + }, + ) + ).snapshot() + return snapshot + + +def test_a_candidate_set_restating_a_dac2a_reading_keeps_its_other_readings_options( + monkeypatch: pytest.MonkeyPatch, +): + """A candidate set of a DAC2a reading and a CRS reading restates the DAC2a one as its + bilateral destinations. Its trailing slot options keep the CRS reading's events and + withhold the DAC2a reading's, which read the table under a spec_id the set does not + show, as a composite answer carries none.""" + snapshot = _dac2a_and_crs_snapshot(monkeypatch) + crs = provider._ask( + "health aid from the United States to Kenya, 2015 to 2020", time="2015-2020" + ) + dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) + assert provider._source(crs) is None and provider._source(dac2a) is None + + def events(spec_id: str) -> list[FacetOptionsEvent]: + return [FacetOptionsEvent(spec_id=spec_id, axis="unit", options=[], truncated=False)] + + async def run() -> tuple[RegionResult, list[str]]: + async with scoped_concurrency(): + + async def probe() -> _OptionProbeResult: + return (*events(dac2a.spec_id), *events(crs.spec_id)), () + + before = RegionResult( + variable_text="aid", + status="candidates", + specs=(dac2a, crs), + no_data_reason=None, + warnings=(), + timing_by_step={}, + slot_options_task=asyncio.ensure_future(probe()), + ) + after = _readings(before, reads=_reads(snapshot), selection=ScopeState()) + after = composite_answer.without_restated_options(before, after) + assert after.slot_options_task is not None + served, _ = await after.slot_options_task + return after, [event.spec_id for event in served] + + after, served = asyncio.run(run()) + + restated, kept = after.specs + assert provider._source(restated) == provider._DAC2A_BILATERAL + assert kept.spec_id == crs.spec_id + assert served == [crs.spec_id] + + +@pytest.mark.parametrize( + ("entries", "source"), + [ + pytest.param((), provider._DAC2A_BILATERAL, id="no-entry"), + pytest.param((explicit("provider:contributions"),), None, id="stated-total"), + pytest.param((carried("provider:contributions"),), None, id="carried-total"), + pytest.param((explicit("provider:bilateral"),), provider._DAC2A_BILATERAL, id="stated"), + ], +) +def test_a_reading_beside_the_answer_honours_the_part_s_provider_entry( + monkeypatch: pytest.MonkeyPatch, entries: tuple[Entry, ...], source: str | None +): + """A candidate, alternate, flagged or nearest reading of DAC2a's total ODA is read as the + definite answer is: a stated or carried `provider:contributions` keeps total ODA, and + otherwise the reading is restated as the bilateral destinations.""" + snapshot = _dac2a_and_crs_snapshot(monkeypatch) + dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) + before = RegionResult( + variable_text="aid", + status="candidates", + specs=(dac2a,), + no_data_reason=None, + warnings=(), + timing_by_step={}, + alternate_match=dac2a, + flagged_match=(dac2a,), + nearest_real=(dac2a,), + ) + + after = _readings(before, reads=_reads(snapshot), selection=ScopeState(entries=entries)) + + assert after.alternate_match is not None + assert after.flagged_match is not None and after.nearest_real is not None + readings = (*after.specs, after.alternate_match, *after.flagged_match, *after.nearest_real) + assert [provider._source(spec) for spec in readings] == [source] * 4 + + +def test_a_set_left_with_a_crs_reading_answers_the_composite_its_scope_compiles( + monkeypatch: pytest.MonkeyPatch, +): + """A candidate set of a DAC2a reading whose population holds no row and a CRS reading of + total health aid is left with the CRS reading, whose scope decision compiles the inclusive + composite though no DAC2a default serves it as one. It answers that composite as a fresh + definite answer of the reading does: the composite's spec_id, the compiled scope and its + components, and the closure that reads its preview, never the table under a composite's + scope.""" + snapshot = _dac2a_and_crs_snapshot(monkeypatch) + dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) + _components(monkeypatch) + kept: list[tuple[RegionResult, ResolveState]] = [] + real = pipeline._with_scope_choices + + def capture(result: RegionResult, state: ResolveState) -> RegionResult: + kept.append((result, state)) + return real(result, state) + + monkeypatch.setattr(pipeline, "_with_scope_choices", capture) + _stream(_fresh()) + (table, state), *_ = kept + fresh = real(table, state) + assert fresh.scope_choices is not None and fresh.spec.spec_id != table.spec.spec_id + read = composite_answer.population_table_answer + + def holding_no_dac2a_row(spec: Spec, identity: BaseIdentity, *, reads: Any) -> Any: + if spec.spec_id == dac2a.spec_id: + return composite_answer._unread_population("no_observations") + return read(spec, identity, reads=reads) + + monkeypatch.setattr(composite_answer, "population_table_answer", holding_no_dac2a_row) + before = dataclasses.replace(table, status="candidates", specs=(dac2a, table.spec)) + + after = _readings(before, reads=_reads(snapshot), selection=state.scope_selection) + + assert (after.status, after.spec.spec_id) == ("definite", fresh.spec.spec_id) + assert after.scope_choices is not None + assert after.scope_choices.compiled == fresh.scope_choices.compiled + assert after.scope_choices.compiled.components + assert isinstance(after.observations_relaunch, CompositeRelaunch) + + +def test_a_restated_reading_relaunches_no_table_probe_on_a_replay( + monkeypatch: pytest.MonkeyPatch, +): + """A set whose own reading is restated as DAC2a's bilateral destinations carries no + closure that reads the table again, nor the facts a part memo hit binds them from: a + replay that launches the trailing tasks it lacks (`promote_relaunch.bind_relaunches`, + `enrichment._launch_trailing_tasks`) launches none under the composite's spec_id, as a + composite answer launches none.""" + snapshot = _dac2a_and_crs_snapshot(monkeypatch) + dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) + launched: list[str] = [] + + def relaunch(*args: Any) -> Any: + launched.append(repr(args)) + raise AssertionError("a restated reading relaunched a table probe") + + before = RegionResult( + variable_text="aid", + status="candidates", + specs=(dac2a,), + no_data_reason=None, + warnings=(), + timing_by_step={}, + slot_options_relaunch=relaunch, + place_options_relaunch=relaunch, + observations_relaunch=relaunch, + promote_relaunch=cast(Any, object()), + ) + + after = _readings(before, reads=_reads(snapshot), selection=ScopeState()) + replayed = bind_relaunches( + without_relaunches(after), + graph=cast(Any, None), + sdmx=cast(Any, None), + coverage=snapshot, + options=cast( + Any, + SimpleNamespace( + stream_slot_options=True, + stream_observations=True, + materialise_graph=object(), + place_options_pool=object(), + ), + ), + ) + launched_region = _launch_trailing_tasks( + replayed, launch_slot_options=True, launch_observations=True + ) + + assert provider._source(after.spec) == provider._DAC2A_BILATERAL + assert ( + launched_region.slot_options_task, + launched_region.place_options_task, + launched_region.observations_task, + ) == (None, None, None) + assert launched == [] + + +def test_a_population_read_keeps_the_request_s_warnings_and_replaces_the_read_s( + monkeypatch: pytest.MonkeyPatch, +): + """A fresh answer read again within its composite's population keeps the warnings raised + before its read and takes every other from that read: a whole-table count, and a + currency omission naming an agency the population leaves out, do not survive, whether + or not the population read raises their code.""" + _dac2a_and_crs_snapshot(monkeypatch) + table = provider._resubmitted(provider._dac2a_identity("Net"), None) + + def warning(code: str, message: str) -> Warning: + return Warning(code=code, severity="warn", message=message) + + words = warning("query_qualifier_dropped", "words") + fresh = RegionResult( + variable_text="aid", + status="definite", + specs=(table,), + no_data_reason=None, + warnings=( + words, + warning("role_scope_defaulted", "both"), + warning("currency_unmapped_omitted", "European Commission"), + ), + timing_by_step={}, + ) + answered = dataclasses.replace(fresh, warnings=(warning("role_scope_defaulted", "one"),)) + + grafted = composite_answer.with_population_read(fresh, answered, request_warnings=(words,)) + + assert [(w.code, w.message) for w in grafted.warnings] == [ + ("query_qualifier_dropped", "words"), + ("role_scope_defaulted", "one"), + ] + + +def test_a_cache_hit_launches_no_place_options_for_a_composite_answer( + monkeypatch: pytest.MonkeyPatch, +): + """A response cache hit launches the place options a fresh run launches: one task per + definite table answer, none for a composite answer, whose place options a fresh run + drops (`result.with_composite_answer`).""" + composite_part = provider._posted_dac2a(monkeypatch, provider._DAC2A_BILATERAL, provider._KEN) + table_part = provider._resubmitted_part( + provider._dac2a_identity("Gross", destination=provider._DEVELOPING), None + ) + assert provider._source(composite_part.spec) == provider._DAC2A_BILATERAL + assert provider._source(table_part.spec) is None + launched: list[str] = [] + + def recorded(spec: Spec, **_kwargs: Any) -> Any: + launched.append(spec.spec_id) + return spec.spec_id + + monkeypatch.setattr(place_options, "place_options_task", recorded) + launch = place_options.replayed_place_options( + stream=True, + graph=cast(Any, None), + sdmx=cast(Any, None), + materialise_graph=cast(Any, object()), + materialise_sdmx=None, + pool=cast(Any, object()), + coverage=cast(Any, None), + population_memo=None, + ) + assert launch is not None + + launch(cast(Any, SimpleNamespace(kind="resolution", parts=[composite_part, table_part]))) + + assert launched == [table_part.spec.spec_id] + + +def test_a_restated_reading_reads_the_request_catalogue_whatever_scope_the_part_read_under( + monkeypatch: pytest.MonkeyPatch, +): + """A candidate restated as DAC2a's bilateral destinations is settled and restated through + the request's catalogue, as the definite answer's composite is, so a source scope the + part's own reads ran under narrows none of its figures.""" + snapshot = _dac2a_and_crs_snapshot(monkeypatch) + dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) + reads = _reads(snapshot) + scopes: dict[str, list[frozenset[str] | None]] = {"default": [], "assemble": []} + default, assemble = composite_answer.bilateral_default_reading, composite.assemble_composite + + def recorded_default(*args: Any, **kwargs: Any) -> Any: + scopes["default"].append(current_catalogue().scope) + return default(*args, **kwargs) + + def recorded_assemble(*args: Any, **kwargs: Any) -> Any: + scopes["assemble"].append(current_catalogue().scope) + return assemble(*args, **kwargs) + + monkeypatch.setattr(composite_answer, "bilateral_default_reading", recorded_default) + monkeypatch.setattr(composite_answer, "assemble_composite", recorded_assemble) + before = RegionResult( + variable_text="aid", + status="candidates", + specs=(dac2a,), + no_data_reason=None, + warnings=(), + timing_by_step={}, + ) + + with scoped_reads(frozenset({provider._DAC2A})): + (restated,) = _readings(before, reads=reads, selection=ScopeState()).specs + + assert provider._source(restated) == provider._DAC2A_BILATERAL + assert scopes == {"default": [reads.catalogue.scope], "assemble": [reads.catalogue.scope]} + + +def test_a_restated_answer_rekeys_only_its_own_keyed_facts(monkeypatch: pytest.MonkeyPatch): + """Restating a DAC2a answer as its bilateral destinations moves its own warnings, + unpublished-whole facts and assumption echoes to the composite's spec_id, and drops its + own all-recipients total, which the composite cannot stand beside. Another reading's + warnings and a warning keyed to no reading stand as they are.""" + snapshot = _dac2a_and_crs_snapshot(monkeypatch) + crs = provider._ask( + "health aid from the United States to Kenya, 2015 to 2020", time="2015-2020" + ) + table = provider._resubmitted(provider._dac2a_identity("Net"), None) + restated = _readings( + RegionResult( + variable_text="aid", + status="candidates", + specs=(table,), + no_data_reason=None, + warnings=(), + timing_by_step={}, + ), + reads=_reads(snapshot), + selection=ScopeState(), + ).spec + assert provider._source(restated) == provider._DAC2A_BILATERAL + + def warning(code: str, spec_id: str | None) -> Warning: + params = {"total": provider._ALL_RECIPIENTS} if code == "whole_total_available" else None + return Warning(code=code, severity="info", message=code, spec_id=spec_id, params=params) + + region = RegionResult( + variable_text="aid", + status="definite", + specs=(table, crs), + no_data_reason=None, + warnings=( + warning("role_scope_defaulted", table.spec_id), + warning("whole_total_available", table.spec_id), + warning("whole_total_available", crs.spec_id), + warning("whole_total_available", None), + ), + timing_by_step={}, + unpublished_whole_facts={table.spec_id: (), crs.spec_id: ()}, + reading_assumption_echoes={table.spec_id: (), crs.spec_id: ()}, + ) + + answered = result.with_composite_answer( + region, restated, observations_relaunch=cast(Any, lambda _spec_id: None) + ) + + assert [(w.code, w.spec_id) for w in answered.warnings] == [ + ("role_scope_defaulted", restated.spec_id), + ("whole_total_available", crs.spec_id), + ("whole_total_available", None), + ] + assert set(answered.unpublished_whole_facts) == {restated.spec_id, crs.spec_id} + assert set(answered.reading_assumption_echoes) == {restated.spec_id, crs.spec_id} + + +def test_a_merge_of_composite_parts_with_no_axis_states_the_composite_scope( + monkeypatch: pytest.MonkeyPatch, +): + """A DAC2a ranking over every donor has no axis, and its single part states a scope with + no choice whose compiled identity is its bilateral destinations. A same-shape merge of + such parts states that scope too, as the field's description says, never none.""" + facts = provider._dac2a_facts(provider._ALL_YEARS) + identity = provider._dac2a_identity("Net", donor=None, destination=None) + single = provider._fresh_scope(identity, facts) + assert single is not None and single.choices == [] + assert provider._provenance(single.compiled.identity) == provider._DAC2A_BILATERAL + monkeypatch.setattr(pipeline, "build_part_scope", lambda **_: single) + monkeypatch.setattr(pipeline, "scope_facts", lambda *_, **__: facts) + spec = SimpleNamespace(definition=SimpleNamespace(identity=identity)) + merged = cast(RegionResult, SimpleNamespace(spec=spec, operations=())) + group = [cast(RegionResult, SimpleNamespace(scope_choices=single))] * 2 + + assert pipeline.merged_scope_choices(merged, group, coverage=facts.coverage) == single + + +def _readings( + region: RegionResult, *, reads: ComponentReads, selection: ScopeState +) -> RegionResult: + """`region` with its readings restated (`with_bilateral_readings`) under scope facts of + no operation, as a part with no answer phase decision reads them.""" + return composite_answer.with_bilateral_readings( + region, + reads=reads, + selection=selection, + facts=ScopeFacts(index=get_index(), coverage=reads.coverage), + ) diff --git a/tests/engine/test_dac2a_destination_populations.py b/tests/engine/test_dac2a_destination_populations.py new file mode 100644 index 000000000..77aaa7be9 --- /dev/null +++ b/tests/engine/test_dac2a_destination_populations.py @@ -0,0 +1,1391 @@ +"""A DAC2a composite that counts bilateral flows reads bilateral-side destinations alone, at +Search's identity resubmit door, at its observations preview, and on a fresh answer. + +DAC2a aid type 206 holds a row for each multilateral agency and a `DACAllRecipients` row +spanning both sides, so an open destination that reads every row counts core contributions +beside the countries' rows. The fixture holds France's 206 to Ethiopia, to the European +Commission and to `DACAllRecipients`, and its imputed 106 to Ethiopia. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import logging +import re +import threading +import time +from collections.abc import Callable +from typing import Any + +import pytest + +import dc_graph +from code_index.composites import CompositeNotServable, destination_population +from code_index.declarations import DAC2A_INCLUSIVE +from code_index.shared import get_index +from dc_sdmx import SdmxClient as MaterialiseSdmxClient +from origin_catalogue import current_catalogue +from search.engine.clients.graph import Facet +from search.engine.concurrency import scoped_concurrency +from search.engine.core import ResolveRecord +from search.engine.degraded_reads import note_degraded_read, recording_degraded_reads +from search.engine.departures import measure_not_applied, warning_key +from search.engine.entry import dataset_resubmit +from search.engine.errors import EngineInputError +from search.engine.events import ObservationsEvent +from search.engine.quantity import NamedDenominator +from search.engine.region import ( + RegionResult, + composite_answer, + phase_answer, + phase_place_materialise, + pipeline, + result, + scope_decision, +) +from search.engine.region.composite_answer import ( + ComponentReads, + as_merged_composite, + population_table_answer, +) +from search.engine.region.grounding import role_scope_defaulted_warning +from search.models import ( + BaseIdentity, + DatasetResubmitInput, + DefinitePart, + IdentityBindingValue, + NoDataPart, + OptionAvailability, + ParsedEntity, + ParsedInput, + ParsedPart, + Resolution, + ResolveOptions, + ResolveRequest, + ResubmitIdentityEntry, + Spec, + Warning, +) +from search.scope import composite as scope_composite +from search.scope import provider +from search.scope.state import ScopeState +from search.scope.tables import ScopeFacts +from series_coverage.index import CoverageIndex +from series_coverage.schema import FacetMetadata, RoleNames +from tests.engine._fake_coverage import FakeCoverageService, patch_coverage +from tests.engine._fake_sdmx import FakeSdmxClient +from tests.engine._harness import ( + ObsFacetFixture, + build_materialise_double, + offline_events, + offline_resolve, +) +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_whole_total_bind_e2e import ( + _ALL_RECIPIENTS, + _DAC2A, + _DAC2A_ATTRIBUTED, + _DAC2A_MEMBER, + _DEU, + _ETH, + _FRA, + _VARIABLE, + _graph, +) +from tests.materialise.conftest import FakeGraphReader + +_BILATERAL = "provenance/ONEDAC2aBilateral" +_AGENCY = "EuropeanCommission" +_KEN = "country/KEN" +_OFFICIAL = "DACOfficialDonors" +_YEARS = (2019, 2020, 2021) +_NAMES = RoleNames(donor="financingProvider", recipient="financingRecipient") +# What each (stat var, donor, recipient) holds in every year. +_HELD = { + (_DAC2A_MEMBER, _FRA, _ETH): 100, + (_DAC2A_MEMBER, _FRA, _AGENCY): 200, + (_DAC2A_MEMBER, _FRA, _ALL_RECIPIENTS): 500, + (_DAC2A_ATTRIBUTED, _FRA, _ETH): 7, + (_DAC2A_MEMBER, _OFFICIAL, _ETH): 900, + (_DAC2A_MEMBER, _OFFICIAL, _AGENCY): 300, + # Germany's total ODA in the window went to the agency alone. + (_DAC2A_MEMBER, _DEU, _AGENCY): 400, +} +_SPURIOUS = {_AGENCY, _ALL_RECIPIENTS} + + +# The imputed line holds a recipient France's total ODA holds no row to, so an inclusive answer +# lists one recipient its bilateral table lacks. +_IMPUTED_ONLY = {**_HELD, (_DAC2A_ATTRIBUTED, _FRA, _KEN): 11} + +# The bilateral population's total (`RolePopulation.total`), which a composite naming no +# destination binds in its place. +_DEVELOPING = "DACDevelopingCountries" +# Rows at the population's total in place of the total spanning both sides, so a part naming +# no destination reads the bilateral composite at that total. +_AT_TOTAL = { + **{key: value for key, value in _HELD.items() if key[2] != _ALL_RECIPIENTS}, + (_DAC2A_MEMBER, _FRA, _DEVELOPING): 150, +} + + +@pytest.fixture(autouse=True) +def _coverage(monkeypatch: pytest.MonkeyPatch) -> CoverageIndex: + return _patched_coverage(monkeypatch, _HELD) + + +def _patched_coverage( + monkeypatch: pytest.MonkeyPatch, held: dict[tuple[str, str, str], int] +) -> CoverageIndex: + return patch_coverage( + monkeypatch, + [(sv, "dac2a", donor, recipient, list(_YEARS)) for (sv, donor, recipient) in held], + role_names=dict.fromkeys((_DAC2A_MEMBER, _DAC2A_ATTRIBUTED), _NAMES), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + + +def _graph_with_parties(): + """The whole-total fixture graph holding every party a population read posts, as the + resubmit's reference check reads them.""" + graph = _graph(variable=_VARIABLE, members=(_DAC2A_MEMBER, _DAC2A_ATTRIBUTED)) + population = destination_population(get_index(), _DAC2A) + parties = (*population.members, population.total, _AGENCY, _ALL_RECIPIENTS) + for dcid in parties: + graph._nodes.setdefault(dcid, {"label": dcid, "type": "Organization"}) + return graph + + +def _sdmx(held: dict[tuple[str, str, str], int] = _HELD) -> FakeSdmxClient: + observations: dict[str, dict] = {} + for (sv, donor, recipient), value in held.items(): + facet = Facet( + earliest_date=str(_YEARS[0]), + latest_date=str(_YEARS[-1]), + obs_count=len(_YEARS), + dates=tuple(str(y) for y in _YEARS), + values=(float(value),) * len(_YEARS), + provenance_id=_DAC2A, + unit="USD", + facet_id="dac2a", + ) + key = (("financingProvider", donor), ("financingRecipient", recipient)) + observations.setdefault(sv, {})[key] = [facet] + return FakeSdmxClient(observations=observations, availability={}) + + +def _materialise() -> tuple[FakeGraphReader, MaterialiseSdmxClient]: + """The materialise graph and SDMX doubles an open destination's preview reads per pair, + holding the same series as `_sdmx`.""" + graph, sdmx, _requests = build_materialise_double( + { + (sv, (("financingProvider", donor), ("financingRecipient", recipient))): [ + ObsFacetFixture( + facet_id="dac2a", + points=tuple((str(year), float(value)) for year in _YEARS), + provenance_id=_DAC2A, + unit="USD", + ) + ] + for (sv, donor, recipient), value in _HELD.items() + } + ) + return graph, sdmx + + +def _identity(provenance: str) -> BaseIdentity: + return BaseIdentity.model_validate( + { + "identity_version": "spec1", + "shape": { + "kind": "statvar_schema", + "population_type": "FinancialFlow", + "measured_property": "amount", + "stat_type": "measuredValue", + "measurement_qualifier": "Nominal", + }, + "slots": [ + {"property": "dac2aAidType", "binding": _ref("DAC2aAidType206")}, + {"property": "financialFlowType", "binding": _ref("Disbursement")}, + {"property": "nettingBasis", "binding": _ref("Gross")}, + ], + "entities": {"origin": [_FRA]}, + "open_roles": ["destination"], + "requested_window": {"kind": "range", "start_year": 2019, "end_year": 2021}, + "facets": {"provenance": _ref(provenance)}, + } + ) + + +def _ref(ref: str) -> dict[str, str]: + return {"state": "value", "ref": ref} + + +def _resubmit(identity: BaseIdentity, *, observations: bool = False) -> ResolveRequest: + return ResolveRequest( + input=DatasetResubmitInput( + entries=[ResubmitIdentityEntry(variable_text=_VARIABLE, identity=identity)] + ), + response_options=ResolveOptions(include_observations=observations), + ) + + +def _part(response: Any) -> DefinitePart: + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, DefinitePart), part + return part + + +def _resolve_resubmit( + identity: BaseIdentity, *, held: dict[tuple[str, str, str], int] = _HELD +) -> DefinitePart: + return _part( + offline_resolve( + _resubmit(identity), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(held), + ) + ) + + +@pytest.mark.parametrize( + ("provenance", "series"), + [ + pytest.param(_BILATERAL, 1, id="bilateral"), + pytest.param(DAC2A_INCLUSIVE, 2, id="inclusive"), + ], +) +def test_a_resubmit_counts_no_agency_or_spanning_series_for_an_open_destination( + provenance: str, series: int +) -> None: + spec = _resolve_resubmit(_identity(provenance)).spec + + headline = spec.resolved.headline + assert headline is not None + assert headline.role_counts["financingRecipient"] == 1 + assert headline.series == series + + +def _preview(identity: BaseIdentity) -> list[tuple[str, str, float]]: + """Each series of the streamed preview of `identity` as (stat var, destination, 2021 value).""" + materialise_graph, materialise_sdmx = _materialise() + events = offline_events( + _resubmit(identity, observations=True), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + materialise_graph=materialise_graph, + materialise_sdmx=materialise_sdmx, + stream_slot_options=True, + ) + (event,) = [e for e in events if isinstance(e, ObservationsEvent)] + read: list[tuple[str, str, float]] = [] + for series in event.observations.series: + value = next(p.value for p in series.points if p.date == "2021") + assert value is not None + destination = next(e.ref for e in series.entities if e.role == "destination") + read.append((series.stat_var.ref, destination, value)) + return read + + +@pytest.mark.parametrize( + ("provenance", "total"), + [ + pytest.param(_BILATERAL, 100.0, id="bilateral"), + pytest.param(DAC2A_INCLUSIVE, 100.0 + 7.0, id="inclusive"), + ], +) +def test_a_composite_preview_holds_the_country_row_and_inclusive_adds_106( + provenance: str, total: float +) -> None: + read = _preview(_identity(provenance)) + + assert {destination for _sv, destination, _value in read} == {_ETH} + assert sum(value for _sv, _destination, value in read) == total + + +def _fresh_request( + entities: list[tuple[str, str]], *, message: str, open_role: str | None +) -> ResolveRequest: + return ResolveRequest( + input=ParsedInput( + parts=[ + ParsedPart( + variable_text=_VARIABLE, + entities=[ParsedEntity(text=text, role_text=role) for role, text in entities], + open_roles=[open_role] if open_role else [], + time_text="2019-2021", + ) + ], + user_message=message, + ) + ) + + +_FRANCE = ("from", "France") +# H1 to H6 of the headline set that read DAC2a: France with every recipient open, with the +# inclusive scope stated, to all recipients, and every donor with no provider axis. +_SCENARIOS: dict[str, Callable[[], ResolveRequest]] = { + "H1": lambda: _fresh_request( + [_FRANCE], message="official development assistance from France", open_role="recipient" + ), + "H2": lambda: _fresh_request( + [_FRANCE], + message="official development assistance from France including imputed multilateral", + open_role="recipient", + ), + "H5": lambda: _fresh_request( + [_FRANCE, ("to", "all recipients")], + message="official development assistance from France to all recipients", + open_role=None, + ), + "H6": lambda: _fresh_request( + [("from", "All Donors, Total")], + message="official development assistance from all donors", + open_role="recipient", + ), + # A part naming no destination, read over `_AT_TOTAL`: France's ODA, and the top donors. + "D2": lambda: _fresh_request( + [_FRANCE], message="official development assistance from France", open_role=None + ), + "D2-ranking": lambda: _fresh_request( + [], message="top donors of official development assistance", open_role="donor" + ), +} + + +def _answer(spec: Spec, part: DefinitePart) -> dict[str, Any]: + resolved = spec.resolved + dump = lambda model: None if model is None else model.model_dump(mode="json") # noqa: E731 + return { + "scope": dump(resolved.scope), + "headline": dump(resolved.headline), + "coverage": dump(resolved.coverage), + "warnings": sorted((w.code, w.message) for w in part.warnings), + } + + +def _resolve_fresh(scenario: str, *, held: dict[tuple[str, str, str], int] = _HELD) -> DefinitePart: + return _part( + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(held), + ) + ) + + +@pytest.mark.parametrize( + ("scenario", "held"), + [ + *((scenario, _HELD) for scenario in ("H1", "H2", "H5", "H6")), + pytest.param("D2", _AT_TOTAL, id="D2"), + pytest.param("D2-ranking", _AT_TOTAL, id="D2-ranking"), + ], +) +def test_a_fresh_answer_equals_a_resubmit_of_its_definition( + scenario: str, held: dict[tuple[str, str, str], int], monkeypatch: pytest.MonkeyPatch +) -> None: + _patched_coverage(monkeypatch, held) + fresh = _resolve_fresh(scenario, held=held) + + again = _resolve_resubmit(fresh.spec.definition.identity, held=held) + + assert again.spec.spec_id == fresh.spec.spec_id + assert _answer(again.spec, again) == _answer(fresh.spec, fresh) + + +# What a fresh part holds from the request's words or its door, which a resubmit of its +# definition holds none of: the parsed part it restates, the places each text bound, the +# scope its words and binders stated, and its timings. Departures quote the request's words. +_WORD_FIELDS = frozenset({"restatement", "entity_refs", "timing_ms"}) + + +def _read_fields(part: DefinitePart) -> dict[str, Any]: + """`part`'s dump without the fields its request's words decide, its departures without + the words they quote.""" + dump = part.model_dump(mode="json", exclude=set(_WORD_FIELDS)) + dump["spec"]["resolved"].pop("scope_choices") + dump["departures"] = [ + {key: value for key, value in departure.items() if key not in ("asked", "text")} + for departure in dump["departures"] + ] + return dump + + +# What a fresh region holds from the request's words or its door, which a resubmit region of +# its definition holds otherwise: the part's words and timings, the phrases and places its +# words bound, the scope its binders stated, the relaunch facts and deadline of the door, +# and the closure reading the composite's preview over the door's clients. +_WORD_REGION_FIELDS = frozenset( + { + "variable_text", + "timing_by_step", + "admitted_qualifier_phrases", + "scope_choices", + "entity_words", + "entity_refs", + "promote_relaunch", + "reads_deadline", + "observations_relaunch", + } +) + + +def _region_fields(region: RegionResult) -> dict[str, str]: + """Each field of `region` the request's words do not decide, as its text with object + addresses left out, so two closures over the same code compare equal.""" + return { + field.name: re.sub(r" at 0x[0-9a-f]+", "", repr(getattr(region, field.name))) + for field in dataclasses.fields(region) + if field.name not in _WORD_REGION_FIELDS + } + + +@pytest.mark.parametrize("scenario", ["H1", "H2", "H6"]) +def test_a_fresh_composite_answer_reads_every_field_a_resubmit_reads( + scenario: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """Every field of a fresh composite answer the rows it read decide, in its region and in + its part, its Spec whole, its warnings and its departures, is a resubmit's of its + definition: the fresh answer is that read with the request's words added, so no read + field keeps the whole table's.""" + regions: list[RegionResult] = [] + scope_choices, as_composite = pipeline._with_scope_choices, dataset_resubmit._as_composite + + def fresh_region(result: RegionResult, state: Any) -> RegionResult: + regions.append(scope_choices(result, state)) + return regions[-1] + + def resubmit_region(region: RegionResult, identity: BaseIdentity, **kwargs: Any) -> Any: + regions.append(as_composite(region, identity, **kwargs)) + return regions[-1] + + monkeypatch.setattr(pipeline, "_with_scope_choices", fresh_region) + monkeypatch.setattr(dataset_resubmit, "_as_composite", resubmit_region) + fresh = _part( + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + again = _resolve_resubmit(fresh.spec.definition.identity) + + assert _read_fields(again) == _read_fields(fresh) + fresh_read, resubmitted = regions + assert _region_fields(resubmitted) == _region_fields(fresh_read) + + +@pytest.mark.parametrize( + ("scenario", "source"), + [ + pytest.param("H1", _BILATERAL, id="H1"), + pytest.param("H2", DAC2A_INCLUSIVE, id="H2"), + # All recipients spans both sides, so the default reads total ODA there. + pytest.param("H5", None, id="H5"), + # A ranking over every donor has no provider axis and still reads the composite. + pytest.param("H6", _BILATERAL, id="H6"), + ], +) +def test_a_fresh_dac2a_answer_reads_the_composite_its_destinations_allow( + scenario: str, source: str | None +) -> None: + part = _part( + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + pin = (part.spec.definition.identity.facets or {}).get("provenance") + assert (pin.ref if isinstance(pin, IdentityBindingValue) else None) == source + + +@pytest.mark.parametrize("scenario", ["H1", "H2"]) +def test_a_fresh_bilateral_reading_lists_no_agency_in_its_scope(scenario: str) -> None: + part = _part( + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + scope = part.spec.resolved.scope + assert scope is not None + assert not {member.ref for member in scope.members} & _SPURIOUS + + +def _reads(coverage: CoverageIndex) -> ComponentReads: + """The fixture's clients and coverage, as a part's component reads hold the request's.""" + return ComponentReads( + graph=_graph_with_parties(), + sdmx=_sdmx(), + coverage=FakeCoverageService(coverage).snapshot(), + materialise_graph=None, + materialise_sdmx=None, + rate_cache=None, + deadline=None, + catalogue=current_catalogue(), + ) + + +def _in_request_scope[T](call: Callable[[], T]) -> T: + """`call`'s result, called inside a request's concurrency scope + (`concurrency.scoped_concurrency`), as a direct call below `core.resolve_events` needs.""" + + async def scoped() -> T: + async with scoped_concurrency(): + return call() + + return asyncio.run(scoped()) + + +def _definite(spec: Spec) -> RegionResult: + return RegionResult( + variable_text=_VARIABLE, + status="definite", + specs=(spec,), + no_data_reason=None, + warnings=(), + timing_by_step={}, + ) + + +def test_a_merged_composite_reads_its_table_within_the_population( + _coverage: CoverageIndex, +) -> None: + """A same-shape merge of bilateral composite answers rebuilds its table Spec from its + members' slots and plans; the composite it is restated as reads that table within the + population, as a resubmit of the merged composite does, so a merged table Spec counting + the agency leaves no trace. Two donors never fold, since a donor is no slot + (`same_shape._resolve_diff_key`), so the merge is driven with a merged table Spec read + over the whole table.""" + whole = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec + scope = whole.resolved.scope + assert scope is not None and {member.ref for member in scope.members} >= _SPURIOUS + member = _resolve_resubmit(_identity(_BILATERAL)).spec + + merged = as_merged_composite(_definite(whole), [_definite(member)], reads=_reads(_coverage)) + + assert merged.spec.spec_id == member.spec_id + resolved, again = merged.spec.resolved, member.resolved + assert (resolved.scope, resolved.headline, resolved.coverage) == ( + again.scope, + again.headline, + again.coverage, + ) + + +def test_a_merged_composite_whose_population_holds_no_row_answers_no_data( + _coverage: CoverageIndex, +) -> None: + """A same-shape merge whose merged table read holds no row within the population + answers that read's no_data, as a reading the read drops does, never the whole table's + read restated as the composite: Germany's total ODA in the window went to the agency + alone.""" + table = _identity(_DAC2A).model_copy(update={"facets": None}) + whole = _resolve_resubmit(table.model_copy(update={"entities": {"origin": [_DEU]}})).spec + member = _resolve_resubmit(_identity(_BILATERAL)).spec + + merged = as_merged_composite(_definite(whole), [_definite(member)], reads=_reads(_coverage)) + + assert (merged.status, merged.specs, merged.no_data_reason) == ( + "no_data", + (), + "no_observations", + ) + + +def test_a_candidate_restated_as_the_bilateral_composite_lists_no_agency_in_its_scope( + _coverage: CoverageIndex, +) -> None: + """A candidate set's DAC2a total ODA reading, resolved over every party, is read again + within the bilateral population before it is restated as the composite, as a fresh + definite answer is, so its scope and headline count the country alone.""" + table = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec + scope = table.resolved.scope + assert scope is not None and {member.ref for member in scope.members} >= _SPURIOUS + reads = _reads(_coverage) + candidates = RegionResult( + variable_text=_VARIABLE, + status="candidates", + specs=(table,), + no_data_reason=None, + warnings=(), + timing_by_step={}, + ) + + (restated,) = _readings(candidates, reads=reads, selection=ScopeState()).specs + + pin = (restated.definition.identity.facets or {}).get("provenance") + assert isinstance(pin, IdentityBindingValue) and pin.ref == _BILATERAL + scope = restated.resolved.scope + assert scope is not None + assert {member.ref for member in scope.members} == {_ETH} + headline = restated.resolved.headline + assert headline is not None and headline.role_counts["financingRecipient"] == 1 + + +@pytest.mark.parametrize("shown", ["candidate", "alternate"]) +def test_a_reading_s_population_read_keys_its_warnings_to_the_reading( + _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch, shown: str +) -> None: + """A reading pinning DAC2a's provenance is read within the bilateral population under the + composite's table identity, whose spec_id is not the reading's. Every warning that read + raises, keyed to its own Spec or to none, follows the reading to the composite it is + restated as, so none names a Spec the response does not hold, whether the reading is the + set's own candidate or the alternate a refusal offers.""" + table = _resolve_resubmit(_identity(_DAC2A)).spec + population_read = composite_answer.population_table_answer + + def keyed(spec: Spec, identity: BaseIdentity, *, reads: ComponentReads) -> RegionResult: + read = population_read(spec, identity, reads=reads) + assert read is not None and read.status == "definite" + assert read.spec.spec_id != spec.spec_id + own = Warning( + code="date_coverage_partial", + severity="info", + message="the read's", + spec_id=read.spec.spec_id, + ) + return dataclasses.replace(read, warnings=(*read.warnings, own)) + + monkeypatch.setattr(composite_answer, "population_table_answer", keyed) + region = ( + dataclasses.replace(_definite(table), status="candidates") + if shown == "candidate" + else RegionResult( + variable_text=_VARIABLE, + status="no_data", + specs=(), + no_data_reason="no_observations", + warnings=(), + timing_by_step={}, + alternate_match=table, + ) + ) + + read = _readings(region, reads=_reads(_coverage), selection=ScopeState()) + + (restated,) = (*read.specs, *([read.alternate_match] if read.alternate_match else ())) + assert (restated.definition.identity.facets or {}).get("provenance") == ( + IdentityBindingValue(ref=_BILATERAL) + ) + assert [w.spec_id for w in read.warnings if w.code == "date_coverage_partial"] == [ + restated.spec_id + ] + assert {w.spec_id for w in read.warnings} <= {None, restated.spec_id} + + +def test_a_composite_whose_population_holds_no_row_answers_no_data_with_the_request_s_words( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh answer settled on the bilateral composite whose population holds no row in + the window answers what a resubmit of its definition answers, no_data, under the + request's own words, and never the whole table's read of the agency. The coverage the + decision reads offers the composite only where it serves a year, so the decision is made + to elect it here, as a coverage snapshot the read disagrees with would.""" + _offer_the_bilateral_composite(monkeypatch) + request = _fresh_request( + [("from", "Germany")], + message="official development assistance from Germany", + open_role="recipient", + ) + + response = offline_resolve( + request, graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=_sdmx() + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart), part + assert part.variable_text == _VARIABLE + assert part.no_data.reason == "no_observations" + + +@pytest.mark.parametrize( + ("error", "reason", "degraded"), + [ + pytest.param( + EngineInputError( + "no rows", + code="facet_not_available", + status=422, + pointer="/input/identity/entities/destination", + ), + "facet_not_available", + False, + id="refused", + ), + pytest.param( + dc_graph.GraphDeadlineExceededError("deadline"), "timed_out", True, id="deadline" + ), + pytest.param(dc_graph.GraphReadError("lost"), "no_observations", True, id="lost-read"), + ], +) +def test_a_population_read_that_raises_answers_no_data_with_the_request_s_words( + error: Exception, reason: str, degraded: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """A fresh answer whose population read raises answers no_data with the read's reason + under the request's own words, degraded where the read was lost: never the whole + table's read of every party, and never a refusal pointing at an identity the request + did not post.""" + + def raising(**_kwargs: Any) -> RegionResult: + raise error + + monkeypatch.setattr(composite_answer, "resolve_spec_resubmit", raising) + record = ResolveRecord() + + response = offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + record=record, + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart), part + assert (part.variable_text, part.no_data.reason) == (_VARIABLE, reason) + assert record.done is not None and record.done.degraded_leg is degraded + + +def test_a_request_warning_follows_the_population_read_s_spec_id() -> None: + """A warning the pass raised about its whole-table answer before the population read is + keyed to the read's Spec, the one the part answers, so it never names a spec_id the + response does not show.""" + whole = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec + read = _resolve_resubmit( + _identity(_DAC2A).model_copy(update={"facets": None, "entities": {"origin": [_DEU]}}) + ).spec + assert whole.spec_id != read.spec_id + words = Warning( + code="query_qualifier_dropped", severity="info", message="the words", spec_id=whole.spec_id + ) + fresh = dataclasses.replace(_definite(whole), warnings=(words,)) + + grafted = composite_answer.with_population_read( + fresh, _definite(read), request_warnings=(words,) + ) + + assert [(w.code, w.spec_id) for w in grafted.warnings] == [ + ("query_qualifier_dropped", read.spec_id) + ] + + +def test_a_composite_answer_keeps_the_warnings_raised_before_its_read( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A warning the pass raised before it read the elected shape, here one standing for the + request's words, stays on a fresh answer read again within its composite's population, + beside that read's own; the whole-table read's role scope does not.""" + words = Warning(code="query_qualifier_dropped", severity="info", message="the words") + run_materialise_task = phase_place_materialise._run_materialise_task + + async def raising_first(state: Any) -> Any: + state.warnings.append(words) + return await run_materialise_task(state) + + monkeypatch.setattr(phase_place_materialise, "_run_materialise_task", raising_first) + + part = _part( + offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + assert sorted((w.code, w.message) for w in part.warnings) == [ + ("query_qualifier_dropped", "the words"), + ( + "role_scope_defaulted", + "No recipient was named, so the answer covers the one recipient with data from France.", + ), + ] + + +def _offer_the_bilateral_composite(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the coverage a scope reads offer the bilateral composite whatever years it + serves, as a snapshot the read disagrees with would.""" + bilateral_option = provider._bilateral_option + + def available(*args: Any) -> Any: + option = bilateral_option(*args) + return option.model_copy(update={"availability": OptionAvailability(status="available")}) + + monkeypatch.setattr(provider, "_bilateral_option", available) + + +def test_a_candidate_whose_population_holds_no_row_is_dropped_not_restated( + _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch +) -> None: + """A candidate set's reading settled on the bilateral composite whose population holds no + row in the window is dropped, never restated over the whole table's read of the agency, + and a set left with none refuses with the reason that read gives.""" + _offer_the_bilateral_composite(monkeypatch) + table = _resolve_resubmit( + _identity(_DAC2A).model_copy(update={"facets": None, "entities": {"origin": [_DEU]}}) + ).spec + candidates = dataclasses.replace(_definite(table), status="candidates") + + read = _readings(candidates, reads=_reads(_coverage), selection=ScopeState()) + + assert (read.status, read.specs, read.no_data_reason) == ("no_data", (), "no_observations") + + +@pytest.mark.parametrize("with_france", [False, True], ids=["none-left", "one-left"]) +def test_a_dropped_reading_takes_its_warnings_and_departures_with_it( + _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch, with_france: bool +) -> None: + """A candidate whose population holds no row is dropped with the warnings keyed to it + and the departure facts they disclose, whether the set refuses or is left with one + reading.""" + _offer_the_bilateral_composite(monkeypatch) + table = _identity(_DAC2A).model_copy(update={"facets": None}) + germany = _resolve_resubmit(table.model_copy(update={"entities": {"origin": [_DEU]}})).spec + france = _resolve_resubmit(table).spec + departed = Warning( + code="query_qualifier_dropped", + severity="degraded", + message="Germany's words", + spec_id=germany.spec_id, + ) + candidates = dataclasses.replace( + _definite(germany), + status="candidates", + specs=(france, germany) if with_france else (germany,), + warnings=(departed,), + departure_facts={ + warning_key(departed): measure_not_applied(asked="words", property_dcid=None) + }, + ) + + read = _in_request_scope( + lambda: _readings(candidates, reads=_reads(_coverage), selection=ScopeState()) + ) + + assert read.status == ("definite" if with_france else "no_data") + assert departed.code not in {w.code for w in read.warnings} + assert warning_key(departed) not in read.departure_facts + + +def test_a_composite_answer_keeps_a_denominator_its_words_name_unverified( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`denominator_unverified` holds the base the question's words name against the shape + the answer reads, and no row decides it, so the warning raised once the read is + interpreted stands on a fresh answer read again within its composite's population. The + words are named at that check, so no earlier quantity gate reads them.""" + note = phase_place_materialise.note_denominator_unverified + + def naming_a_base(state: Any, shape: Any, *, spec_id: str | None) -> None: + state.named_denominator = NamedDenominator(ref=None, base_words="people") + note(state, shape, spec_id=spec_id) + + monkeypatch.setattr(phase_answer, "note_denominator_unverified", naming_a_base) + monkeypatch.setattr(phase_place_materialise, "check_denominator", lambda *_: "unverified") + + part = _part( + offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + assert (part.spec.definition.identity.facets or {}).get("provenance") == ( + IdentityBindingValue(ref=_BILATERAL) + ) + assert [w.spec_id for w in part.warnings if w.code == "denominator_unverified"] == [ + part.spec.spec_id + ] + + +def test_an_inclusive_answer_counts_the_recipients_of_both_lines( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh inclusive answer's scope, role axis fact and defaulted-scope warning count the + recipients of the bilateral and imputed lines together, as its headline and the mint's + frame do: the table identity it was resolved for reads total ODA alone, which holds no row + to Kenya.""" + _patched_coverage(monkeypatch, _IMPUTED_ONLY) + + part = _part( + offline_resolve( + _SCENARIOS["H2"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(_IMPUTED_ONLY), + ) + ) + + resolved = part.spec.resolved + assert resolved.headline is not None + assert resolved.headline.role_counts["financingRecipient"] == 2 + assert resolved.scope is not None + assert [member.ref for member in resolved.scope.members] == [_ETH, _KEN] + assert [(f.role, f.member_count) for f in resolved.role_axis_facts or ()] == [ + ("destination", 2) + ] + # The warning carries its count in its words alone, as "both" for two. + (defaulted,) = [w.message for w in part.warnings if w.code == "role_scope_defaulted"] + assert "both recipients" in defaulted + + +def test_a_composite_recounts_the_defaulted_scope_warning_on_its_axis_whatever_it_says( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A table answer's `role_scope_defaulted` warning names its axis in `params`, so the + composite it is restated as states its own count on that axis whatever words the table's + warning holds, here a label the request spelled its own way; a warning on another axis + stands.""" + _patched_coverage(monkeypatch, _IMPUTED_ONLY) + composite = _resolve_fresh("H2", held=_IMPUTED_ONLY).spec + facts = [ + fact.model_copy(update={"member_count": 1}) + for fact in composite.resolved.role_axis_facts or () + ] + table = composite.model_copy( + update={ + "spec_id": "table", + "resolved": composite.resolved.model_copy(update={"role_axis_facts": facts}), + } + ) + recipients = role_scope_defaulted_warning( + "destination", + "No recipient was named, so the answer covers the one recipient with data from FR.", + ).model_copy(update={"spec_id": "table"}) + donors = role_scope_defaulted_warning("origin", "No provider was named.").model_copy( + update={"spec_id": "table"} + ) + + recounted = result.recounted_scope((recipients, donors), table, composite, own=False) + + assert [w.message for w in recounted] == [ + "No recipient was named, so the answer covers both recipients with data from France.", + donors.message, + ] + + +def test_an_inclusive_answer_lists_the_parties_its_role_axis_fact_counts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Where a component's series reach a total on the open axis, here `DACAllRecipients`, + an inclusive answer's scope lists the recipients its role axis fact counts, so the total + the count leaves out is no member either.""" + _patched_coverage(monkeypatch, _IMPUTED_ONLY) + served_parties = scope_composite._served_parties + + def reaching_a_total(served: Any, coverage: Any) -> Any: + held = served_parties(served, coverage) + return {**held, "recipient": held["recipient"] | {_ALL_RECIPIENTS}} + + monkeypatch.setattr(scope_composite, "_served_parties", reaching_a_total) + + part = _resolve_fresh("H2", held=_IMPUTED_ONLY) + + resolved = part.spec.resolved + assert resolved.scope is not None + assert [member.ref for member in resolved.scope.members] == [_ETH, _KEN] + assert [(f.role, f.member_count) for f in resolved.role_axis_facts or ()] == [ + ("destination", 2) + ] + + +@pytest.mark.parametrize( + ("origin", "destination"), + [("financingProvider", "financingRecipient"), ("creditor", "debtor")], +) +def test_a_component_counts_the_places_every_role_key_binds( + _coverage: CoverageIndex, origin: str, destination: str +) -> None: + """A component's years count the series under the places its entities bind whatever + key names the role (`role_key_axis`), as its destinations are bound: Germany's ODA to + Ethiopia holds no series, however the identity spells its roles.""" + identity = _identity(_DAC2A).model_dump(mode="json", exclude_none=True) + del identity["facets"] + del identity["open_roles"] + facts = ScopeFacts(index=get_index(), coverage=FakeCoverageService(_coverage).snapshot()) + + def years(entities: dict[str, list[str]]) -> dict[int, int]: + return dict( + scope_composite.component_years({**identity, "entities": entities}, facts=facts).counts + ) + + named = years({"origin": [_DEU], "destination": [_ETH]}) + assert named == years({origin: [_DEU], destination: [_ETH]}) + assert not any(named.values()) + + +def test_an_inclusive_answer_whose_component_parties_cannot_be_listed_says_so( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Where the snapshot cannot list the parties of a component serving a year, an + inclusive answer keeps the open axes of its table read, which count the bilateral line's + recipients alone, and logs a WARNING naming the composite and the component.""" + _patched_coverage(monkeypatch, _IMPUTED_ONLY) + open_axes = scope_composite._open_axes + + def unsettled(stated: Any, served: Any, declaration: Any, *, facts: Any) -> Any: + *listed, last = served + return open_axes( + stated, [*listed, dataclasses.replace(last, members=None)], declaration, facts=facts + ) + + monkeypatch.setattr(scope_composite, "_open_axes", unsettled) + + with caplog.at_level(logging.WARNING, logger=scope_composite.__name__): + part = _resolve_fresh("H2", held=_IMPUTED_ONLY) + + scope = part.spec.resolved.scope + assert scope is not None and [member.ref for member in scope.members] == [_ETH] + (record,) = [r for r in caplog.records if r.name == scope_composite.__name__] + assert (record.levelno, record.args[0]) == (logging.WARNING, DAC2A_INCLUSIVE) + + +def test_a_candidate_set_left_with_one_reading_answers_as_its_resubmit( + _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch +) -> None: + """A candidate set whose other reading holds no row within its population is left with + one reading, so it answers that reading definite, as a resubmit of its composite does, + and not as a set of one, stating the scope a definite answer states: compiled to the + composite, with its components.""" + _offer_the_bilateral_composite(monkeypatch) + table = _identity(_DAC2A).model_copy(update={"facets": None}) + france = _resolve_resubmit(table).spec + germany = _resolve_resubmit(table.model_copy(update={"entities": {"origin": [_DEU]}})).spec + candidates = dataclasses.replace( + _definite(france), status="candidates", specs=(france, germany) + ) + + read = _in_request_scope( + lambda: _readings(candidates, reads=_reads(_coverage), selection=ScopeState()) + ) + + again = _resolve_resubmit(_identity(_BILATERAL)).spec + assert (read.status, [spec.spec_id for spec in read.specs]) == ("definite", [again.spec_id]) + assert (read.spec.resolved.scope, read.spec.resolved.headline) == ( + again.resolved.scope, + again.resolved.headline, + ) + assert read.scope_choices is not None + assert read.scope_choices.compiled.spec_id == again.spec_id + assert read.scope_choices.compiled.components + + +def test_population_reads_take_the_request_s_limit_and_report_their_lost_reads( + _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch, search_settings: Any +) -> None: + """Two or more population reads run in worker threads, and two parts' reads together run + at most the request's concurrency limit at once. A read one of them loses reaches the + request's record, so no cache stores the answer.""" + search_settings(max_variables=2, max_variable_concurrency=2) + lock, running, peak = threading.Lock(), [0], [0] + + def reading(spec: Any, identity: Any, *, reads: Any) -> None: + with lock: + running[0] += 1 + peak[0] = max(peak[0], running[0]) + time.sleep(0.05) + note_degraded_read(f"read of {spec}") + with lock: + running[0] -= 1 + + monkeypatch.setattr(composite_answer, "population_table_answer", reading) + parts: list[Any] = [ + {name: (name, None) for name in (f"{part}-first", f"{part}-second")} for part in ("a", "b") + ] + reads = _reads(_coverage) + + async def read() -> tuple[list[dict[str, Any]], list[str]]: + async with scoped_concurrency(): + with recording_degraded_reads() as record: + answered = await asyncio.gather( + *( + asyncio.to_thread(composite_answer._population_reads, part, reads=reads) + for part in parts + ) + ) + return answered, record.reads + + answered, lost = asyncio.run(read()) + + assert answered == [dict.fromkeys(part) for part in parts] + assert sorted(lost) == sorted(f"read of {name}" for part in parts for name in part) + assert peak[0] == 2 + + +def test_a_composite_naming_no_destination_is_read_again_at_its_population_total( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A composite whose identity names no destination binds the population's total in its + place (`RolePopulation.total`), which a whole-table answer of the same words never read, + so a fresh answer settled on it is read again at that total, as a resubmit of it reads + it.""" + assert destination_population(get_index(), _DAC2A).total == _DEVELOPING + reads = dataclasses.replace( + _reads(_patched_coverage(monkeypatch, _AT_TOTAL)), sdmx=_sdmx(_AT_TOTAL) + ) + composite = _identity(_BILATERAL).model_copy(update={"open_roles": []}) + table = _resolve_resubmit( + _identity(_DAC2A).model_copy(update={"facets": None}), held=_AT_TOTAL + ).spec + + read = population_table_answer(table, composite, reads=reads) + + assert read is not None and read.status == "definite" + assert read.spec.definition.identity.entities["destination"] == [_DEVELOPING] + resolved = read.spec.resolved + assert resolved.headline is not None + assert (resolved.headline.series, resolved.headline.role_counts["financingRecipient"]) == ( + 1, + 1, + ) + + +@pytest.mark.parametrize( + ("scenario", "axes"), + [ + pytest.param( + "D2", + { + "measure_basis": "measure_basis:gross", + "provider": "provider:bilateral", + }, + id="D2", + ), + pytest.param("D2-ranking", {}, id="D2-ranking"), + ], +) +def test_a_fresh_answer_naming_no_destination_states_the_scope_of_its_own_identity( + scenario: str, axes: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """A fresh answer naming no destination reads the bilateral composite at the population's + total, and that read never changes the part's identity or the scope decided on it: the + answer and its scope's base name no destination, and France's part keeps its provider + switch to total ODA, contributions included.""" + _patched_coverage(monkeypatch, _AT_TOTAL) + + part = _resolve_fresh(scenario, held=_AT_TOTAL) + + identity = part.spec.definition.identity + assert "destination" not in identity.entities + assert (identity.facets or {}).get("provenance") == IdentityBindingValue(ref=_BILATERAL) + choices = part.spec.resolved.scope_choices + assert choices is not None + assert "destination" not in choices.base.identity.entities + assert {choice.axis: choice.active for choice in choices.choices} == axes + offered = [ + option.option_id + for choice in choices.choices + if choice.axis == "provider" + for option in choice.options + ] + assert ("provider:contributions" in offered) == (scenario == "D2") + + +def test_a_population_read_the_composite_refuses_answers_no_data( + _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch +) -> None: + """A composite that refuses the read of its population answers no_data, so the caller + never states the whole table's read of every party under the composite's spec_id.""" + + def refusing(*_args: Any, **_kwargs: Any) -> Any: + raise EngineInputError("refused", code="requested_option_unavailable", status=422) + + monkeypatch.setattr(composite_answer, "composite_table_identity", refusing) + table = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec + + read = population_table_answer(table, _identity(_BILATERAL), reads=_reads(_coverage)) + + assert read is not None + assert (read.status, read.specs, read.no_data_reason) == ("no_data", (), "facet_not_available") + + +def test_a_fresh_bilateral_answer_s_component_carries_the_bound_population() -> None: + """The compiled scope's component identity is the one the mint reads: its destination + bound to the population's members, the list a client posting the component reads.""" + part = _part( + offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + choices = part.spec.resolved.scope_choices + assert choices is not None and choices.compiled.components is not None + (component,) = choices.compiled.components + population = destination_population(get_index(), _DAC2A) + assert component.identity.entities["destination"] == list(population.members) + assert not component.identity.open_roles + + +# A DAC2a destination the code index holds no row for, so no side declaration places it and +# the bilateral population is unavailable (`code_index.composites.population_check`). +_UNCLASSIFIED = "DACRecipientOnNoSide" + + +def _withheld(part: DefinitePart) -> list[tuple[str | None, dict[str, Any] | None]]: + """The spec_id and params of each `bilateral_reading_unavailable` warning `part` holds.""" + return [ + (w.spec_id, w.params) for w in part.warnings if w.code == "bilateral_reading_unavailable" + ] + + +def _years_short(monkeypatch: pytest.MonkeyPatch) -> None: + """Coverage where the bilateral destinations hold every donor's rows in 2019 and 2020 + alone, while the agency's rows run to 2021, so the composite covers fewer years than the + table.""" + patch_coverage( + monkeypatch, + [ + (sv, "dac2a", donor, recipient, [2019, 2020] if recipient == _ETH else list(_YEARS)) + for (sv, donor, recipient) in _HELD + ], + role_names=dict.fromkeys((_DAC2A_MEMBER, _DAC2A_ATTRIBUTED), _NAMES), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + + +def _slot_refused(monkeypatch: pytest.MonkeyPatch) -> None: + """The bilateral composite refuses what the part binds at its aid type slot.""" + served_components = provider.served_components + + def refusing(document: dict[str, Any], *, facts: Any) -> Any: + if document.get("facets", {}).get("provenance", {}).get("ref") == _BILATERAL: + raise CompositeNotServable("refused", ref=_BILATERAL, pointer="/slots/0") + return served_components(document, facts=facts) + + monkeypatch.setattr(provider, "served_components", refusing) + + +def _decision_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """The part's scope decision raises once the bilateral default has read it.""" + + def refusing(*_args: Any, **_kwargs: Any) -> Any: + raise CompositeNotServable("refused", ref=DAC2A_INCLUSIVE, pointer="/slots/0") + + monkeypatch.setattr(scope_decision, "settle_provider", refusing) + + +@pytest.mark.parametrize( + ("drift", "why"), + [ + pytest.param( + lambda monkeypatch: _patched_coverage( + monkeypatch, {**_HELD, (_DAC2A_MEMBER, _OFFICIAL, _UNCLASSIFIED): 50} + ), + "destination_unclassified", + id="population-unavailable", + ), + pytest.param(_years_short, "option_not_covered", id="years-not-covered"), + pytest.param(_slot_refused, "dac2aAidType", id="slot-refused"), + pytest.param(_decision_raises, "scope_undecided", id="decision-raises"), + ], +) +def test_a_ranking_over_every_donor_says_when_it_cannot_read_bilateral_destinations( + monkeypatch: pytest.MonkeyPatch, drift: Callable[[pytest.MonkeyPatch], Any], why: str +) -> None: + """A fresh ranking over every donor has no provider axis to state why it reads total ODA, + so where the bilateral default cannot apply, for a population no side declaration fully + places, a composite covering fewer years than the answer, a composite refusing a slot + the part binds, or a scope decision that raises, it warns + `bilateral_reading_unavailable` on its own spec_id with the reason.""" + drift(monkeypatch) + + part = _part( + offline_resolve( + _SCENARIOS["H6"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + assert (part.spec.definition.identity.facets or {}).get("provenance") is None + assert _withheld(part) == [(part.spec.spec_id, {"why": why})] + + +def test_a_part_with_a_provider_axis_states_an_unavailable_population_on_its_scope( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A donor's part has a provider axis, whose bilateral option already says why it is not + offered, so it carries no `bilateral_reading_unavailable` warning; a ranking over every + donor that reads the composite carries none either.""" + every_donor = _part( + offline_resolve( + _SCENARIOS["H6"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + _patched_coverage(monkeypatch, {**_HELD, (_DAC2A_MEMBER, _FRA, _UNCLASSIFIED): 50}) + + donor = _part( + offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + choices = donor.spec.resolved.scope_choices + assert choices is not None + (provider_choice,) = (c for c in choices.choices if c.axis == "provider") + bilateral = next(o for o in provider_choice.options if o.option_id == "provider:bilateral") + assert bilateral.availability.detail == {"why": "destination_unclassified"} + assert (_withheld(donor), _withheld(every_donor)) == ([], []) + + +def test_a_candidate_over_every_donor_says_when_it_cannot_read_bilateral_destinations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A candidate set's reading over every donor that keeps total ODA because the bilateral + population is unavailable warns as a definite answer does, keyed to that reading.""" + coverage = _patched_coverage( + monkeypatch, {**_HELD, (_DAC2A_MEMBER, _OFFICIAL, _UNCLASSIFIED): 50} + ) + table = _resolve_resubmit( + _identity(_DAC2A).model_copy(update={"facets": None, "entities": {"origin": [_OFFICIAL]}}) + ).spec + candidates = dataclasses.replace(_definite(table), status="candidates") + + read = _readings(candidates, reads=_reads(coverage), selection=ScopeState()) + + assert [spec.spec_id for spec in read.specs] == [table.spec_id] + assert [ + (w.spec_id, w.params) for w in read.warnings if w.code == "bilateral_reading_unavailable" + ] == [(table.spec_id, {"why": "destination_unclassified"})] + + +def _readings( + region: RegionResult, *, reads: ComponentReads, selection: ScopeState +) -> RegionResult: + """`region` with its readings restated (`with_bilateral_readings`) under scope facts of + no operation, as a part with no answer phase decision reads them.""" + return composite_answer.with_bilateral_readings( + region, + reads=reads, + selection=selection, + facts=ScopeFacts(index=get_index(), coverage=reads.coverage), + ) diff --git a/tests/engine/test_dac2a_population_reads.py b/tests/engine/test_dac2a_population_reads.py new file mode 100644 index 000000000..4482f3325 --- /dev/null +++ b/tests/engine/test_dac2a_population_reads.py @@ -0,0 +1,415 @@ +"""Search reads a role a DAC2a composite binds to its bilateral population as the open role +the caller posted, within the population's members: at the resubmit door's role-scope +discovery and whole totals, in each component's preview, in the part memo key, and at the +options door. A posted identity binding the composite to an agency or the total spanning +both sides refuses, naming the dataset whose total ODA reads it. Search's SDMX reads key a +population filter (`dc_sdmx.Among`) by its members. + +The fixture is `test_dac2a_destination_populations`': France's 206 to Ethiopia, to the +European Commission and to `DACAllRecipients`, and its imputed 106 to Ethiopia. The United +States holds a 206 row to the European Commission alone. +""" + +from __future__ import annotations + +import time + +import pytest + +from code_index.composites import ( + PopulationUnavailable, + UnclassifiedDestinations, + destination_population, +) +from code_index.declarations import DAC2A_INCLUSIVE +from code_index.shared import get_index +from dc_sdmx import Among +from search.engine import identity_edit +from search.engine.clients.materialise_sdmx_cache import _entities_key +from search.engine.entry import dataset_entries +from search.engine.errors import EngineInputError +from search.engine.options.options_response_cache import options_response_cache_key +from search.engine.options.resolve import resolve_options_target +from search.engine.options.surface import compute_options +from search.engine.region import composite_answer +from search.engine.region.observations.bulk_pair_reads import _bound_key +from search.engine.response_cache_keys import part_memo_key +from search.engine.retrieval import bound_values +from search.engine.retrieval.bound_values import identity_query +from search.heading import pins +from search.heading.pins import identity_pinned_properties +from search.models import ( + BaseIdentity, + CandidatesPart, + DatasetResubmitInput, + OptionsFor, + OptionsRequest, + ResolveRequest, + ResubmitCandidate, + ResubmitCandidatesEntry, + ResubmitIdentityPayload, +) +from series_coverage.index import CoverageIndex +from series_coverage.schema import FacetMetadata +from tests.engine._fake_coverage import FakeCoverageService, patch_coverage +from tests.engine._harness import offline_resolve +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_dac2a_destination_populations import ( + _AGENCY, + _BILATERAL, + _HELD, + _NAMES, + _SPURIOUS, + _YEARS, + _graph_with_parties, + _identity, + _resolve_resubmit, + _resubmit, + _sdmx, +) +from tests.engine.test_whole_total_bind_e2e import ( + _ALL_RECIPIENTS, + _DAC2A, + _DAC2A_ATTRIBUTED, + _DAC2A_MEMBER, + _ETH, + _VARIABLE, +) +from tests.materialise.conftest import FakeGraphReader +from tests.origin_catalogue.fake_catalogue import FakeCatalogue + +_USA = "country/USA" +_DEVELOPING = "DACDevelopingCountries" +_COMPOSITES = [ + pytest.param(_BILATERAL, id="bilateral"), + pytest.param(DAC2A_INCLUSIVE, id="inclusive"), +] + + +def _rows(held: dict[tuple[str, str, str], int]) -> list: + return [(sv, "dac2a", donor, recipient, list(_YEARS)) for (sv, donor, recipient) in held] + + +def _patch(monkeypatch: pytest.MonkeyPatch, held: dict[tuple[str, str, str], int]): + return patch_coverage( + monkeypatch, + _rows(held), + role_names=dict.fromkeys((_DAC2A_MEMBER, _DAC2A_ATTRIBUTED), _NAMES), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + + +@pytest.fixture(autouse=True) +def _coverage(monkeypatch: pytest.MonkeyPatch): + return _patch(monkeypatch, {**_HELD, (_DAC2A_MEMBER, _USA, _AGENCY): 50}) + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_a_resubmit_discovers_its_open_destination_within_the_population(provenance: str) -> None: + part = _resolve_resubmit(_identity(provenance)) + + scope = part.spec.resolved.scope + assert scope is not None + assert {member.ref for member in scope.members} == {_ETH} + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_a_resubmit_offers_the_population_total_in_place_of_the_spanning_one( + provenance: str, monkeypatch: pytest.MonkeyPatch +) -> None: + _patch(monkeypatch, {**_HELD, (_DAC2A_MEMBER, "country/FRA", _DEVELOPING): 600}) + + part = _resolve_resubmit(_identity(provenance)) + + totals = {w.params["total"] for w in part.warnings if w.code == "whole_total_available"} + assert totals == {_DEVELOPING} + + +@pytest.mark.parametrize("destination", [_AGENCY, _ALL_RECIPIENTS]) +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_a_composite_posted_with_a_destination_off_its_side_refuses( + provenance: str, destination: str +) -> None: + identity = BaseIdentity.model_validate( + _identity(provenance).model_dump(exclude_none=True) + | {"entities": {"origin": ["country/FRA"], "destination": [destination]}, "open_roles": []} + ) + + with pytest.raises(EngineInputError) as refused: + offline_resolve( + _resubmit(identity), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + error = refused.value + assert (error.code, error.status) == ("requested_option_unavailable", 422) + assert error.pointer == "/input/entries/0/identity/entities/destination" + assert error.cause is not None + assert (error.cause.code, error.cause.entities) == ("multilateral_destination", [destination]) + assert [alternative.ref for alternative in error.alternatives or ()] == [_DAC2A] + + +def test_a_population_read_and_a_named_list_of_its_members_key_apart() -> None: + population = destination_population(get_index(), _DAC2A) + identity = BaseIdentity.model_validate( + _identity(_DAC2A).model_dump(exclude_none=True) + | {"entities": {"origin": ["country/FRA"], "destination": list(population.members)}} + | {"open_roles": []} + ) + + def key(payload: ResubmitIdentityPayload) -> str: + return part_memo_key( + payload=payload, + engine_flags={}, + build_commit="b", + graph_generation="g", + coverage_stamp="c", + conversion_catalogue=None, + origin_catalogue="o", + un_system_enabled=False, + today="2026-11-15", + ) + + named = ResubmitIdentityPayload(identity=identity) + within = ResubmitIdentityPayload(identity=identity, populations={"destination": population}) + + assert key(within) != key(named) + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_each_component_preview_reads_its_destination_within_the_population( + provenance: str, monkeypatch: pytest.MonkeyPatch +) -> None: + posted: list[ResubmitIdentityPayload] = [] + resolve = composite_answer.resolve_spec_resubmit + + def spy(*, inp: ResubmitIdentityPayload, **kwargs): + posted.append(inp) + return resolve(inp=inp, **kwargs) + + monkeypatch.setattr(composite_answer, "resolve_spec_resubmit", spy) + + offline_resolve( + _resubmit(_identity(provenance), observations=True), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + assert posted + assert all(set(payload.populations) == {"destination"} for payload in posted) + + +def _target(role: str, provenance: str | None, coverage: CoverageIndex): + """The options target of `role` for the fixture identity under `provenance`, or under + none, the table identity a DAC2a composite's option was set on.""" + identity = _identity(provenance or _DAC2A) + if provenance is None: + identity = identity.model_copy(update={"facets": None}) + request = OptionsRequest(identity=identity, **{"for": OptionsFor(role=role)}) + graph = _graph_with_parties() + graph._nodes.setdefault(_USA, {"label": "United States", "type": "Country"}) + return ( + request, + graph, + resolve_options_target( + request, + graph=graph, + materialise_graph=FakeGraphReader(), + coverage=FakeCoverageService(coverage).snapshot(), + ), + ) + + +def _options(role: str, provenance: str, coverage) -> set[str]: + request, graph, target = _target(role, provenance, coverage) + computed = compute_options( + request, + target, + graph=graph, + deadline=time.monotonic() + 30.0, + coverage=FakeCoverageService(coverage).snapshot(), + ) + return {item.value.ref.ref for item in computed.options.items if item.value.ref is not None} + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_the_destination_options_offer_no_agency_or_spanning_total(provenance: str, _coverage): + offered = _options("destination", provenance, _coverage) + + assert _ETH in offered + assert not offered & _SPURIOUS + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_the_origin_options_list_the_donors_holding_rows_at_the_population( + provenance: str, _coverage +): + offered = _options("origin", provenance, _coverage) + + assert offered == {"country/FRA", "DACOfficialDonors"} + + +@pytest.mark.parametrize("role", ["destination", "origin"]) +def test_a_composite_s_options_key_apart_from_its_table_identity_s(role: str, _coverage) -> None: + def key(provenance: str | None) -> str: + request, _graph, target = _target(role, provenance, _coverage) + return options_response_cache_key( + request, + target, + build_commit="b", + coverage=FakeCoverageService(_coverage).snapshot(), + catalogue=FakeCatalogue(), + ) + + assert key(_BILATERAL) != key(None) + + +def test_a_population_filter_keys_search_s_sdmx_reads_by_its_members() -> None: + members = ("country/ETH", "country/KEN") + among = Among(members=members, side="bilateral") + + assert _entities_key(among) == members + assert _bound_key({"financingRecipient": among}) == (("financingRecipient", members),) + + +_UNPLACED = "DACRecipientOnNoSide" +"""A DAC2a destination the code index holds no row for, so no side declaration places it.""" + + +def _refused_as_unavailable( + error: EngineInputError, *, pointer: str, code: str = "requested_option_unavailable" +) -> None: + assert (error.code, error.status, error.pointer) == (code, 422, pointer) + assert error.cause is not None + assert (error.cause.code, error.cause.entities) == ("provider_not_offered", [_UNPLACED]) + assert [alternative.ref for alternative in error.alternatives or ()] == [_DAC2A] + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_a_composite_whose_coverage_holds_a_destination_on_no_side_refuses_at_resubmit( + provenance: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """The coverage snapshot holds a DAC2a recipient no side declaration places, so the + population would leave it out without a word (`population_check`): a posted composite + refuses as its scope option reads, not offered, with the destination named and total + ODA as the alternative, where Search would otherwise answer a scope the mint refuses.""" + _patch(monkeypatch, {**_HELD, (_DAC2A_MEMBER, "country/FRA", _UNPLACED): 5}) + + with pytest.raises(EngineInputError) as refused: + offline_resolve( + _resubmit(_identity(provenance)), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + _refused_as_unavailable(refused.value, pointer="/input/entries/0/identity/facets/provenance") + + +def _unavailable(provenance: str): + """A `table_identity` whose code index publishes a destination no side places.""" + + def read(_index, _identity): + raise PopulationUnavailable( + UnclassifiedDestinations( + dataset=_DAC2A, destinations=(_UNPLACED,), found_in="code_index" + ), + ref=provenance, + ) + + return read + + +@pytest.mark.parametrize("provenance", _COMPOSITES) +def test_a_composite_whose_code_index_places_no_side_refuses_at_resubmit( + provenance: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A code index publishing a destination no side declaration places makes the + population unavailable where the composite derives (`PopulationUnavailable`): the + resubmit door refuses it as a scope it does not offer, not as a server fault.""" + + monkeypatch.setattr(identity_edit, "table_identity", _unavailable(provenance)) + + with pytest.raises(EngineInputError) as refused: + offline_resolve( + _resubmit(_identity(provenance)), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + _refused_as_unavailable(refused.value, pointer="/input/entries/0/identity/facets/provenance") + + +def test_a_composite_whose_coverage_holds_a_destination_on_no_side_refuses_its_options( + monkeypatch: pytest.MonkeyPatch, +) -> None: + coverage = _patch(monkeypatch, {**_HELD, (_DAC2A_MEMBER, "country/FRA", _UNPLACED): 5}) + + with pytest.raises(EngineInputError) as refused: + _target("destination", _BILATERAL, coverage) + + # The options door registers no scope refusal, so it refuses the facet it cannot serve. + _refused_as_unavailable( + refused.value, pointer="/identity/facets/provenance", code="facet_not_available" + ) + + +@pytest.mark.parametrize("reader", ["pins", "family"]) +def test_a_reader_of_a_composite_s_table_refuses_an_unavailable_population( + reader: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A reader of the table a composite reads, its pins or its family query, refuses an + unavailable population as Search's doors refuse it, not as a server fault.""" + monkeypatch.setattr(pins, "table_identity", _unavailable(_BILATERAL)) + monkeypatch.setattr(bound_values, "table_identity", _unavailable(_BILATERAL)) + read = identity_pinned_properties if reader == "pins" else identity_query + + with pytest.raises(EngineInputError) as refused: + read(_identity(_BILATERAL)) + + _refused_as_unavailable(refused.value, pointer="/input/identity/facets/provenance") + + +def test_the_candidate_door_reads_each_composite_s_table_identity_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A posted composite candidate that the window election and the served bilateral line + leave unchanged derives its table identity once, at the check that refuses a composite + its components cannot serve.""" + read = dataset_entries.composite_table_identity + derived: list[str | None] = [] + + def counted(identity: BaseIdentity, **kwargs: object): + derived.append(identity.facets["provenance"].ref if identity.facets else None) + return read(identity, **kwargs) + + monkeypatch.setattr(dataset_entries, "composite_table_identity", counted) + + response = offline_resolve( + ResolveRequest( + input=DatasetResubmitInput( + entries=[ + ResubmitCandidatesEntry( + variable_text=_VARIABLE, + ordering="intent_rank", + truncated=False, + candidates=[ + ResubmitCandidate(identity=_identity(provenance)) + for provenance in (_BILATERAL, DAC2A_INCLUSIVE) + ], + ) + ] + ) + ), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + (part,) = response.parts + assert isinstance(part, CandidatesPart), part + assert sorted(derived) == sorted([_BILATERAL, DAC2A_INCLUSIVE]) diff --git a/tests/engine/test_grounding_identity_entities.py b/tests/engine/test_grounding_identity_entities.py index aed029744..970e379ce 100644 --- a/tests/engine/test_grounding_identity_entities.py +++ b/tests/engine/test_grounding_identity_entities.py @@ -113,7 +113,7 @@ def _dcids(count: int) -> list[str]: def _unqualified(discovered: DiscoveredScope) -> str: """The message for a query with no bound opposite entity.""" return role_scope_defaulted_message( - discovered, + discovered.axis, member_count=len(discovered.member_dcids), bound_dcids=[], label_of=_RecordingLabels(), @@ -161,7 +161,7 @@ def test_role_scope_defaulted_message_uses_the_origin_axis_noun_and_count(): def test_role_scope_defaulted_message_names_every_bound_origin(labels: list[str], named: str): """A discovered destination scope names bound origins, with counts above three.""" message = role_scope_defaulted_message( - _discovered_scope("destination", 2), + "destination", member_count=2, bound_dcids=_dcids(len(labels)), label_of=_RecordingLabels(labels), @@ -182,7 +182,7 @@ def test_role_scope_defaulted_message_names_every_bound_origin(labels: list[str] def test_role_scope_defaulted_message_names_every_bound_destination(labels: list[str], named: str): """A discovered origin scope names bound destinations.""" message = role_scope_defaulted_message( - _discovered_scope("origin", 2), + "origin", member_count=2, bound_dcids=_dcids(len(labels)), label_of=_RecordingLabels(labels), @@ -193,7 +193,7 @@ def test_role_scope_defaulted_message_names_every_bound_destination(labels: list def test_role_scope_defaulted_message_keeps_the_single_label_text(): """The one-label sentence is the text corpus/standard.yaml quotes.""" message = role_scope_defaulted_message( - _discovered_scope("destination", 2), + "destination", member_count=2, bound_dcids=["country/FRA"], label_of=_RecordingLabels(["France"]), @@ -205,9 +205,7 @@ def test_role_scope_defaulted_message_keeps_the_single_label_text(): def test_role_scope_defaulted_message_looks_up_no_label_without_a_bound_dcid(): label_of = _RecordingLabels() - role_scope_defaulted_message( - _discovered_scope("destination", 2), member_count=2, bound_dcids=[], label_of=label_of - ) + role_scope_defaulted_message("destination", member_count=2, bound_dcids=[], label_of=label_of) assert label_of.calls == [] @@ -216,7 +214,7 @@ def test_role_scope_defaulted_message_looks_up_no_label_for_four_or_more_dcids(c """Counts above three need no label lookup, so graph failures cannot break the warning.""" label_of = _RecordingLabels() message = role_scope_defaulted_message( - _discovered_scope("destination", 2), + "destination", member_count=2, bound_dcids=_dcids(count), label_of=label_of, @@ -230,7 +228,7 @@ def test_role_scope_defaulted_message_looks_up_labels_once_for_the_full_dcid_lis label_of = _RecordingLabels(["A", "B", "C"]) dcids = _dcids(count) role_scope_defaulted_message( - _discovered_scope("destination", 2), member_count=2, bound_dcids=dcids, label_of=label_of + "destination", member_count=2, bound_dcids=dcids, label_of=label_of ) assert label_of.calls == [dcids] diff --git a/tests/engine/test_multilateral_donors.py b/tests/engine/test_multilateral_donors.py index 07e1f0410..8ab064e1e 100644 --- a/tests/engine/test_multilateral_donors.py +++ b/tests/engine/test_multilateral_donors.py @@ -16,8 +16,8 @@ import pytest +from code_index.multilaterals import PartyFacts, is_multilateral, party_facts from code_index.types import PartyRow -from search.engine.multilaterals import PartyFacts, is_multilateral, party_facts from search.models import ( DatasetResubmitInput, DefinitePart, diff --git a/tests/engine/test_promote_shape_sample.py b/tests/engine/test_promote_shape_sample.py index afa09858a..50bc47f47 100644 --- a/tests/engine/test_promote_shape_sample.py +++ b/tests/engine/test_promote_shape_sample.py @@ -44,7 +44,7 @@ _USA = "country/USA" _KEN = "country/KEN" -_CRS_INCLUSIVE, _DAC2A_INCLUSIVE, _HEADLINE = (c.provenance for c in COMPOSITES) +_CRS_INCLUSIVE, _DAC2A_INCLUSIVE, _DAC2A_BILATERAL, _HEADLINE = (c.provenance for c in COMPOSITES) _YEARS = list(range(2005, 2025)) _WINDOW = {"kind": "range", "start_year": 2015, "end_year": 2020} @@ -105,7 +105,7 @@ def _recorded_identities() -> dict[str, dict]: index = get_index() identities: dict[str, dict] = {} for ref, document in _composites().items(): - identities[f"{ref}:table"] = dict(table_identity(index, document)) + identities[f"{ref}:table"] = dict(table_identity(index, document).identity) for component in component_identities(index, document) or (): identities[f"{ref}:{component.role}"] = dict(component.identity) return identities diff --git a/tests/engine/test_request_warning_sites.py b/tests/engine/test_request_warning_sites.py new file mode 100644 index 000000000..2fd2b0629 --- /dev/null +++ b/tests/engine/test_request_warning_sites.py @@ -0,0 +1,282 @@ +"""Every write to a fresh pass's warnings is declared, so a definite answer read again within +its composite's population (`pipeline._within_population`) keeps exactly the warnings the +request's words raised. + +`ResolveState.request_warnings` snapshots the pass's warnings before it interprets its read, +and a warning raised from the request's words after that is recorded through +`ResolveState.from_request`. The population read replaces every other warning about the +answer (`composite_answer.with_population_read`), so a write after the snapshot that is +neither derived from the read's rows nor recorded through `from_request` would drop a warning +the request's words raised. The sweep reads every write site under `search/engine` from the +source and holds it to `_SITES`: a new site fails until it is declared, and a site declared +`request` fails unless its function calls `state.from_request`. +""" + +from __future__ import annotations + +import ast +import pathlib +from collections import Counter +from typing import cast + +_ENGINE = pathlib.Path(__file__).resolve().parents[2] / "src" / "search" / "engine" + +# Callables that only read the list they are passed. +_READERS = frozenset( + {"tuple", "len", "list", "set", "frozenset", "sorted", "enumerate", "any", "all", "iter"} +) +_MUTATORS = frozenset({"append", "extend", "insert", "remove", "clear", "pop", "sort", "reverse"}) + +type Site = tuple[str, str, str] + + +def _is_warnings(node: ast.AST, *, module: str) -> bool: + """Whether `node` is a pass's warnings list: `state.warnings`, or `self.warnings` inside + the state module.""" + if not (isinstance(node, ast.Attribute) and node.attr == "warnings"): + return False + owner = node.value + names = {"state", "self"} if module == "region/state" else {"state"} + return isinstance(owner, ast.Name) and owner.id in names + + +def _site_ops(node: ast.AST, *, module: str, writers: frozenset[str]) -> list[str]: + """The writes `node` makes to a pass's warnings, one op each: a list method, a call that + is passed the list, a call of a state method that writes it (`writers`), an alias of the + list, or an assignment to it or an item of it.""" + ops: list[str] = [] + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Attribute) and _is_warnings(func.value, module=module): + if func.attr in _MUTATORS: + ops.append(func.attr) + elif ( + isinstance(func, ast.Attribute) + and isinstance(func.value, ast.Name) + and func.value.id == "state" + and func.attr in writers + ): + ops.append(f"state.{func.attr}") + passed = [*node.args, *(keyword.value for keyword in node.keywords)] + reader = isinstance(func, ast.Name) and func.id in _READERS + if not reader and any(_is_warnings(arg, module=module) for arg in passed): + ops.append(f"passed:{ast.unparse(func)}") + if ( + isinstance(node, ast.Assign | ast.AnnAssign) + and node.value is not None + and _is_warnings(node.value, module=module) + ): + ops.append("alias") + targets: list[ast.expr] = [] + if isinstance(node, ast.Assign | ast.Delete): + targets = list(node.targets) + elif isinstance(node, ast.AugAssign | ast.AnnAssign): + targets = [node.target] + for target in targets: + held = target.value if isinstance(target, ast.Subscript) else target + if _is_warnings(held, module=module): + ops.append("assign") + return ops + + +def _functions(tree: ast.Module) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: + return [ + node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) + ] + + +def _trees() -> dict[str, ast.Module]: + """Every module under `search/engine`, parsed, by its path below it without suffix.""" + return { + path.relative_to(_ENGINE).with_suffix("").as_posix(): ast.parse( + path.read_text(), filename=str(path) + ) + for path in sorted(_ENGINE.rglob("*.py")) + } + + +def _scanned(trees: dict[str, ast.Module]) -> dict[Site, int]: + """Every write site under `search/engine`, by module, innermost function and op, with + how many times it occurs there.""" + sites: Counter[Site] = Counter() + + def visit(node: ast.AST, module: str, function: str, writers: frozenset[str]) -> None: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + function = node.name + for op in _site_ops(node, module=module, writers=writers): + sites[(module, function, op)] += 1 + for child in ast.iter_child_nodes(node): + visit(child, module, function, writers) + + visit(trees["region/state"], "region/state", "", frozenset()) + writers = frozenset(function for module, function, _ in sites if module == "region/state") + for module, tree in trees.items(): + if module != "region/state": + visit(tree, module, "", writers) + return dict(sites) + + +_R, _Q, _B, _P, _M = "read", "request", "before", "resubmit", "method" + +# Each write site, by module, function and op, with the kind of each occurrence in source +# order: +# - before: runs before the pass snapshots its warnings (`ResolveState.request_warnings`); +# - read: derived from the rows the read returned, which the population read replaces; +# - request: raised from the request's words after the snapshot, so the function records it +# through `state.from_request`; +# - keeps_key: rewrites a held warning's params or drops held warnings, raising none, so a +# snapshot warning keeps its key (`departures.warning_key`); +# - resubmit: the resubmit pass (`_PromoteState`), which takes no snapshot; +# - method: a state method, whose callers are the sites. +_SITES: dict[Site, tuple[str, ...]] = { + ("region/candidate_collapse", "_assemble_candidate_set", "append"): (_Q,), + ("region/candidate_collapse", "_assemble_candidate_set", "extend"): (_R,), + ("region/candidate_collapse", "_assemble_definite", "append"): (_R,), + ("region/candidate_collapse", "_assemble_definite", "passed:_attach_observations"): (_R, _R), + ("region/candidate_collapse", "_assemble_definite", "passed:install_place_warnings"): (_Q,), + ("region/candidate_collapse", "assemble_candidates", "state.state_reelected_reading"): ( + "keeps_key", + ), + ("region/candidate_collapse", "_handle_sparse_survivors", "append"): (_Q, _Q), + ( + "region/candidate_collapse", + "_offer_place_warnings", + "passed:stamp_offered_place_warnings", + ): (_Q,), + ("region/candidate_grounding", "_warn_for_candidates", "extend"): (_R, _R, _R), + ( + "region/candidate_grounding", + "restate_scope_warning", + "passed:restate_role_scope_warnings", + ): (_R,), + ("region/candidate_options", "_attach_candidate_entity_options", "append"): (_R,), + ("region/candidate_options", "_attach_candidate_facet_options", "append"): (_R,), + ("region/candidate_options", "_attach_candidate_slot_options", "append"): (_R,), + ("region/phase_answer", "_bound_unit_refusal", "append"): (_R,), + ("region/phase_answer", "_finalize_definite_result", "passed:_attach_observations"): (_R, _R), + ("region/phase_answer", "_assemble_definite_answer", "state.state_reelected_reading"): ( + "keeps_key", + ), + ("region/phase_answer", "_gather_definite_coverage", "append"): (_R, _R, _R), + # A member's family total is absent from the rows read; the answer is no_data. + ("region/phase_answer", "_ground_elected_answer", "state.depart"): (_R,), + ("region/phase_answer", "_ground_elected_answer", "extend"): (_R,), + ( + "region/phase_answer", + "_install_filtered_answer", + "passed:restate_role_scope_warnings", + ): (_R,), + # The years the request asked beside those the conversion tables cover; the answer's + # identity holds the narrowed window, so its population read departs nothing. + ("region/phase_answer", "_narrowed_to_period", "state.depart"): (_Q,), + # A trailing request's stale and passed-over years name the span its words asked, then + # the deflator omission the read's plan gives. + ("region/phase_answer", "_over_conversion_period", "state.depart"): (_Q, _Q, _Q, _R), + ("region/phase_answer", "_start_assemble_definite", "append"): (_R,), + ("region/phase_election", "_install_settled", "extend"): (_B,), + ("region/phase_election", "_repair_bindings", "append"): (_B, _B), + ("region/phase_election", "_repair_bindings", "extend"): (_B, _B), + ("region/phase_election", "_vet_election", "append"): (_B,), + ("region/phase_place_materialise", "_install_shape", "passed:install_place_warnings"): (_B,), + ("region/phase_place_materialise", "_unbind_refuted_claims", "state.forget_bound"): ( + "keeps_key", + ), + ("region/phase_place_materialise", "_nearest_real_for", "append"): (_R,), + ("region/phase_place_materialise", "_nearest_real_for", "extend"): (_R,), + ("region/phase_place_materialise", "_nearest_real_for", "passed:_attach_observations"): (_R,), + ( + "region/phase_place_materialise", + "_nearest_real_for", + "passed:stamp_offered_place_warnings", + ): (_Q,), + ("region/phase_place_materialise", "_note_named_places", "append"): (_B,), + ( + "region/phase_place_materialise", + "_phase_materialise", + "passed:interpret_materialised", + ): (_R,), + ("region/phase_place_materialise", "_phase_materialise", "passed:settle_place"): (_R,), + ("region/phase_place_materialise", "_phase_place_bindings", "append"): (_B,), + ("region/phase_place_materialise", "_reelect_zero_observation_sibling", "append"): (_B,), + ("region/phase_place_materialise", "note_denominator_unverified", "append"): (_Q,), + ("region/phase_rank_bind", "_phase_bind", "append"): (_B,), + ("region/phase_rank_bind", "_phase_prune_and_rank", "extend"): (_B,), + ("region/phase_rank_bind", "_read_availability", "append"): (_B,), + ("region/phase_shared", "_elected_flagged_match", "extend"): (_R,), + # The raw-text door's unit phrase the answer does not apply. + ("region/phase_shared", "withdraw_typed_unit", "state.depart"): (_Q,), + ("region/promote_finalize", "_bound_unit_refusal", "append"): (_P,), + ("region/promote_ground", "_ground_and_build_spec", "state.depart"): (_P,), + ( + "region/promote_ground", + "_install_narrowed_answer", + "passed:restate_role_scope_warnings", + ): (_P,), + ("region/promote_ground", "_narrowed_to_period", "append"): (_P,), + ("region/promote_ground", "_narrowed_to_period", "state.depart"): (_P,), + ("region/promote_ground", "_over_conversion_period", "append"): (_P,), + ("region/promote_ground", "_over_conversion_period", "state.depart"): (_P,), + ("region/promote_ground", "_phase_ground_build", "state.depart"): (_P,), + ("region/promote_materialise", "_phase_materialise", "passed:interpret_materialised"): (_P,), + ("region/promote_materialise", "_phase_materialise", "passed:settle_place"): (_P,), + ("region/promote_places", "_phase_where_places", "extend"): (_P,), + ("region/promote_reread", "_phase_reread_arcs", "append"): (_P,), + ("region/promote_shape", "_phase_build_shape", "alias"): (_P,), + ("region/state", "depart", "append"): (_M,), + ("region/state", "forget_bound", "assign"): (_M,), + ("region/state", "state_reelected_reading", "assign"): (_M,), + ("region/unrealisable", "unbind_unrealisable", "state.forget_bound"): ("keeps_key",), +} + + +def _function(trees: dict[str, ast.Module], module: str, name: str) -> ast.AST: + (function,) = [node for node in _functions(trees[module]) if node.name == name] + return function + + +def test_every_write_to_a_pass_s_warnings_is_declared() -> None: + """A write site the table does not declare, or declares a different number of times, + fails until its kind is declared.""" + declared = {site: len(kinds) for site, kinds in _SITES.items()} + + assert _scanned(_trees()) == declared + + +def test_a_request_site_records_its_warning_through_from_request() -> None: + """A function declaring a warning raised from the request's words after the snapshot + calls `state.from_request`, so a population read keeps the warning.""" + trees = _trees() + unrecorded = [ + (module, function) + for (module, function, _), kinds in _SITES.items() + if _Q in kinds + and not any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "from_request" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "state" + for node in ast.walk(_function(trees, module, function)) + ) + ] + + assert unrecorded == [] + + +def test_a_resubmit_site_writes_the_resubmit_pass_s_warnings() -> None: + """A site declared `resubmit` writes a `_PromoteState`, the resubmit pass's state, + which takes no snapshot.""" + trees = _trees() + misplaced = [ + (module, function) + for (module, function, _), kinds in _SITES.items() + if _P in kinds + and not any( + arg.arg == "state" + and arg.annotation is not None + and ast.unparse(arg.annotation) == "_PromoteState" + for arg in cast("ast.FunctionDef", _function(trees, module, function)).args.args + ) + ] + + assert misplaced == [] diff --git a/tests/engine/test_role_class_shapes.py b/tests/engine/test_role_class_shapes.py index 0e3387957..c9d054c2a 100644 --- a/tests/engine/test_role_class_shapes.py +++ b/tests/engine/test_role_class_shapes.py @@ -1560,7 +1560,10 @@ def _open_creditor_promote_state( ) state = _PromoteState( inp=cast( - Any, SimpleNamespace(identity=SimpleNamespace(facets=None, open_roles=posted_open)) + Any, + SimpleNamespace( + identity=SimpleNamespace(facets=None, open_roles=posted_open), populations={} + ), ), graph=cast(Any, FakeGraph(raise_on_call=True)), sdmx=cast(Any, None), diff --git a/tests/engine/test_whole_total_bind_e2e.py b/tests/engine/test_whole_total_bind_e2e.py index 1f7146bcd..02e439d64 100644 --- a/tests/engine/test_whole_total_bind_e2e.py +++ b/tests/engine/test_whole_total_bind_e2e.py @@ -25,6 +25,7 @@ import pytest import search.scope_window as scope_window_module +from code_index.declarations import DAC2A_BILATERAL from code_index.shared import get_index from search.engine.bind import SlotBindingDraft from search.engine.binders import BindRequest, BindResult, GeminiBinder @@ -50,9 +51,12 @@ from search.engine.role_text import DoorRole from search.models import ( CandidatesPart, + ChoiceOrigin, DatasetResubmitInput, DefinitePart, EntityGroup, + Entry, + IdentityBindingValue, IdentityShapeSchema, IdentityWindowRange, NoDataPart, @@ -102,6 +106,7 @@ _OFFICIAL = "DACOfficialDonors" _PRIVATE = "DACPrivateDonors" _ALL_RECIPIENTS = "DACAllRecipients" +_DEVELOPING = "DACDevelopingCountries" _VARIABLE = "official development assistance" @@ -247,6 +252,36 @@ def _available(part: DefinitePart) -> list[dict[str, Any] | None]: return [warning.params for warning in part.warnings if warning.code == "whole_total_available"] +def _under_total_oda(spec: Spec) -> DefinitePart: + """The answer to `spec`'s identity resubmitted under `provider:contributions`, DAC2a's + aid type 206 read bare, which counts the core contributions the bilateral reading + leaves out.""" + switch = Entry( + axis="provider", + option_id="provider:contributions", + provenance="explicit", + origin=ChoiceOrigin(choice_id=f"{spec.spec_id}:provider"), + ) + return _part( + offline_resolve( + ResolveRequest( + input=DatasetResubmitInput( + entries=[ + ResubmitIdentityEntry( + variable_text=_VARIABLE, + identity=spec.definition.identity, + scope_entries=[switch], + ) + ] + ) + ), + graph=_graph(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=FakeSdmxClient(observations={}, availability={}), + ) + ) + + def _origin(spec: Spec) -> list[str] | None: return spec.definition.identity.entities.get("origin") @@ -781,7 +816,9 @@ def test_an_alternate_match_states_the_total_its_reading_lists_beside( series in DAC2a alone, so the part refuses `no_observations` and offers DAC2a's reading as its match (`only_alternate_has_data`). That reading lists its donors, and DAC2a publishes the official donors' total with a series to Ethiopia, so the part states the - total, stamped with the match, as the match's definite answer would.""" + total, stamped with the match, as the match's definite answer would. The match reads + DAC2a's bilateral destinations, as that definite answer would, so a client fetching it + reads no agency.""" names = RoleNames(donor="financingProvider", recipient="financingRecipient") patch_coverage( monkeypatch, @@ -809,6 +846,9 @@ def test_an_alternate_match_states_the_total_its_reading_lists_beside( match = answer.no_data.alternate_match assert match is not None assert match.definition.identity.entities == {"destination": [_ETH]} + assert (match.definition.identity.facets or {})["provenance"] == IdentityBindingValue( + ref=DAC2A_BILATERAL + ) assert [ (warning.spec_id, warning.params) for warning in answer.warnings @@ -1764,11 +1804,14 @@ def test_a_place_without_a_direction_beside_a_named_population_takes_the_other_s assert [option.label for option in answer.no_data.options or ()] == ["To Ethiopia"] -def _india_coverage(monkeypatch: pytest.MonkeyPatch, *, aid_type: str = "own") -> None: +def _india_coverage( + monkeypatch: pytest.MonkeyPatch, *, aid_type: str = "own", developing: bool = False +) -> None: """DAC2a holds series in 2021 to India from Germany and from the official donors' total, the latter for the official development assistance member's own aid type or for - `aid_type`, and from India to every recipient and to Ethiopia; the CRS holds India's - series to Ethiopia. No dataset declares India a provider.""" + `aid_type`, and from India to every recipient and to Ethiopia, and with `developing` to + the developing countries' total; the CRS holds India's series to Ethiopia. No dataset + declares India a provider.""" bound = _DAC2A_MEMBER.split("AidType")[1].split("_")[0] held = ( _DAC2A_MEMBER @@ -1783,6 +1826,7 @@ def _india_coverage(monkeypatch: pytest.MonkeyPatch, *, aid_type: str = "own") - (_DAC2A_MEMBER, "dac2a", "country/DEU", _IND, [2021]), (_DAC2A_MEMBER, "dac2a", _IND, _ALL_RECIPIENTS, [2021]), (_DAC2A_MEMBER, "dac2a", _IND, _ETH, [2021]), + *([(_DAC2A_MEMBER, "dac2a", _IND, _DEVELOPING, [2021])] if developing else []), (_CRS_MEMBER, "crs", _IND, _ETH, [2021]), ], role_names=dict.fromkeys((held, _DAC2A_MEMBER, _CRS_MEMBER), names), @@ -1839,11 +1883,7 @@ def test_an_alternate_judges_the_total_under_its_own_bindings( pytest.param( "recipient", {"origin": [_IND]}, - { - "total": _ALL_RECIPIENTS, - "total_label": "all recipients (DAC)", - "whole_of": "official,residual", - }, + {"total": _DEVELOPING, "total_label": "developing countries (DAC)", "whole_of": ""}, id="recipients-listed", ), ], @@ -1855,10 +1895,13 @@ def test_a_candidate_set_states_the_total_beside_the_reading_that_lists_its_side params: dict[str, str], ): """India named with no direction offers a reading of each direction. Opening one role - lists that side member by member in the reading where India stands on the other side, - and DAC2a publishes that side's total with a series beside India, so the set states - the total once, stamped with that reading, as a definite answer does.""" - _india_coverage(monkeypatch) + lists that side member by member in the reading where India stands on the other side. + Each reading reads DAC2a's bilateral destinations, as a definite answer does. DAC2a + publishes the donors' total with a series beside India, so the set states it once, + stamped with that reading. Beside the recipients it states the developing countries' + total, the total of the bilateral side, in place of the total of every recipient, which + holds the core contributions the reading leaves out.""" + _india_coverage(monkeypatch, developing=True) response = _resolve( { @@ -1875,6 +1918,9 @@ def test_a_candidate_set_states_the_total_beside_the_reading_that_lists_its_side (reading,) = [ spec for spec in part.candidates.specs if spec.definition.identity.entities == listed ] + assert (reading.definition.identity.facets or {})["provenance"] == IdentityBindingValue( + ref=DAC2A_BILATERAL + ) stated = [ (warning.spec_id, warning.params) for warning in part.warnings @@ -1883,6 +1929,36 @@ def test_a_candidate_set_states_the_total_beside_the_reading_that_lists_its_side assert stated == [(reading.spec_id, params)] +def test_a_recipients_reading_under_total_oda_states_the_total_beside_its_recipients( + monkeypatch: pytest.MonkeyPatch, +): + """Under `provider:contributions`, DAC2a's aid type 206 read bare, the recipients + listed beside India hold the core contributions the all-recipients total holds, so the + reading states that total as a `DACAllRecipients` warning.""" + _india_coverage(monkeypatch) + response = _resolve( + { + "entities": [{"text": "India"}], + "open_roles": ["recipient"], + "time": "2021", + "variable": _VARIABLE, + } + ) + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, CandidatesPart), part + (reading,) = [ + spec + for spec in part.candidates.specs + if spec.definition.identity.entities == {"origin": [_IND]} + ] + + answer = _under_total_oda(reading) + + assert (answer.spec.definition.identity.facets or {}).get("provenance") is None + assert [warning["total"] for warning in _available(answer) if warning] == [_ALL_RECIPIENTS] + + # DAC2a's gross imputed multilateral aid, a shape of its own beside the official # development assistance member's, whose representative binds another aid type. _DAC2A_ATTRIBUTED = "AttributedAmount_FinancialFlow_Gross_Disbursement_DAC2aAidType106_Nominal" @@ -2032,6 +2108,12 @@ def test_a_resubmitted_candidate_set_states_the_total_for_its_own_reading( if warning.code == "whole_total_available" ] assert stated == [(dac2a.spec_id, _OFFICIAL)] + # The fresh DAC2a reading is the composite of its bilateral destinations, which the set + # posts back and answers as posted. + readings = {spec.spec_id: spec for spec in part.candidates.specs} + assert (readings[dac2a.spec_id].definition.identity.facets or {})[ + "provenance" + ] == IdentityBindingValue(ref=DAC2A_BILATERAL) def test_a_resubmit_refusing_its_posted_aid_type_states_the_total_beside_its_flagged_match( @@ -3114,11 +3196,6 @@ def test_a_population_covers_a_side_its_datasets_publish_no_total_of(dcids, data "total": _ALL_RECIPIENTS, "whole_of": ["official", "residual"], } -_RECIPIENTS_AVAILABLE = { - "total": _ALL_RECIPIENTS, - "total_label": "all recipients (DAC)", - "whole_of": "official,residual", -} @pytest.mark.parametrize( @@ -3171,7 +3248,7 @@ def test_a_population_covers_a_side_its_datasets_publish_no_total_of(dcids, data }, False, None, - [_RECIPIENTS_AVAILABLE], + [], id="recipients-ranked", ), ], @@ -3187,8 +3264,9 @@ def test_a_recipient_side_reads_whole_inside_the_window( to all recipients, or to no recipient named, in 2021 reads it whole. Named in 1983, it binds as a named party outside the window, with no whole reading. To no recipient named over no stated period, or ranked over the recipients, the part reads the recipients. - Only the ranked recipients, listed in 2021, are listed inside the total's window, so - only they state it beside them.""" + Only the ranked recipients are listed inside the total's window, and they read DAC2a's + bilateral destinations, which leave out the core contributions the total holds, so no + part states the total beside its recipients.""" _coverage(monkeypatch, years=(1983, 2021)) answer = _part(_resolve(part)) @@ -3201,6 +3279,32 @@ def test_a_recipient_side_reads_whole_inside_the_window( assert _available(answer) == listed +def test_ranked_recipients_under_total_oda_state_the_total_beside_them( + monkeypatch: pytest.MonkeyPatch, +): + """The ranked recipients of France's aid read DAC2a's aid type 206 bare under + `provider:contributions`, which holds the core contributions the all-recipients total + holds, so the part states that total as a `DACAllRecipients` warning.""" + _coverage(monkeypatch, years=(1983, 2021)) + ranked = _part( + _resolve( + { + "entities": [_FROM_FRANCE], + "operations": [ + {"over": "destination", "phrase": "top 10 recipients", "verb": "rank"} + ], + "time": "2021", + "variable": _VARIABLE, + } + ) + ) + + answer = _under_total_oda(ranked.spec) + + assert (answer.spec.definition.identity.facets or {}).get("provenance") is None + assert [warning["total"] for warning in _available(answer) if warning] == [_ALL_RECIPIENTS] + + @pytest.mark.parametrize( ("official", "entities"), [ diff --git a/tests/fixtures/first_answer_place_scenarios.json b/tests/fixtures/first_answer_place_scenarios.json index 24be44468..503dd4f10 100644 --- a/tests/fixtures/first_answer_place_scenarios.json +++ b/tests/fixtures/first_answer_place_scenarios.json @@ -466,7 +466,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -1225,7 +1225,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -1512,7 +1512,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -2140,7 +2140,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -2393,7 +2393,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -2933,7 +2933,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -3537,7 +3537,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -4524,7 +4524,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -4816,7 +4816,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -5560,7 +5560,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -5876,7 +5876,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -6555,7 +6555,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -6884,7 +6884,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -7393,7 +7393,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -7986,7 +7986,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -8632,7 +8632,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -8882,7 +8882,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -9076,7 +9076,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -10032,7 +10032,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -10639,7 +10639,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -11362,7 +11362,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -11612,7 +11612,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -12221,7 +12221,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -12860,7 +12860,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -13612,7 +13612,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -14153,7 +14153,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -14355,7 +14355,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -15061,7 +15061,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -15599,7 +15599,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -16074,7 +16074,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -16536,7 +16536,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -17011,7 +17011,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -17519,7 +17519,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -17981,7 +17981,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -18205,7 +18205,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -19109,7 +19109,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -19339,7 +19339,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -19945,7 +19945,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -20175,7 +20175,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -20714,7 +20714,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -21178,7 +21178,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -21642,7 +21642,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -21815,7 +21815,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -22528,7 +22528,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -23094,7 +23094,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -23707,7 +23707,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -24417,7 +24417,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -24647,7 +24647,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -25377,7 +25377,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -25607,7 +25607,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -26264,7 +26264,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -26992,7 +26992,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -27229,7 +27229,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -27408,7 +27408,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -28330,7 +28330,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -28560,7 +28560,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -29333,7 +29333,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -29563,7 +29563,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -29742,7 +29742,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -30611,7 +30611,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -30814,7 +30814,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -31738,7 +31738,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -32049,7 +32049,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -32770,7 +32770,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -33081,7 +33081,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -33792,7 +33792,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -34534,7 +34534,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -34764,7 +34764,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -35488,7 +35488,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -35718,7 +35718,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -36414,7 +36414,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -36644,7 +36644,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -37119,7 +37119,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -37777,7 +37777,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -38049,7 +38049,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -38711,7 +38711,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -38983,7 +38983,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -40045,7 +40045,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -40317,7 +40317,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -40930,7 +40930,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -41241,7 +41241,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -41865,7 +41865,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, @@ -42095,7 +42095,7 @@ "served_count": 1 }, "trace": { - "capability_version": "cap_b29a91445f7a4da3", + "capability_version": "cap_eed8849e3076f272", "conversion": null, "default_reading": { "applied": true, diff --git a/tests/graph_release/test_service.py b/tests/graph_release/test_service.py index 883895135..89513db88 100644 --- a/tests/graph_release/test_service.py +++ b/tests/graph_release/test_service.py @@ -24,6 +24,7 @@ import graph_release.service as service_module from artifact_publication.uri import write_uri_bytes +from code_index.composites import PopulationCheck, UnclassifiedDestinations, population_check from code_index.shared import get_index from datasets_api.admission import MaterialisePermits from datasets_api.app import datasets_surface @@ -795,3 +796,32 @@ def test_an_operation_runs_on_the_release_served_when_it_is_delivered( _recheck(service, clock) assert self._bound_release(service) == [second.release_id] + + +def test_opening_a_release_checks_its_destination_populations_against_its_coverage( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + """A population missing a destination coverage serves is logged as the release opens, and + the release still serves; the session release's own populations hold every destination.""" + loaded = current_release().loaded + views = current_release().views + assert population_check(views.code_index, views.coverage).unclassified == {} + drift = UnclassifiedDestinations( + dataset="provenance/OECDDAC2A", destinations=("SomeNewAgency",), found_in="coverage" + ) + checked: list[tuple[object, object]] = [] + + def check(index, coverage): + checked.append((index, coverage)) + return PopulationCheck(unclassified={drift.dataset: drift}) + + monkeypatch.setattr(service_module, "population_check", check) + caplog.set_level(logging.INFO, logger=service_module.__name__) + + opened = service_module.build_release_views(loaded) + + assert checked == [(opened.code_index, opened.coverage)] + (event,) = _events(caplog, "destination_population_unavailable") + assert event["dataset"] == drift.dataset + assert event["found_in"] == "coverage" + assert event["destinations"] == ["SomeNewAgency"] diff --git a/tests/materialise/test_composite.py b/tests/materialise/test_composite.py index a6f6109b8..b570dc464 100644 --- a/tests/materialise/test_composite.py +++ b/tests/materialise/test_composite.py @@ -20,7 +20,7 @@ stream_frame_sha256, ) -from code_index.composites import DECLARED_COMPOSITES, composite_digest +from code_index.composites import DECLARED_COMPOSITES from code_index.declarations import COMPOSITES, CRS_INCLUSIVE from code_index.types import ComponentRecipe, ComponentRole, CompositeDeclaration from materialise import MaterialisationRequest, MaterialisationResult @@ -47,6 +47,8 @@ _STACKED = DECLARED_COMPOSITES[CRS_INCLUSIVE] _SPLICED = next(c for c in COMPOSITES if c.kind == "spliced") _ENRICHMENT = EnrichmentLookup(FakeGraphReader()) +# The composite digest a mint passes in, recorded as it is. +_DIGEST = "sha256:composed" # CRS and imputed members of one sector. The imputed dcid sorts before the CRS one, so the # composed frame leads with the zero-filling component's rows. @@ -199,6 +201,7 @@ def _compose( role_columns=_ROLES, enrichment=_ENRICHMENT, parties={}, + digest=_DIGEST, ) @@ -441,7 +444,7 @@ def test_the_composite_keeps_the_import_key_only_when_every_component_shares_it( dependencies = _compose([bilateral, imputed]).dependencies assert dependencies.imported_data == expected - assert dependencies.composites == ((_STACKED.provenance, composite_digest(_STACKED)),) + assert dependencies.composites == ((_STACKED.provenance, _DIGEST),) def test_the_composed_frame_records_each_components_measurement_method_under_one_unit(): diff --git a/tests/materialise/test_dependencies.py b/tests/materialise/test_dependencies.py index 1de53f15b..59a757dc0 100644 --- a/tests/materialise/test_dependencies.py +++ b/tests/materialise/test_dependencies.py @@ -5,14 +5,19 @@ from __future__ import annotations +import gzip +import json from dataclasses import replace from typing import Any import httpx import pytest +import code_index.composites as composites_module from code_index.composites import declared_composite_digest -from code_index.declarations import CRS_INCLUSIVE +from code_index.declarations import COMPOSITES, CRS_INCLUSIVE, DAC2A_BILATERAL +from code_index.loader import parse_index +from code_index.scope_conventions import destination_side from dc_sdmx import SdmxClient from graph_release.context import current_release from materialise import EnrichmentLookup, MaterialisationRequest, materialise_group @@ -33,11 +38,22 @@ from origin_catalogue import DataCommons, bound_catalogue from origin_catalogue.origins import BASE, DFO, UN_SYSTEM from series_coverage.schema import FacetMetadata, RoleNames +from tests.code_index.support import INDEX_FIXTURE from tests.engine._fake_coverage import build_coverage_index from tests.origin_catalogue.fake_catalogue import FakeCatalogue, provenance_row from .conftest import FakeGraphReader + +def _parse_index(): + with gzip.open(INDEX_FIXTURE) as handle: + return parse_index(json.load(handle), source="fixture", composites=COMPOSITES) + + +# A code index declaring every composite, whose DAC2a population every destination places. +_INDEX = _parse_index() + + _STAT_VAR = "dcid:MalariaGrantDisbursements" _BARE_STAT_VAR = "MalariaGrantDisbursements" _RECIPE_ID = next(iter(sorted(RECIPE_BODIES))) @@ -181,7 +197,11 @@ def test_a_refreshed_fingerprint_reads_every_stored_ref_in_one_call(): window = election_window(RequestedWindow(kind="range", start_year=2015), clock_year=2026) refreshed = refreshed_dependencies( - _BASE, [_request("A", window), _request("B", window)], graph=graph, coverage=None + _BASE, + [_request("A", window), _request("B", window)], + graph=graph, + coverage=None, + code_index=_INDEX, ) assert graph.node_arcs_many_calls == [ @@ -212,7 +232,7 @@ def test_a_refreshed_fingerprint_reads_every_stored_ref_in_one_call(): def test_a_composite_head_rebuilds_when_its_deployed_declaration_changes( stored_digest, provenance, changed ): - digest = stored_digest or declared_composite_digest(provenance) + digest = stored_digest or declared_composite_digest(provenance, index=_INDEX) stored = replace(_BASE, composites=((provenance, digest),)) graph = FakeGraphReader( arcs={ @@ -223,7 +243,11 @@ def test_a_composite_head_rebuilds_when_its_deployed_declaration_changes( window = election_window(RequestedWindow(kind="range", start_year=2015), clock_year=2026) current = replace( refreshed_dependencies( - stored, [_request("A", window), _request("B", window)], graph=graph, coverage=None + stored, + [_request("A", window), _request("B", window)], + graph=graph, + coverage=None, + code_index=_INDEX, ), recipes=stored.recipes, ) @@ -235,6 +259,45 @@ def test_a_composite_head_rebuilds_when_its_deployed_declaration_changes( ) +def test_a_composite_whose_population_is_unavailable_rebuilds_without_raising( + monkeypatch: pytest.MonkeyPatch, +): + """Under a code index whose side declarations no longer place a DAC2a destination, the + bilateral composite has no digest: the check reports it changed, so the head rebuilds and + the mint answers the refusal, and nothing raises out of the check.""" + stored = replace( + _BASE, + composites=((DAC2A_BILATERAL, declared_composite_digest(DAC2A_BILATERAL, index=_INDEX)),), + ) + + def unplaced_kenya(row, *, datasets): + return None if row.dcid == "country/KEN" else destination_side(row, datasets=datasets) + + monkeypatch.setattr(composites_module, "destination_side", unplaced_kenya) + drifted = _parse_index() + graph = FakeGraphReader( + arcs={ + "provenance/OECDDAC2A": {"lastDataRefreshDate": ("2026-10-07",)}, + "provenance/ONEConversionTablesDAC": {"lastDataRefreshDate": ("2026-09-30",)}, + } + ) + window = election_window(RequestedWindow(kind="range", start_year=2015), clock_year=2026) + + current = replace( + refreshed_dependencies( + stored, + [_request("A", window), _request("B", window)], + graph=graph, + coverage=None, + code_index=drifted, + ), + recipes=stored.recipes, + ) + + assert current.composites == ((DAC2A_BILATERAL, None),) + assert dependency_changes(stored.as_json(), current) == (f"composites:{DAC2A_BILATERAL}",) + + def test_a_source_with_no_date_on_both_sides_is_unchanged(): current = replace(_BASE, sources=(("provenance/IMF", None),)) assert dependency_changes(current.as_json(), current) == () @@ -564,7 +627,9 @@ def test_a_refresh_during_the_fetch_records_the_pre_fetch_token_and_the_next_che assert result.coverage.rows == members assert result.dependencies.sources == (("OECD_DAC2A", "2026-10-07"),) - current = refreshed_dependencies(result.dependencies, requests, graph=graph, coverage=None) + current = refreshed_dependencies( + result.dependencies, requests, graph=graph, coverage=None, code_index=_INDEX + ) assert dependency_changes(result.dependencies.as_json(), current) == ("sources:OECD_DAC2A",) diff --git a/tests/materialise/test_facets.py b/tests/materialise/test_facets.py index 7d47c6a61..256af7323 100644 --- a/tests/materialise/test_facets.py +++ b/tests/materialise/test_facets.py @@ -17,7 +17,7 @@ import materialise.facets as facets_module from dc_graph import earliest_date, latest_date from dc_graph.client import _build_facet, _parse_ordered_facets -from dc_sdmx import SdmxCapExceededError +from dc_sdmx import Among, SdmxCapExceededError from dfo_platform.problem import Alternative from materialise.errors import ( CollectionProbeCeilingExceededError, @@ -513,8 +513,21 @@ def test_probe_facets_role_bearing_with_no_bound_role_covers_every_coverage_row( coverage=coverage, ) + # A population binds its members, as a list naming them does. + population = probe_facets( + availability=NO_SCOPE_READ, + stat_var="MalariaGrantDisbursements", + bound_entities=(), + settle=True, + graph=graph, + declared_roles=("financingProvider", "financingRecipient"), + role_bindings={"financingRecipient": Among(members=("country/KEN",), side="bilateral")}, + coverage=coverage, + ) + assert [(c.facet_id, c.obs_count) for c in unbound.candidates] == [("facet1", 3)] assert [(c.facet_id, c.obs_count) for c in kenya.candidates] == [("facet1", 2)] + assert [(c.facet_id, c.obs_count) for c in population.candidates] == [("facet1", 2)] # Coverage rows serve unmasked, with a role bound or with none. assert unbound.admission == kenya.admission == CoverageAdmission() assert sum(graph.calls.values()) == 0 diff --git a/tests/presentation_store/test_frame_store.py b/tests/presentation_store/test_frame_store.py index 29a708cee..e16cf1252 100644 --- a/tests/presentation_store/test_frame_store.py +++ b/tests/presentation_store/test_frame_store.py @@ -49,6 +49,8 @@ malaria_ethiopia_frame_ref, malaria_ethiopia_rows, ) +from code_index.composites import DECLARED_COMPOSITES +from code_index.declarations import DAC2A_BILATERAL from code_index.members import canonical_identity from code_index.query import CodeIndex from code_index.shared import get_index @@ -3779,6 +3781,30 @@ def test_load_cites_the_distinct_upstream_provenances_of_a_published_ops_frame() } +def test_load_cites_a_composite_ops_frame_by_its_component_tables(): + """An ops frame keeping a composite's provenance cites the composite's component table, + as a version of the composite does, and reads no node for the composite, which the graph + does not hold.""" + snapshots, artifacts, frame_ref, _ = _ops_happy_path_fixture() + rows = [{**row, "provenance": DAC2A_BILATERAL} for row in malaria_ethiopia_rows()] + data = json.dumps(build_frame_doc(role_keys=("recipient",), rows=rows)).encode("utf-8") + digest = artifacts.seed(tier=Tier.working, data=data) + snapshots.locations[(frame_ref.spec_id, frame_ref.content_version)] = [ + _location(digest=digest, tier=Tier.working.value) + ] + (table,) = {recipe.dataset for recipe in DECLARED_COMPOSITES[DAC2A_BILATERAL].components} + graph = _FakeGraphReader(arcs={table: {"typeOf": ("Provenance",), "name": ("OECD DAC2a",)}}) + + resolved = _store(snapshots=snapshots, artifacts=artifacts, graph=graph).load( + frame_ref=frame_ref + ) + + citation = cast("Mapping[str, Any]", resolved.manifest["citation"]) + assert [(source["ref"], source["label"]) for source in citation["sources"]] == [ + (table, "OECD DAC2a") + ] + + def test_load_caches_an_ops_frame_s_citation_with_the_frame(): """A cache hit serves the citation without reading the graph again.""" snapshots, artifacts, frame_ref, _ = _ops_happy_path_fixture() diff --git a/tests/render/test_render_departures.py b/tests/render/test_render_departures.py index 1d7317ef9..af81027ee 100644 --- a/tests/render/test_render_departures.py +++ b/tests/render/test_render_departures.py @@ -1013,6 +1013,42 @@ "You named Korea without a direction. OECD DAC2a lists Korea among its providers, so " "this reads it as the creditor.", ), + # A DAC2a answer naming a multilateral recipient reads the donor's core contributions to + # it, worded by what the recipient is: an agency, the DAC's total over agencies, or the + # DAC's line for agencies it does not name. + ( + DepartureFact( + code="core_contribution_read", + binding_ref="where:", + shown="European Commission", + recipient="party", + ), + "aid", + "European Commission is a multilateral organisation, so these figures are core " + "contributions to it, counted in total ODA.", + ), + ( + DepartureFact( + code="core_contribution_read", + binding_ref="where:", + shown="Multilateral Organisations, Total", + recipient="total", + ), + "aid", + "Multilateral Organisations, Total is the DAC's total over multilateral agencies, so " + "these figures are core contributions to them, counted in total ODA.", + ), + ( + DepartureFact( + code="core_contribution_read", + binding_ref="where:", + shown="Other UN", + recipient="residual", + ), + "aid", + "Other UN is the DAC's line for multilateral agencies it does not name, so these " + "figures are core contributions to them, counted in total ODA.", + ), # crs-010: the check covered only the ODA default the engine chose. ( DepartureFact(code="scope_limited", binding_ref="how:crsTypeOfFlow", limits=("ODA",)), @@ -1291,6 +1327,14 @@ "You named Kenya without a direction. Only the data with Kenya as the recipient " "holds values, so this reads it as the recipient.", ), + ( + DepartureFact( + code="core_contribution_read", binding_ref="where:", shown="European Commission" + ), + "aid", + "European Commission is a multilateral organisation, so its figures would be core " + "contributions to it, counted in total ODA.", + ), ( DepartureFact(code="scope_limited", binding_ref="how:crsTypeOfFlow", limits=("ODA",)), "aid", diff --git a/tests/scope/test_composite_preview.py b/tests/scope/test_composite_preview.py index 152ee6aad..8e74e8789 100644 --- a/tests/scope/test_composite_preview.py +++ b/tests/scope/test_composite_preview.py @@ -30,7 +30,7 @@ from tests.code_index.aid_identity import education_aid_identity from tests.engine._fake_coverage import FakeCoverageService, build_coverage_index -_CRS_INCLUSIVE, _DAC2A_INCLUSIVE, _HEADLINE = COMPOSITES +_CRS_INCLUSIVE, _DAC2A_INCLUSIVE, _DAC2A_BILATERAL, _HEADLINE = COMPOSITES def _served(declaration: CompositeDeclaration, *years: range) -> list[ServedComponent]: diff --git a/tests/scope/test_provider.py b/tests/scope/test_provider.py index dead9e15f..672265640 100644 --- a/tests/scope/test_provider.py +++ b/tests/scope/test_provider.py @@ -17,8 +17,9 @@ import pytest -from code_index.composites import component_identities +from code_index.composites import CompositeNotServable, component_identities from code_index.scope_conventions import ( + BILATERAL_SOURCES, INCLUSIVE_SOURCES, OPTION_LABELS, PROVIDER_CONVENTIONS, @@ -26,13 +27,18 @@ PUBLISHED_PROVIDER_SCOPES, ) from code_index.shared import get_index +from origin_catalogue import bound_catalogue +from search.engine import departures +from search.engine.region.scope_decision import decide_scope from search.engine.spec_id import identity_spec_id +from search.heading.departure_fact import DepartureFact from search.models import ( BaseIdentity, ChoiceOrigin, DatasetResubmitInput, DefinitePart, Entry, + IdentityBindingSet, IdentityBindingValue, IdentityWindowRange, ParsedEntity, @@ -48,20 +54,23 @@ SlotBinding, Spec, ) +from search.render.departures import departure_text from search.scope import provider +from search.scope.basis import BASIS_RULE from search.scope.build import build_part_scope from search.scope.compile import Refusal from search.scope.edits import ScopeTableError, apply_binding from search.scope.provider import ( _not_offered, _settled, + elect_latest_window, inclusive_reach, provider_rule, settle_provider, ) from search.scope.rules import AXIS_RULES from search.scope.state import AxisRecord, ScopeState -from search.scope.tables import ScopeFacts +from search.scope.tables import AxisTable, ScopeFacts from series_coverage.schema import ABSENT_ENTITY, FacetMetadata, RoleNames from tests.engine._fake_coverage import ( FakeCoverageService, @@ -72,6 +81,7 @@ from tests.engine._fake_sdmx import FakeSdmxClient from tests.engine._harness import offline_resolve from tests.fixtures import FakeGraph +from tests.origin_catalogue.fake_catalogue import listing from tests.scope import test_basis as basis from tests.scope._axes import carried, explicit, switch from tests.scope.test_basis import _crs_member, _graph, _PickingLLM @@ -2182,6 +2192,12 @@ def test_a_record_of_the_bilateral_scope_strips_the_composite_the_part_holds(): # years both lines hold for the part. _DAC2A = "provenance/OECDDAC2A" _DAC2A_INCLUSIVE = "provenance/ONEDAC2aBilateralAndImputedMultilateral" +_DAC2A_BILATERAL = "provenance/ONEDAC2aBilateral" +_EC = "EuropeanCommission" +(_DAC2A_SPLIT,) = BILATERAL_SOURCES +_SEN = "country/SEN" +_ALL_RECIPIENTS = "DACAllRecipients" +_DEVELOPING = "DACDevelopingCountries" _DAC2A_ODA = { "Gross": "Amount_FinancialFlow_Gross_Disbursement_DAC2aAidType206_Nominal", "Net": "Amount_FinancialFlow_Net_Disbursement_DAC2aAidType206_Nominal", @@ -2190,11 +2206,22 @@ def test_a_record_of_the_bilateral_scope_strips_the_composite_the_part_holds(): def _dac2a_identity( - netting: str, *, window: tuple[int, int] = (2015, 2020), donor: str = _USA + netting: str, + *, + window: tuple[int, int] = (2015, 2020), + donor: str | None = _USA, + destination: str | None = _KEN, ) -> BaseIdentity: - """DAC2a ODA from `donor` to Kenya on the `netting` basis over `window`.""" + """DAC2a ODA from `donor`, every donor where it is None, to `destination`, an open + destination where it is None, on the `netting` basis over `window`.""" + entities = { + role: [dcid] + for role, dcid in (("origin", donor), ("destination", destination)) + if dcid is not None + } return BaseIdentity.model_validate( { + **({"open_roles": ["destination"]} if destination is None else {}), "identity_version": "spec1", "shape": { "kind": "statvar_schema", @@ -2214,20 +2241,30 @@ def _dac2a_identity( }, {"property": "nettingBasis", "binding": {"state": "value", "ref": netting}}, ], - "entities": {"origin": [donor], "destination": [_KEN]}, + "entities": entities, "requested_window": {"kind": "range", "start_year": window[0], "end_year": window[1]}, } ) -def _dac2a_facts(imputed_years: Sequence[int], *, donor: str = _USA) -> ScopeFacts: +def _dac2a_facts( + imputed_years: Sequence[int], *, donor: str = _USA, destinations: Sequence[str] = (_KEN,) +) -> ScopeFacts: """The fixture code index and a coverage snapshot holding DAC2a's gross and net ODA from - `donor` to Kenya in every year from 2005 to 2024, and its imputed multilateral ODA in - `imputed_years`.""" + `donor` to each of `destinations` in every year from 2005 to 2024, and its imputed + multilateral ODA in `imputed_years`.""" members = (*_DAC2A_ODA.values(), _DAC2A_IMPUTED) rows = [ - *((dcid, "dac2a", donor, _KEN, list(_ALL_YEARS)) for dcid in _DAC2A_ODA.values()), - *([(_DAC2A_IMPUTED, "dac2a", donor, _KEN, list(imputed_years))] if imputed_years else []), + *( + (dcid, "dac2a", donor, destination, list(_ALL_YEARS)) + for dcid in _DAC2A_ODA.values() + for destination in destinations + ), + *( + (_DAC2A_IMPUTED, "dac2a", donor, destination, list(imputed_years)) + for destination in destinations + if imputed_years + ), ] index = build_coverage_index( rows, @@ -2247,18 +2284,26 @@ def _dac2a_facts(imputed_years: Sequence[int], *, donor: str = _USA) -> ScopeFac def test_a_dac2a_total_offers_bilateral_plus_its_own_imputed_line_on_gross_and_net( netting: str, label: str ): - """A donor's DAC2a ODA holds the bilateral scope by default and offers the inclusive - scope from DAC2a's own imputed line on gross and on net flows, a net part adding the - gross imputed line as published under a label that says so. A switch binds the DAC2a - composite and keeps the part's basis, and a switch back compiles to the part's spec_id.""" + """A donor's DAC2a ODA bound to no composite holds total ODA, the contributions scope, + and offers the inclusive scope from DAC2a's own imputed line on gross and on net flows, a + net part adding the gross imputed line as published under a label that says so. A switch + binds the DAC2a composite and keeps the part's basis. A switch back to the bilateral + scope reads the bilateral destinations, and one to total ODA compiles to the part's + spec_id.""" facts = _dac2a_facts(_ALL_YEARS) - identity = _dac2a_identity(netting) + # An open destination: over Kenya alone total ODA reads the bilateral figures, so the + # composite offers no switch to it (`test_a_dac2a_composite_over_bilateral_destinations_ + # offers_no_total_oda`). + identity = _dac2a_identity(netting, destination=None) table = provider_rule(identity, None, facts) record = settle_provider(identity, _total(), facts) assert table is not None and record is not None - assert (record.active, record.rule) == ("provider:bilateral", "bilateral_recipient_total") + assert (record.active, record.rule) == ( + "provider:contributions", + "contribution_recipient_total", + ) inclusive = table.option("provider:inclusive") assert inclusive is not None assert inclusive.availability.status == "available" @@ -2273,12 +2318,19 @@ def test_a_dac2a_total_offers_bilateral_plus_its_own_imputed_line_on_gross_and_n assert (compiled.facets or {})["provenance"] == IdentityBindingValue(ref=_DAC2A_INCLUSIVE) assert _slot_ref(compiled, "nettingBasis") == netting held = build_part_scope(base=compiled, entries=[], records={}, facts=facts) - back = build_part_scope( + bilateral = build_part_scope( base=compiled, entries=[switch("provider:bilateral")], records={}, facts=facts ) - assert isinstance(held, PartScope) and isinstance(back, PartScope) + back = build_part_scope( + base=compiled, entries=[switch("provider:contributions")], records={}, facts=facts + ) + assert isinstance(held, PartScope) + assert isinstance(bilateral, PartScope) and isinstance(back, PartScope) (choice,) = (c for c in held.choices if c.axis == "provider") assert choice.active == "provider:inclusive" + assert (bilateral.compiled.identity.facets or {})["provenance"] == IdentityBindingValue( + ref=_DAC2A_BILATERAL + ) assert back.compiled.spec_id == identity_spec_id(identity) @@ -2393,7 +2445,7 @@ def test_the_dac2a_inclusive_scope_serves_no_donor_total_holding_a_multilateral( _RULE_OF = { - "provider:bilateral": "bilateral_recipient_total", + "provider:bilateral": "bilateral_destinations", "provider:inclusive": "bilateral_plus_imputed", } @@ -2401,7 +2453,7 @@ def test_the_dac2a_inclusive_scope_serves_no_donor_total_holding_a_multilateral( @pytest.mark.parametrize( ("message", "active", "source"), [ - pytest.param(None, "provider:bilateral", None, id="default"), + pytest.param(None, "provider:bilateral", _DAC2A_BILATERAL, id="default"), pytest.param( "aid from the United States to Kenya in 2019 including imputed multilateral", "provider:inclusive", @@ -2413,9 +2465,9 @@ def test_the_dac2a_inclusive_scope_serves_no_donor_total_holding_a_multilateral( def test_a_dac2a_recipient_answer_serves_its_own_inclusive_composite_when_asked( monkeypatch, message: str | None, active: str, source: str | None ): - """A donor's aid to a recipient, which DAC2a serves on net flows, holds the bilateral - scope by default. Asked to include imputed multilateral aid, the fresh search answers - DAC2a's own composite on the same net basis, not ONE's imputed table.""" + """A donor's aid to a recipient, which DAC2a serves on net flows, reads its bilateral + destinations by default. Asked to include imputed multilateral aid, the fresh search + answers DAC2a's own composite on the same net basis, not ONE's imputed table.""" members = (*basis._DAC2A_MEMBERS, _DAC2A_IMPUTED) patch_coverage( monkeypatch, @@ -2504,6 +2556,11 @@ def test_a_crs_answer_states_its_components_sum_only_on_the_composite( def _resubmitted(identity: BaseIdentity, scope_entries: list[Entry] | None) -> Spec: """`identity` posted on a dataset resubmit with `scope_entries`.""" + return _resubmitted_part(identity, scope_entries).spec + + +def _resubmitted_part(identity: BaseIdentity, scope_entries: list[Entry] | None) -> DefinitePart: + """The part answering `identity` posted on a dataset resubmit with `scope_entries`.""" row = basis._CELLS["recipient-all-sectors"] members = basis._ALL_MEMBERS graph = FakeGraph( @@ -2511,6 +2568,14 @@ def _resubmitted(identity: BaseIdentity, scope_entries: list[Entry] | None) -> S _USA: {"label": "United States", "type": "Country"}, _KEN: {"label": "Kenya", "type": "Country"}, _OFFICIAL_DONORS: {"label": "Official Donors, Total", "type": "Organization"}, + _ALL_RECIPIENTS: {"label": "All Recipients, Total", "type": "Organization"}, + _DEVELOPING: {"label": "Developing Countries, Total", "type": "Organization"}, + "DACAllMultilateralRecipients": { + "label": "Multilateral Organisations, Total", + "type": "Organization", + }, + "DACOtherUnitedNations": {"label": "Other UN", "type": "Organization"}, + "EuropeanCommission": {"label": "European Commission", "type": "Organization"}, **{dcid: basis._member_node(dcid) for dcid in members}, }, detect={row.variable: {"svs": list(members), "cosine_scores": [0.9] * len(members)}}, @@ -2533,7 +2598,27 @@ def _resubmitted(identity: BaseIdentity, scope_entries: list[Entry] | None) -> S assert isinstance(response, Resolution), response (part,) = response.parts assert isinstance(part, DefinitePart), part - return part.spec + return part + + +def _posted_dac2a(monkeypatch: pytest.MonkeyPatch, source: str, destination: str) -> DefinitePart: + """The part answering DAC2a ODA from the United States to `destination`, bound to the + composite `source` and posted on a dataset resubmit with no scope entries.""" + members = (*basis._DAC2A_MEMBERS, _DAC2A_IMPUTED) + patch_coverage( + monkeypatch, + [ + (dcid, "dac2a", _USA, held, list(range(2015, 2021))) + for dcid in members + for held in (destination, _DEVELOPING) + ], + role_names=dict.fromkeys(members, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + identity = _pinned( + _dac2a_identity("Gross", destination=destination), {"state": "value", "ref": source} + ) + return _resubmitted_part(identity, None) @pytest.mark.parametrize( @@ -2553,8 +2638,9 @@ def test_an_inclusive_composite_over_providers_it_counts_twice_serves_the_bilate ): """DAC2a's inclusive composite adds the named donors' imputed share of multilateral spending. Posted over a total that holds the multilaterals, or over every donor, it would - count their outflows twice, so the resubmit serves the bilateral line and reports the - inclusive scope dropped as a carried entry the part cannot take.""" + count their outflows twice, so the resubmit serves the bilateral line, DAC2a's bilateral + destinations as a fresh search reads them, and reports the inclusive scope dropped as a + carried entry the part cannot take.""" spec = _dac2a_inclusive_answer(monkeypatch) base = _scope(spec).base.identity others = {role: refs for role, refs in base.entities.items() if role != "origin"} @@ -2585,7 +2671,7 @@ def test_an_inclusive_composite_over_providers_it_counts_twice_serves_the_bilate served = _resubmitted(posted, entries) - assert _source(served) is None + assert _source(served) == _DAC2A_BILATERAL assert served.definition.identity.entities.get("origin") == origin scope = served.resolved.scope_choices assert scope is not None @@ -2607,13 +2693,14 @@ def test_an_inclusive_composite_over_a_donor_keeps_it_on_resubmit(monkeypatch): def test_a_switch_to_the_dac2a_inclusive_scope_states_its_components_sum(monkeypatch): """A plan switch posts the bilateral answer's scope base with the inclusive option as an - explicit entry. The resubmit answers DAC2a's composite and states that a groupby sum adds - its two lines, as the fresh answer does.""" + explicit entry. The fresh answer reads DAC2a's bilateral destinations, a stacked + composite of one line, which a groupby sum serves as that line. The resubmit answers + DAC2a's inclusive composite and states that a groupby sum adds its two lines.""" _dac2a_inclusive_answer(monkeypatch) row = basis._CELLS["recipient-all-sectors"] bilateral = basis._ask(row.variable, time=row.time, recipient=row.recipient) - assert _source(bilateral) is None - assert bilateral.resolved.stat_vars.sums_components is False + assert _source(bilateral) == _DAC2A_BILATERAL + assert bilateral.resolved.stat_vars.sums_components is True scope = _scope(bilateral) # The default bilateral scope holds no entry, so the switch adds the provider's. assert all(entry.axis != "provider" for entry in scope.entries) @@ -2624,3 +2711,729 @@ def test_a_switch_to_the_dac2a_inclusive_scope_states_its_components_sum(monkeyp assert _source(served) == _DAC2A_INCLUSIVE assert served.resolved.stat_vars.served_count == 2 assert served.resolved.stat_vars.sums_components is True + + +# DAC2a's total ODA (aid type 206) counts a donor's bilateral flows to countries and regions and +# its core contributions to the multilateral agencies DAC2a lists as recipients. A fresh part +# reads the bilateral destinations alone (`provenance/ONEDAC2aBilateral`) where it names no +# destination or only bilateral ones; a part naming an agency or all recipients reads total ODA. + + +def _fresh_scope(identity: BaseIdentity, facts: ScopeFacts, *entries: Entry) -> PartScope | None: + """The scope a fresh search states for an answer of `identity` whose request stated + `entries` and asked for a total.""" + return decide_scope( + identity_spec_id(identity), + identity, + selection=ScopeState(entries=entries, total=True), + facts=facts, + ).scope + + +def _provenance(identity: BaseIdentity) -> str | None: + pin = (identity.facets or {}).get("provenance") + return pin.ref if isinstance(pin, IdentityBindingValue) else None + + +def test_a_fresh_dac2a_part_switched_to_gross_from_its_base_keeps_its_bilateral_reading(): + """A fresh part of France's DAC2a ODA over every recipient settles on the bilateral + destinations by default. A switch from net to gross posted over the scope's base with + its entries still reads the composite, so the agencies stay out: the default is not an + entry, so the base must carry the composite.""" + facts = _dac2a_facts(_ALL_YEARS, donor=_FRA) + identity = _dac2a_identity("Net", donor=_FRA, destination=None) + + scope = _fresh_scope(identity, facts) + assert scope is not None + assert _provenance(scope.compiled.identity) == _DAC2A_BILATERAL + entries = [e for e in scope.entries if e.axis != "measure_basis"] + + switched = build_part_scope( + base=scope.base.identity, + entries=[*entries, switch("measure_basis:gross")], + records={}, + facts=facts, + ) + + assert isinstance(switched, PartScope) + assert _provenance(switched.compiled.identity) == _DAC2A_BILATERAL + assert _slot_ref(switched.compiled.identity, "nettingBasis") == "Gross" + + +def test_a_fresh_dac2a_part_at_a_country_offers_no_switch_to_total_oda(): + """Over a country DAC2a's total ODA holds no core contribution, so a fresh part settled + on its bilateral destinations offers no switch to it.""" + facts = _dac2a_facts(_ALL_YEARS) + identity = _dac2a_identity("Net") + + scope = _fresh_scope(identity, facts) + + assert scope is not None + assert _provenance(scope.compiled.identity) == _DAC2A_BILATERAL + (choice,) = (c for c in scope.choices if c.axis == "provider") + total = next(o for o in choice.options if o.option_id == "provider:contributions") + assert (total.availability.code, total.availability.detail) == ( + "provider_not_offered", + {"why": "destination_bilateral"}, + ) + + +@pytest.mark.parametrize("destination", [_KEN, None], ids=["kenya", "open"]) +def test_bilateral_plus_multilateral_on_dac2a_reads_the_inclusive_composite( + destination: str | None, +): + """Words summing the bilateral and multilateral lines read DAC2a's inclusive composite, + its bilateral destinations plus its imputed multilateral line, at a country and over + every recipient alike. DAC2a's total ODA holds the contributions scope too, but its + core contributions are recipients' rows, so the words do not name it, as they name + DAC1's total ODA.""" + facts = _dac2a_facts(_ALL_YEARS) + identity = _dac2a_identity("Net", destination=destination) + + scope = decide_scope( + identity_spec_id(identity), + identity, + selection=ScopeState( + entries=(explicit("provider:inclusive"),), total=True, sums_providers=True + ), + facts=facts, + ).scope + + assert scope is not None + assert _provenance(scope.compiled.identity) == _DAC2A_INCLUSIVE + (choice,) = (c for c in scope.choices if c.axis == "provider") + assert (choice.active, choice.provenance) == ("provider:inclusive", "explicit") + assert scope.dropped_entries == [] + + +@pytest.mark.parametrize( + ("donor", "destination", "entries", "source", "choice"), + [ + pytest.param( + _USA, + None, + (), + _DAC2A_BILATERAL, + ("provider:bilateral", "bilateral_destinations"), + id="top-recipients", + ), + pytest.param( + _USA, + _SEN, + (), + _DAC2A_BILATERAL, + ("provider:bilateral", "bilateral_destinations"), + id="senegal", + ), + pytest.param(None, None, (), _DAC2A_BILATERAL, None, id="all-donors"), + # With no provider axis no table settles a stated entry, so the default serves it. + pytest.param( + None, + None, + (explicit("provider:contributions"),), + None, + None, + id="all-donors-stated-contributions", + ), + pytest.param( + None, + None, + (carried("provider:contributions"),), + None, + None, + id="all-donors-carried-contributions", + ), + pytest.param( + None, + None, + (explicit("provider:bilateral"),), + _DAC2A_BILATERAL, + None, + id="all-donors-stated-bilateral", + ), + pytest.param( + _USA, + None, + (explicit("provider:contributions"),), + None, + ("provider:contributions", "contribution_recipient_total"), + id="stated-contributions", + ), + pytest.param( + _USA, + None, + (explicit("provider:inclusive"),), + _DAC2A_INCLUSIVE, + ("provider:inclusive", "bilateral_plus_imputed"), + id="stated-inclusive", + ), + pytest.param( + _USA, + _EC, + (), + None, + ("provider:contributions", "contribution_recipient_total"), + id="european-commission", + ), + pytest.param( + _USA, + _ALL_RECIPIENTS, + (), + None, + ("provider:contributions", "contribution_recipient_total"), + id="all-recipients", + ), + ], +) +def test_a_fresh_dac2a_total_reads_its_bilateral_destinations_unless_it_names_an_agency( + donor: str | None, + destination: str | None, + entries: tuple[Entry, ...], + source: str | None, + choice: tuple[str, str] | None, +): + """A fresh DAC2a ODA part naming no recipient, or a country, reads the composite that + keeps the bilateral destinations alone, with or without a provider axis, as for a + ranking over every donor; a stated inclusive scope still reads DAC2a's own imputed line. + A part naming a multilateral agency, or all recipients, holds total ODA, and so does a + part whose stated or carried entry asks for it. On a part with no provider axis such an + entry, and a stated bilateral one the default serves, is not reported as dropped.""" + facts = _dac2a_facts(_ALL_YEARS, destinations=(destination or _KEN,)) + identity = _dac2a_identity("Net", donor=donor, destination=destination) + # A binder settled the basis, so an explicit entry on an axis the part lacks is the + # part's own, and is reported where it is not applied (`ScopeState.attached`). + measure_basis = AxisRecord( + axis="measure_basis", + rule=BASIS_RULE, + active="measure_basis:net", + provenance="default", + ) + + scope = decide_scope( + identity_spec_id(identity), + identity, + selection=ScopeState(entries=entries, records={"measure_basis": measure_basis}, total=True), + facts=facts, + ).scope + + # A part that reads total ODA with no axis to state a choice on states no scope. + assert scope is not None or (source, choice) == (None, None) + compiled = identity if scope is None else scope.compiled.identity + assert _provenance(compiled) == source + assert compiled.entities == identity.entities + held = ( + [] if scope is None else [(c.active, c.rule) for c in scope.choices if c.axis == "provider"] + ) + assert held == ([] if choice is None else [choice]) + assert scope is None or scope.dropped_entries == [] + + +@pytest.mark.parametrize("netting", ["Gross", "Net"]) +def test_a_dac2a_total_to_an_agency_offers_no_bilateral_reading_and_says_why(netting: str): + """DAC2a ODA to the European Commission is the donor's core contribution to it: the + part holds total ODA, and the bilateral and inclusive scopes are unavailable under a + code naming the agency, so a stated bilateral scope settles on total ODA.""" + facts = _dac2a_facts(_ALL_YEARS, destinations=(_EC,)) + identity = _dac2a_identity(netting, destination=_EC) + + table = provider_rule(identity, None, facts) + record = settle_provider( + identity, ScopeState(entries=(explicit("provider:bilateral"),), total=True), facts + ) + + assert table is not None and record is not None + assert table.held == "provider:contributions" + assert record.active == "provider:contributions" + for option_id in ("provider:bilateral", "provider:inclusive"): + option = table.option(option_id) + assert option is not None + assert ( + option.availability.status, + option.availability.code, + option.availability.detail, + ) == ("unavailable", "multilateral_destination", {"destination": _EC}) + + +def test_a_dac2a_total_to_all_recipients_reads_developing_countries_when_cut(): + """DAC2a ODA to all recipients holds total ODA. Its bilateral and inclusive scopes read + the DAC's total of developing countries in its place, the bilateral side of all + recipients, as their labels say, and a switch back reads all recipients again.""" + facts = _dac2a_facts(_ALL_YEARS, destinations=(_ALL_RECIPIENTS, _DEVELOPING)) + identity = _dac2a_identity("Gross", destination=_ALL_RECIPIENTS) + + table = provider_rule(identity, None, facts) + + assert table is not None and table.held == "provider:contributions" + for option_id, source, label in ( + ("provider:bilateral", _DAC2A_BILATERAL, _DAC2A_SPLIT.label), + ("provider:inclusive", _DAC2A_INCLUSIVE, OPTION_LABELS.provider_inclusive), + ): + option = table.option(option_id) + assert option is not None + assert option.availability.status == "available" + assert option.label == OPTION_LABELS.provider_rebound.format( + label=label, rebound="developing countries", spanning="all recipients" + ) + switched = build_part_scope( + base=identity, entries=[switch(option_id)], records={}, facts=facts + ) + assert isinstance(switched, PartScope) + compiled = switched.compiled.identity + assert (_provenance(compiled), compiled.entities["destination"]) == (source, [_DEVELOPING]) + back = build_part_scope( + base=switched.base.identity, + entries=[switch("provider:contributions")], + records={}, + facts=facts, + ) + assert isinstance(back, PartScope) + assert back.compiled.spec_id == identity_spec_id(identity) + + +@pytest.mark.parametrize("destination", [None, _KEN]) +def test_a_resubmitted_bare_dac2a_total_reads_as_total_oda(destination: str | None): + """A stored DAC2a ODA identity binding no composite reads back as total ODA, the + contributions scope, under its own spec_id, and offers the bilateral destinations and + the inclusive scope as switches.""" + facts = _dac2a_facts(_ALL_YEARS) + identity = _dac2a_identity("Net", destination=destination) + + built = build_part_scope(base=identity, entries=[], records={}, facts=facts) + + assert isinstance(built, PartScope) + (choice,) = (c for c in built.choices if c.axis == "provider") + assert (choice.active, choice.rule) == ( + "provider:contributions", + "contribution_recipient_total", + ) + assert built.compiled.spec_id == identity_spec_id(identity) + bilateral = next(o for o in choice.options if o.option_id == "provider:bilateral") + assert (bilateral.availability.status, bilateral.binding) == ( + "available", + ProviderBinding(scope="bilateral", provenance=_DAC2A_BILATERAL), + ) + + +@pytest.mark.parametrize( + ("destination", "recipient"), + [ + pytest.param(_EC, "party", id="agency"), + pytest.param("DACAllMultilateralRecipients", "total", id="total-of-agencies"), + pytest.param("DACOtherUnitedNations", "residual", id="unnamed-agencies"), + ], +) +def test_a_dac2a_answer_to_an_agency_says_its_figures_are_core_contributions( + monkeypatch, destination: str, recipient: str +): + """US aid to a multilateral recipient reads total ODA, and the answer says the figures are + core contributions, worded by what the recipient is: an agency, the DAC's total over the + agencies, or its line for the agencies it does not name.""" + patch_coverage( + monkeypatch, + [(dcid, "dac2a", _USA, destination, list(_ALL_YEARS)) for dcid in basis._DAC2A_MEMBERS], + role_names=dict.fromkeys(basis._DAC2A_MEMBERS, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + + part = _resubmitted_part(_dac2a_identity("Net", destination=destination), []) + + assert _source(part.spec) is None + shown = get_index().parties[destination].label + fact = DepartureFact( + code="core_contribution_read", binding_ref="where:", shown=shown, recipient=recipient + ) + assert [(d.code, d.shown, d.text) for d in part.departures] == [ + ( + "core_contribution_read", + shown, + departure_text(fact, family=None, shows_data=True), + ) + ] + + +@pytest.mark.parametrize( + ("destination", "recipient"), + [ + pytest.param(_EC, "party", id="agency"), + pytest.param("DACAllMultilateralRecipients", "total", id="total-of-agencies"), + ], +) +def test_every_donor_s_dac2a_aid_to_an_agency_says_its_figures_are_core_contributions( + monkeypatch, destination: str, recipient: str +): + """DAC2a ODA from every donor to a multilateral recipient, as in "top donors to UNICEF", + has no provider axis, since it names no donor, and still says its figures are core + contributions: the departure reads the destination's side, not a provider option.""" + patch_coverage( + monkeypatch, + [(dcid, "dac2a", _USA, destination, list(_ALL_YEARS)) for dcid in basis._DAC2A_MEMBERS], + role_names=dict.fromkeys(basis._DAC2A_MEMBERS, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + + part = _resubmitted_part(_dac2a_identity("Net", donor=None, destination=destination), []) + + assert _source(part.spec) is None + assert part.spec.resolved.scope_choices is None or not any( + c.axis == "provider" for c in part.spec.resolved.scope_choices.choices + ) + shown = get_index().parties[destination].label + fact = DepartureFact( + code="core_contribution_read", binding_ref="where:", shown=shown, recipient=recipient + ) + assert [(d.code, d.shown, d.text) for d in part.departures] == [ + ( + "core_contribution_read", + shown, + departure_text(fact, family=None, shows_data=True), + ) + ] + + +_SAU = "country/SAU" + + +@pytest.mark.parametrize( + ("countries", "status", "active", "source"), + [ + # Saudi Arabia reports its total ODA with no breakdown by recipient: its rows sit at + # the DAC's totals alone, so its recipients cannot be ranked without the agencies. + pytest.param((), "unavailable", "provider:contributions", None, id="no-breakdown"), + pytest.param( + (2015, 2016, 2017, 2018, 2019, 2020), + "available", + "provider:bilateral", + _DAC2A_BILATERAL, + id="breakdown-every-year", + ), + ], +) +def test_a_donor_with_no_breakdown_by_recipient_keeps_total_oda( + countries: tuple[int, ...], status: str, active: str, source: str | None +): + """A donor's top recipients read the bilateral destinations only in years where a + country, region or unallocated line holds the donor's rows. A donor whose rows sit at the + DAC's totals alone keeps total ODA, and the bilateral option is unavailable for want of + coverage.""" + members = (*_DAC2A_ODA.values(), _DAC2A_IMPUTED) + rows = [ + *( + (dcid, "dac2a", _SAU, total, list(_ALL_YEARS)) + for dcid in _DAC2A_ODA.values() + for total in (_DEVELOPING, _ALL_RECIPIENTS) + ), + *( + (dcid, "dac2a", _SAU, _KEN, list(countries)) + for dcid in _DAC2A_ODA.values() + if countries + ), + ] + index = build_coverage_index( + rows, + role_names=dict.fromkeys(members, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + facts = ScopeFacts(index=get_index(), coverage=FakeCoverageService(index).snapshot()) + identity = _dac2a_identity("Net", donor=_SAU, destination=None) + + scope = _fresh_scope(identity, facts) + + assert scope is not None + assert _provenance(scope.compiled.identity) == source + (choice,) = (c for c in scope.choices if c.axis == "provider") + assert choice.active == active + bilateral = next(o for o in choice.options if o.option_id == "provider:bilateral") + assert bilateral.availability.status == status + assert bilateral.availability.code == (None if status == "available" else "option_not_covered") + + +@pytest.mark.parametrize( + ("country_years", "status", "year"), + [ + # 2023 is held by the DAC's total of developing countries alone, which an open read of + # the composite leaves out, so the composite's latest year is 2022. + pytest.param(range(2015, 2023), "partial", 2022, id="totals-alone-newest"), + pytest.param(range(2015, 2024), "available", 2023, id="country-newest"), + ], +) +def test_the_latest_bilateral_reading_elects_the_year_its_composite_reads( + country_years: range, status: str, year: int +): + """The bilateral option of a part asking for the latest year elects its year over the + rows the composite's population holds, as a `latest` read of the composite does + (`provider.elect_latest_window`), so the two name one year. A year the DAC's totals + alone hold is no year of the composite, so the option is partial over the year before.""" + members = (*_DAC2A_ODA.values(), _DAC2A_IMPUTED) + rows = [ + *( + (dcid, "dac2a", _USA, _DEVELOPING, list(range(2015, 2024))) + for dcid in _DAC2A_ODA.values() + ), + *((dcid, "dac2a", _USA, _KEN, list(country_years)) for dcid in _DAC2A_ODA.values()), + ] + index = build_coverage_index( + rows, + role_names=dict.fromkeys(members, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + facts = ScopeFacts(index=get_index(), coverage=FakeCoverageService(index).snapshot()) + document = _dac2a_identity("Net", destination=None).model_dump(mode="json", exclude_none=True) + identity = BaseIdentity.model_validate({**document, "requested_window": {"kind": "latest"}}) + composite = _pinned(identity, {"state": "value", "ref": _DAC2A_BILATERAL}) + + table = provider_rule(identity, None, facts) + elected = elect_latest_window(composite, facts).requested_window + + assert table is not None + option = table.option("provider:bilateral") + assert option is not None + assert elected == IdentityWindowRange(start_year=year, end_year=year) + assert option.availability.status == status + assert option.binding == ProviderBinding( + scope="bilateral", provenance=_DAC2A_BILATERAL, window=(year, year) + ) + + +_UNCLASSIFIED = "DACRecipientOnNoSide" +"""A DAC2a destination the code index holds no row for, so no side holds it.""" + + +def test_a_dac2a_total_naming_a_destination_on_no_side_keeps_total_oda(): + """A donor's DAC2a rows at a destination no side declaration places leave the bilateral + population unavailable (`code_index.composites.population_check`): its members would + leave the destination out without a word, and the mint refuses the composite. A fresh + part keeps total ODA, and its bilateral option is not offered for that reason.""" + facts = _dac2a_facts(_ALL_YEARS, destinations=(_KEN, _UNCLASSIFIED)) + identity = _dac2a_identity("Net", destination=None) + + scope = _fresh_scope(identity, facts) + + assert scope is not None + assert _provenance(scope.compiled.identity) is None + (choice,) = (c for c in scope.choices if c.axis == "provider") + assert choice.active == "provider:contributions" + bilateral = next(o for o in choice.options if o.option_id == "provider:bilateral") + assert ( + bilateral.availability.status, + bilateral.availability.code, + bilateral.availability.detail, + ) == ("unavailable", "provider_not_offered", {"why": "destination_unclassified"}) + + +# The shape of France's 2023 DAC2a answer: total ODA at two countries and an unallocated line +# (bilateral), at an agency and the DAC's unnamed multilateral recipient (multilateral), and +# at the DAC's totals of all recipients and of all multilateral recipients. +_FRANCE_SHAPED_DESTINATIONS = ( + _SEN, + _KEN, + "DACAfricaUnallocated", + _EC, + "DACOtherMultilateralRecipient", + _ALL_RECIPIENTS, + "DACAllMultilateralRecipients", +) + + +@pytest.mark.parametrize( + ("source", "recipients", "rows", "series"), + [ + # Total ODA counts every destination but the two totals. + pytest.param(None, 5, 42, 7, id="contributions"), + # The bilateral destinations alone: the two countries and the unallocated line. + pytest.param(_DAC2A_BILATERAL, 3, 18, 3, id="bilateral"), + ], +) +def test_a_dac2a_bilateral_answer_counts_the_recipients_its_file_holds( + monkeypatch, source: str | None, recipients: int, rows: int, series: int +): + """An open-destination DAC2a answer read to its bilateral destinations counts, in its + sentence, its destination fact and its download estimate, the recipients and rows the + mint keeps, leaving out the agencies and totals that total ODA still counts.""" + patch_coverage( + monkeypatch, + [ + (dcid, "dac2a", _USA, destination, list(range(2015, 2021))) + for dcid in basis._DAC2A_MEMBERS + for destination in _FRANCE_SHAPED_DESTINATIONS + ], + role_names=dict.fromkeys(basis._DAC2A_MEMBERS, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + identity = _dac2a_identity("Net", destination=None) + if source is not None: + document = identity.model_dump(mode="json", exclude_none=True) + document["facets"] = {"provenance": {"state": "value", "ref": source}} + identity = BaseIdentity.model_validate(document) + + # The deployed catalogue lists DAC2a as "OECD DAC2A"; a sentence names its declared label. + with bound_catalogue(listing({_DAC2A: "OECD DAC2A"})): + spec = _resubmitted(identity, []) + resolved = spec.resolved + + assert _source(spec) == source + assert [(f.role, f.member_count) for f in resolved.role_axis_facts or ()] == [ + ("destination", recipients) + ] + assert f"to {recipients} recipients" in (resolved.rendered_sentence or "") + assert resolved.download is not None + assert (resolved.download.rows, resolved.download.series) == (rows, series) + # The bilateral reading cites DAC2a as total ODA does, by its declared label. + assert (resolved.rendered_sentence or "").endswith("OECD DAC2a.") + + +def test_each_table_names_its_provider_options_by_what_they_count(): + """The CRS and DAC1 name the bilateral and contributions scopes in their general words. + DAC2a's bilateral option names the destinations it keeps and the core contributions it + leaves out, and its own total reads as total ODA, since it counts every destination.""" + crs = provider_rule(_bilateral_identity(), None, _facts()) + dac1 = provider_rule( + _dac1_identity(_DAC1_TOTAL_ODA, (_USA,)), None, _dac1_facts(*_DAC1_MEMBERS_BY_CODE) + ) + dac2a = provider_rule(_dac2a_identity("Net", destination=None), None, _dac2a_facts(_ALL_YEARS)) + + def labels(table: AxisTable | None) -> dict[str, str]: + assert table is not None + return {option.option_id: option.label for option in table.options} + + general = { + "provider:bilateral": OPTION_LABELS.provider_bilateral, + "provider:contributions": OPTION_LABELS.provider_contributions, + } + assert labels(crs)["provider:bilateral"] == general["provider:bilateral"] + assert {k: v for k, v in labels(dac1).items() if k in general} == general + assert {k: v for k, v in labels(dac2a).items() if k in general} == { + "provider:bilateral": _DAC2A_SPLIT.label, + "provider:contributions": _DAC2A_SPLIT.total_label, + } + assert _DAC2A_SPLIT.label != OPTION_LABELS.provider_bilateral + + +@pytest.mark.parametrize( + ("destination", "held", "status"), + [ + # Over Kenya alone total ODA holds no core contribution: it reads the bilateral + # figures, so a composite reading offers no switch to it. + pytest.param(_KEN, _DAC2A_BILATERAL, "unavailable", id="country-bilateral"), + pytest.param(_KEN, _DAC2A_INCLUSIVE, "unavailable", id="country-inclusive"), + # An open destination still holds the agencies' rows. + pytest.param(None, _DAC2A_BILATERAL, "available", id="open"), + pytest.param(None, _DAC2A_INCLUSIVE, "available", id="open-inclusive"), + # A bare total holds the option itself. + pytest.param(_KEN, None, "available", id="country-total-held"), + ], +) +def test_a_dac2a_composite_over_bilateral_destinations_offers_no_total_oda( + destination: str | None, held: str | None, status: str +): + """The total ODA switch from a DAC2a composite is not offered where every destination the + part names is on the bilateral side, since its figures equal the bilateral ones; a part + holding total ODA keeps it, and an open destination offers it.""" + facts = _dac2a_facts(_ALL_YEARS) + identity = _dac2a_identity("Net", destination=destination) + if held is not None: + identity = _pinned(identity, {"state": "value", "ref": held}) + + table = provider_rule(identity, None, facts) + + assert table is not None + option = table.option("provider:contributions") + assert option is not None + assert option.availability.status == status + if status == "unavailable": + assert (option.availability.code, option.availability.detail) == ( + "provider_not_offered", + {"why": "destination_bilateral"}, + ) + + +def test_a_dac2a_total_naming_a_destination_on_no_side_offers_neither_cut_reading(): + """The inclusive composite's bilateral line is cut as the bilateral composite is, so a + destination on no side withholds both readings for the same reason.""" + facts = _dac2a_facts(_ALL_YEARS, destinations=(_KEN, _UNCLASSIFIED)) + + table = provider_rule(_dac2a_identity("Gross", destination=None), None, facts) + + assert table is not None + for option_id in ("provider:bilateral", "provider:inclusive"): + option = table.option(option_id) + assert option is not None + assert ( + option.availability.status, + option.availability.code, + option.availability.detail, + ) == ("unavailable", "provider_not_offered", {"why": "destination_unclassified"}) + + +def test_a_bilateral_composite_that_refuses_its_part_is_not_offered_for_that_reason( + monkeypatch: pytest.MonkeyPatch, +): + """A bilateral composite whose components cannot serve what the part binds is not + offered, naming the slot at fault, as the inclusive option names it; it does not read as + a composite serving no year (`option_not_covered`).""" + served_components = provider.served_components + + def refusing(document, *, facts): + pin = document.get("facets", {}).get("provenance", {}) + if pin.get("ref") == _DAC2A_BILATERAL: + raise CompositeNotServable("refused", ref=_DAC2A_BILATERAL, pointer="/slots/0") + return served_components(document, facts=facts) + + monkeypatch.setattr(provider, "served_components", refusing) + identity = _dac2a_identity("Net", destination=None) + + table = provider_rule(identity, None, _dac2a_facts(_ALL_YEARS)) + + assert table is not None + option = table.option("provider:bilateral") + assert option is not None + assert ( + option.availability.status, + option.availability.code, + option.availability.detail, + ) == ("unavailable", "provider_not_offered", {"why": "dac2aAidType"}) + + +def test_a_fresh_dac2a_scope_reads_its_bilateral_option_once(monkeypatch: pytest.MonkeyPatch): + """The default and the provider table of one fresh part read the same bilateral option + and destination rule, once per scope build.""" + reads: list[str] = [] + read_option, read_rule = provider._read_bilateral_option, provider._destination_rule + + def counted_option(*args): + reads.append("option") + return read_option(*args) + + def counted_rule(*args): + reads.append("rule") + return read_rule(*args) + + monkeypatch.setattr(provider, "_read_bilateral_option", counted_option) + monkeypatch.setattr(provider, "_destination_rule", counted_rule) + + scope = _fresh_scope(_dac2a_identity("Net", destination=None), _dac2a_facts(_ALL_YEARS)) + + assert scope is not None + assert sorted(reads) == ["option", "rule"] + + +def test_a_dac2a_answer_whose_provenance_set_names_a_composite_states_no_core_contribution( + monkeypatch, +): + """A definition naming a composite in a provenance set cannot be read as a published + total (`code_index.composites.composite_ref` refuses it), so the core-contribution + departure reads none rather than raising while the answer's departures are stated.""" + patch_coverage( + monkeypatch, + [(dcid, "dac2a", _USA, _EC, list(_ALL_YEARS)) for dcid in basis._DAC2A_MEMBERS], + role_names=dict.fromkeys(basis._DAC2A_MEMBERS, _FLOW), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + spec = _resubmitted_part(_dac2a_identity("Net", destination=_EC), []).spec + pinned = spec.definition.identity.model_copy( + update={"facets": {"provenance": IdentityBindingSet(refs=[_DAC2A, _DAC2A_BILATERAL])}} + ) + spec = spec.model_copy( + update={"definition": spec.definition.model_copy(update={"identity": pinned})} + ) + + assert departures._core_contribution_facts(spec) == [] diff --git a/uv.lock b/uv.lock index adb4b46b6..82e7d2768 100644 --- a/uv.lock +++ b/uv.lock @@ -726,8 +726,8 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, - { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, + { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, { name = "pyarrow", specifier = ">=24" }, { name = "pydantic", specifier = ">=2.13" }, { name = "rfc8785", specifier = ">=0.1.4" }, @@ -804,11 +804,11 @@ requires-dist = [ { name = "dfo-data-tools", extras = ["generation"], marker = "extra == 'resolve'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'coverage'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'formats'", editable = "packages/dfo-data-tools" }, - { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, - { name = "dfo-spec-kit-components", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, - { name = "dfo-spec-kit-descriptions", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, - { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, - { name = "dfo-spec-kit-provenance-docs", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, + { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit-components", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit-descriptions", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit-provenance-docs", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, { name = "fastapi", marker = "extra == 'surface'", specifier = ">=0.138" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'generation'", specifier = ">=2.55" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'sdmx'", specifier = ">=2.55" }, @@ -834,7 +834,7 @@ provides-extras = ["sdmx", "formats", "surface", "resolve", "generation", "cover [[package]] name = "dfo-spec-kit" version = "0.2.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=2f7462a91408a18ff2bf97872fed735a8c052382#2f7462a91408a18ff2bf97872fed735a8c052382" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } dependencies = [ { name = "rfc8785" }, ] @@ -842,12 +842,12 @@ dependencies = [ [[package]] name = "dfo-spec-kit-client-contract" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=2f7462a91408a18ff2bf97872fed735a8c052382#2f7462a91408a18ff2bf97872fed735a8c052382" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } [[package]] name = "dfo-spec-kit-components" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=2f7462a91408a18ff2bf97872fed735a8c052382#2f7462a91408a18ff2bf97872fed735a8c052382" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } dependencies = [ { name = "dfo-spec-kit-descriptions" }, ] @@ -855,7 +855,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-descriptions" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=2f7462a91408a18ff2bf97872fed735a8c052382#2f7462a91408a18ff2bf97872fed735a8c052382" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } dependencies = [ { name = "rfc8785" }, ] @@ -863,7 +863,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-frame-grammar" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=2f7462a91408a18ff2bf97872fed735a8c052382#2f7462a91408a18ff2bf97872fed735a8c052382" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } dependencies = [ { name = "dfo-spec-kit" }, { name = "pyarrow" }, @@ -873,7 +873,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-provenance-docs" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=2f7462a91408a18ff2bf97872fed735a8c052382#2f7462a91408a18ff2bf97872fed735a8c052382" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } dependencies = [ { name = "markdown-it-py" }, ] @@ -3094,7 +3094,7 @@ requires-dist = [ { name = "dfo-data-tools", extras = ["resolve", "sdmx", "coverage"], marker = "extra == 'graph'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'sdmx'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["surface", "resolve", "coverage", "sdmx"], marker = "extra == 'engine'", editable = "packages/dfo-data-tools" }, - { name = "dfo-spec-kit", marker = "extra == 'engine'", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, + { name = "dfo-spec-kit", marker = "extra == 'engine'", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, { name = "fastapi", marker = "extra == 'api'", specifier = ">=0.138" }, { name = "google-auth", marker = "extra == 'datasets'", specifier = ">=2.34" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'eval'", specifier = ">=2.34" }, @@ -3127,7 +3127,7 @@ lint = [ ] test = [ { name = "dfo-analysis-engine", editable = "packages/dfo-analysis-engine" }, - { name = "dfo-spec-kit-client-contract", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=2f7462a91408a18ff2bf97872fed735a8c052382" }, + { name = "dfo-spec-kit-client-contract", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, { name = "frictionless", specifier = ">=5.19.0" }, { name = "pytest", specifier = ">=8" }, { name = "pytest-split", specifier = ">=0.10.0" }, From 17c78beadbc3267aa9f0b59bb35ad2181959863e Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 14:33:05 +0100 Subject: [PATCH 02/12] fix(search): keep the Data Commons and publisher a composite's tables share A DAC2a total now defaults to the ONEDAC2aBilateral composite, and assemble_composite built its provenance facet with only basis, ref, label and account, so resolved.facets.provenance lost data_commons and source. observations.composite_facet_ref resolves a composite's facet from its component tables' catalogue rows: source and data_commons are kept when every component names the same one, and left out otherwise. Both the series facets and the resolved facet use it. --- src/search/README.md | 4 +- src/search/observations.py | 25 +++++++- src/search/scope/composite.py | 14 +++-- .../test_dac2a_destination_populations.py | 31 +++++++++- tests/engine/test_provenance.py | 60 ++++++++++++++++++- 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/src/search/README.md b/src/search/README.md index 71a716f91..15a798302 100644 --- a/src/search/README.md +++ b/src/search/README.md @@ -23,7 +23,7 @@ A part may name a publisher's series code in `ParsedPart.source_code`, such as t A UN candidate's node facts come from the catalogue's UN tables (`origin_catalogue.graph.OriginGraphClient`), so deriving its shape sends the UN deployment no node request, on a first answer and on a resubmit. A UN shape holds the variables recall found, as any shape does, with the slots and members they carry. The UN deployment is still read for the resolve, observations, SDMX availability and the label of a facet's unit. -Every provenance on the wire names its source and the Data Commons its data comes from (`dfo`, `base` or `un_system`), read with its label from the request's origin catalogue (a provenance the catalogue does not list keeps its graph label and has no source or Data Commons): `resolved.facets.provenance`, each series' `facet.provenance` and `source_refs`, and `trace.resolved_sources`. Each of these provenances also carries `documentation`, the Graph API route of its document, `/api/graph/documentation/{dcid}`, except a `per_series` resolved facet, which names no provenance. Each shape lists the Data Commons holding data for its members in `data_commons`, and a shape whose members all come through one upstream source names it in `upstream`. SDMX serves DFO's role-bearing variables only, so the role-series read raises `UpstreamVariableRoutedToSdmx` before it sends a variable the catalogue places upstream. +Every provenance on the wire names its source and the Data Commons its data comes from (`dfo`, `base` or `un_system`), read with its label from the request's origin catalogue (a provenance the catalogue does not list keeps its graph label and has no source or Data Commons): `resolved.facets.provenance`, each series' `facet.provenance` and `source_refs`, and `trace.resolved_sources`. Each of these provenances also carries `documentation`, the Graph API route of its document, `/api/graph/documentation/{dcid}`, except a `per_series` resolved facet, which names no provenance. A declared composite (`code_index.composites`), which neither the catalogue nor the graph holds, names the source and the Data Commons its component tables' catalogue rows share, leaves out either one where the rows differ or a component table is unlisted, and carries no `documentation`, since the Graph API documents each component table and no composite (`search.observations.composite_facet_ref`). Each shape lists the Data Commons holding data for its members in `data_commons`, and a shape whose members all come through one upstream source names it in `upstream`. SDMX serves DFO's role-bearing variables only, so the role-series read raises `UpstreamVariableRoutedToSdmx` before it sends a variable the catalogue places upstream. ## Submodules @@ -193,7 +193,7 @@ A part starts its pass with the scope its request posts (`stated_scope.part_scop The pass's terminal builds the scope in memory from the records, the release's code index and the request's coverage snapshot (`pipeline._with_scope_choices`). A part with no record carries none. Only the entries the binders applied compile (`AxisRecord.entry`), while the answer's identity holds the option the binder settled (`scope.build.build_part_scope`), so every choice states the option the answer serves. The provider axis is settled over the answer's identity once the binders have run (`scope.provider.settle_provider`). A stated or carried entry applies where its option is available, or for a stated entry partial, and the answer can serve it: the scope it holds, the inclusive composite, or the bilateral scope, which removes that composite (`scope.edits.applies_in_pass`). Otherwise the declared convention applies where the answer can serve it, else the scope the answer holds, else the bilateral scope, else the first available option, so the record holds only an option it marks available. A choice names the convention's rule only where the request matched the convention and the part holds the convention's scope. Otherwise it names the rule of the option it holds (`scope.tables.AxisTable.rules`), which describes that option's own scope: on a part the CRS serves, `bilateral_flows` for the bilateral scope and `bilateral_plus_imputed` for the inclusive one (`code_index.scope_conventions.PROVIDER_SCOPE_RULES`), and on a published total the row's rule. A resubmit matches no convention, so it names the option's own rule. `code_index.scope_conventions.PROVIDER_CONVENTIONS` counts a named donor's total of a CRS sector or recipient part as bilateral flows plus the imputed multilateral share when both tables cover every year the bilateral answer holds. Every convention reads the CRS, so a published donor total matches none and holds the provider scope its DAC1 aid type declares (`code_index.scope_conventions.PUBLISHED_PROVIDER_SCOPES`), under the row's rule, where every provider the part names is a donor: a country, the EU institutions (`DONOR_ORGANISATIONS`), or a total of a group whose membership holds countries alone (the G7, the DAC countries). A part naming any other organisation or total, the DAC's total of the multilaterals among them, or no provider, has no provider record. DAC1's total ODA counts bilateral flows plus core contributions to multilaterals in the year they are paid, so it holds `provider:contributions`, labelled "bilateral plus core contributions to multilaterals", under the rule `contribution_total`. Its `provider:bilateral` option binds DAC1's bilateral ODA (aid type 1015), judged per donor on its basis. A resubmit judges each explicit entry on the identity the other explicit entries' edits produce (`scope.build._requested_tables`), so a request switching to gross and to bilateral flows together judges bilateral ODA on gross and compiles as the two switches do one after the other. A year the total holds is covered when every donor holding the total in that year holds bilateral ODA in it, so a donor holding no total in a year places no demand on it. Where no donor holds the total in any year of the window, the option is available if every donor holds bilateral ODA in the window and unavailable otherwise. Otherwise the option is available where every year the total holds is covered, partial over the covered years where those are fewer, with the latest run of them as the window a switch reads (`SlotBinding.window`), and unavailable where none is, each with `option_not_covered`. On the OECD headline a year counts on the basis the headline reads it on, net flows before 2018 and grant equivalents from 2018. A `latest` read elects one year for the whole pool, so on a part asking for the latest year the option elects bilateral ODA's year among those every donor holding the total in its elected year holds it in, and a switch reads that one year. The option is available where that year is the total's elected year, partial over it, stated in the label, where it is another, and unavailable where the donors share no year or no donor holds the total in its elected year. One coverage scan per table reads every donor's years. Its `provider:inclusive` option is unavailable with `contributions_counted`, since it counts the contributions in place of an imputed share of the multilaterals' spending, with the total's dataset in `detail`. DAC1's bilateral ODA holds `provider:bilateral` under the rule `bilateral_total`, offers `provider:contributions` as total ODA under the same coverage check, and its inclusive option is not offered (`provider_not_offered`, why `dataset`). A pool naming the EU institutions beside an EU member state, or beside a total of a group holding one, has no provider record, since total ODA counts the member's core contributions to the institutions and the institutions' own aid. The members are those the membership policy holds, on any day of the part's window (today for a latest read), for the group `REDISTRIBUTED_PROVIDERS` declares as the institutions' funders, so the United Kingdom counts over any window holding a year before it left in 2020. Where the policy holds no members for that group, every pool with the institutions counts. A pool of the EU institutions and a donor outside the EU has a provider record. A fresh search answers the total it resolved, so a stated "bilateral" on a DAC1 total is reported in `dropped_entries` as `stated_choice_not_applied` with the cause `not_served`, and a switch to it reads bilateral ODA. A stated "including multilateral" is reported with `contributions_counted`. Words naming multilateral ODA state `provider:multilateral` only on a part whose provider table offers it, DAC1's. Elsewhere, as on a part the CRS or DAC2a serves, the entry is dropped before the axis settles (`scope.provider.without_unoffered_multilateral`), so it is neither applied nor reported. The other entries that attach to the part, those on an axis its binders settled and, of those made for a concept, the ones for the concept the axis was asked for, and the stated ones on an axis the part lacks (`ScopeState.attached`), are reported in `dropped_entries`. A carried one is reported as `carried_choice_not_applied` and a stated one as `stated_choice_not_applied`, with the cause (`entry_on_axis`, `axis_absent`, the option's partial or unavailable reason, or `not_served`). -A composite option (the inclusive provider scope, or the OECD headline per year for a donor window crossing 2018) binds a declared composite provenance as the identity's source (`code_index.composites`). When the part settles one, the scope's base is the composite, and the answer is restated as it (`scope.composite.assemble_composite`). The restated answer carries the composite's spec_id and identity, its coverage and headline over the years its components serve, its StatVar counts summed over the components, its provenance facet naming the composite, and `compiled.components` with each component's role, table, years, StatVar count and identity. Its `download` facts count the file a Datasets mint of the composite writes, from the coverage snapshot in memory (`scope.composite.composite_download`), with `rows_exact` false, since the count reads no facet the mint elects. Its observation preview reads each component identity with `resolve_spec_resubmit`, which elects the component's members as the mint does, in worker threads that run together, and composes the previews under the composite's rules (`scope.composite.compose_observations`). Each component keeps the points in the years it serves, under its own StatVar and `source_refs`, and the event or embedded preview carries the composite's spec_id. A streamed request reads it in a trailing task launched once the answer is restated (`region.composite_answer.CompositeRelaunch`), so the `response` event waits on nothing new, and a buffered request reads it inline. The answer phase decides the answer's scope from the identity the Spec will hold before any preview read launches (`region.scope_decision.decide_scope`, over `spec_build.answer_identity`), and the restatement reuses that decision, so a fresh composite answer reads no preview of its table identity, streamed or buffered. A dataset_resubmit resolves the composite's table identity with no preview and reads each component through the part memo. A composite whose components serve no point omits its preview with `observations_no_points`, or the first reason a component's own preview omits with, or `observations_probe_failed` when a component answered nothing. A component read that fails upstream, crosses the deadline or answers nothing leaves its years out and warns `observations_probe_failed`, or `observations_timed_out` for the deadline, naming the component's role, so the answer stands. Each component reads through the request's catalogue under its own pin alone, as the mint reads it, whatever source scope the part read under. A same-shape merge previews the read its merged Spec makes: composite answers, which merge only with each other, answer the composite over the merged identity (`region.composite_answer.as_merged_composite`), whose spec_id, components, years, counts, download facts and preview cover every member's entities and slots, and any other merge reads its preview from the read plan the merge rebuilds in memory, the plan its download counts, as a single answer reads its own, or, where the merge rebuilds no plan, with one resubmit of its merged identity. The inclusive option serves the years both tables cover, read from the coverage snapshot in memory. Over a window the imputed table covers in part it is `partial`, with the latest run of covered years as its own window (`ProviderBinding.window`), which a switch or the request's own words apply. A stated inclusive scope ("including imputed multilateral") is then served over that run, the scope's base keeps the years asked for, so a switch to bilateral answers them, and the part carries a `year_unavailable` departure on `when:`: "Imputed multilateral aid covers only 2011 to 2024, so this shows those years." A carried entry on a partial option is dropped, since only the user may change the period asked for. For a `latest` window it covers the latest year both tables cover: the scope's base keeps the `latest` window, and the composite compiles to that one year (`provider.elect_latest_window`), so a switch to bilateral answers the CRS's own latest year. An option a part cannot serve is not applied as its default, and a donor with no grant-equivalent series keeps net flows. Search elects and resolves a composite over the identity its option was set on, with its first component's population bound (`code_index.composites.table_identity`). That read holds the first component's rows alone, so the counts of what the composite serves read every component. A fresh search does not refuse its scope. Parts a same-shape merge folds state the scope of the merged identity, built as a resubmit of it builds it from the union of the members' entries (`pipeline.merged_scope_choices`). To switch an option, post the scope's `base.identity` as a `dataset_resubmit` identity entry with its `entries` as `scope_entries`, the entry on the switched axis replaced by the switch (an explicit entry whose origin names the choice), so each axis holds one entry. The engine compiles them over the identity, resolves the result, and refuses `requested_option_unavailable` or `conflicting_requested_options` (422) at the scope entry it cannot apply. A sector part's whole, the same dataset over every sector, is asked the same way: post `base.identity` with the slot its sector options bind (`SlotBinding.slot`) set `unbound`, and as `scope_entries` an explicit entry from each other choice holding its active option. The whole then keeps the part's donor, recipient, basis, prices, window and provider scope, or refuses `requested_option_unavailable` at the entry holding an option it cannot serve, such as the inclusive scope for a donor the imputed table carries no rows for. A part whose scope states no sector choice binds no sector and has no whole. +A composite option (the inclusive provider scope, or the OECD headline per year for a donor window crossing 2018) binds a declared composite provenance as the identity's source (`code_index.composites`). When the part settles one, the scope's base is the composite, and the answer is restated as it (`scope.composite.assemble_composite`). The restated answer carries the composite's spec_id and identity, its coverage and headline over the years its components serve, its StatVar counts summed over the components, its provenance facet naming the composite with the source and Data Commons its components share, and `compiled.components` with each component's role, table, years, StatVar count and identity. Its `download` facts count the file a Datasets mint of the composite writes, from the coverage snapshot in memory (`scope.composite.composite_download`), with `rows_exact` false, since the count reads no facet the mint elects. Its observation preview reads each component identity with `resolve_spec_resubmit`, which elects the component's members as the mint does, in worker threads that run together, and composes the previews under the composite's rules (`scope.composite.compose_observations`). Each component keeps the points in the years it serves, under its own StatVar and `source_refs`, and the event or embedded preview carries the composite's spec_id. A streamed request reads it in a trailing task launched once the answer is restated (`region.composite_answer.CompositeRelaunch`), so the `response` event waits on nothing new, and a buffered request reads it inline. The answer phase decides the answer's scope from the identity the Spec will hold before any preview read launches (`region.scope_decision.decide_scope`, over `spec_build.answer_identity`), and the restatement reuses that decision, so a fresh composite answer reads no preview of its table identity, streamed or buffered. A dataset_resubmit resolves the composite's table identity with no preview and reads each component through the part memo. A composite whose components serve no point omits its preview with `observations_no_points`, or the first reason a component's own preview omits with, or `observations_probe_failed` when a component answered nothing. A component read that fails upstream, crosses the deadline or answers nothing leaves its years out and warns `observations_probe_failed`, or `observations_timed_out` for the deadline, naming the component's role, so the answer stands. Each component reads through the request's catalogue under its own pin alone, as the mint reads it, whatever source scope the part read under. A same-shape merge previews the read its merged Spec makes: composite answers, which merge only with each other, answer the composite over the merged identity (`region.composite_answer.as_merged_composite`), whose spec_id, components, years, counts, download facts and preview cover every member's entities and slots, and any other merge reads its preview from the read plan the merge rebuilds in memory, the plan its download counts, as a single answer reads its own, or, where the merge rebuilds no plan, with one resubmit of its merged identity. The inclusive option serves the years both tables cover, read from the coverage snapshot in memory. Over a window the imputed table covers in part it is `partial`, with the latest run of covered years as its own window (`ProviderBinding.window`), which a switch or the request's own words apply. A stated inclusive scope ("including imputed multilateral") is then served over that run, the scope's base keeps the years asked for, so a switch to bilateral answers them, and the part carries a `year_unavailable` departure on `when:`: "Imputed multilateral aid covers only 2011 to 2024, so this shows those years." A carried entry on a partial option is dropped, since only the user may change the period asked for. For a `latest` window it covers the latest year both tables cover: the scope's base keeps the `latest` window, and the composite compiles to that one year (`provider.elect_latest_window`), so a switch to bilateral answers the CRS's own latest year. An option a part cannot serve is not applied as its default, and a donor with no grant-equivalent series keeps net flows. Search elects and resolves a composite over the identity its option was set on, with its first component's population bound (`code_index.composites.table_identity`). That read holds the first component's rows alone, so the counts of what the composite serves read every component. A fresh search does not refuse its scope. Parts a same-shape merge folds state the scope of the merged identity, built as a resubmit of it builds it from the union of the members' entries (`pipeline.merged_scope_choices`). To switch an option, post the scope's `base.identity` as a `dataset_resubmit` identity entry with its `entries` as `scope_entries`, the entry on the switched axis replaced by the switch (an explicit entry whose origin names the choice), so each axis holds one entry. The engine compiles them over the identity, resolves the result, and refuses `requested_option_unavailable` or `conflicting_requested_options` (422) at the scope entry it cannot apply. A sector part's whole, the same dataset over every sector, is asked the same way: post `base.identity` with the slot its sector options bind (`SlotBinding.slot`) set `unbound`, and as `scope_entries` an explicit entry from each other choice holding its active option. The whole then keeps the part's donor, recipient, basis, prices, window and provider scope, or refuses `requested_option_unavailable` at the entry holding an option it cannot serve, such as the inclusive scope for a donor the imputed table carries no rows for. A part whose scope states no sector choice binds no sector and has no whole. On the SSE stream, every per-variable event (`shape`, `sentence`, `step`) carries `variable_index`, the same index the terminal `response` event's `parts` list is keyed by. Trailing per-spec events (`slot_options`, `entity_options`, `facet_options`, `observations`) join on `spec_id` instead, since a same-shape collapse can put two parts on the same spec. `sentence` events stay keyed by `variable_index`, one per part; the composed envelope sentence only ever rides the terminal `response` event, alongside the rest of the envelope. A part whose shape carries a measure family bind elected gets a first `sentence` event right after bind, naming only its measured thing from that election ("gross disbursements"), without the flow type and the year, followed by a second `sentence` event once materialise resolves the served rows, which adds both and refines that phrase when the served measure family differs from what bind elected. A part with no elected family (most indicators outside the CRS-style financial-flow axes) gets only the post-materialise `sentence` event, because materialise resolves its measured thing from the served StatVar. diff --git a/src/search/observations.py b/src/search/observations.py index aa683545d..1f5f28a45 100644 --- a/src/search/observations.py +++ b/src/search/observations.py @@ -30,6 +30,7 @@ from code_index.composites import DECLARED_COMPOSITES from code_index.query import CodeIndex +from code_index.types import CompositeDeclaration from description.blocks import documentation_link from materialise.enrichment import read_party_kinds from materialise.provenance_doc import ProvenanceNameReader, provenance_names @@ -161,7 +162,7 @@ def provenance_facet_ref(dcid: str, labels: Mapping[str, str]) -> SeriesFacetRef nor the catalogue holds, takes its declared label and no document route.""" composite = DECLARED_COMPOSITES.get(dcid) if composite is not None: - return SeriesFacetRef(ref=dcid, label=composite.label) + return composite_facet_ref(composite) row = current_catalogue().provenance(dcid) documentation = documentation_link(dcid) if row is None: @@ -172,6 +173,28 @@ def provenance_facet_ref(dcid: str, labels: Mapping[str, str]) -> SeriesFacetRef ) +def composite_facet_ref(composite: CompositeDeclaration) -> SeriesFacetRef: + """The provenance-axis facet ref of a declared composite, which neither the graph nor the + catalogue holds: its declared label, and the source and Data Commons its component tables' + catalogue rows give when every row names the same one. A component table the catalogue + does not list, or rows that name different sources or Data Commons, leave that field out. + It carries no document route, since the Graph API documents each component table and no + composite.""" + rows = [current_catalogue().provenance(recipe.dataset) for recipe in composite.components] + listed = [row for row in rows if row is not None] + if len(listed) < len(rows): + return SeriesFacetRef(ref=composite.provenance, label=composite.label) + sources = {(row.source.ref, row.source.label) for row in listed} + origins = {(row.data_commons.id, row.data_commons.label) for row in listed} + source, origin = _source_and_origin(listed[0]) + return SeriesFacetRef( + ref=composite.provenance, + label=composite.label, + source=source if len(sources) == 1 else None, + data_commons=origin if len(origins) == 1 else None, + ) + + type SeriesRank = tuple[bool, int, bool, float, SeriesKey] diff --git a/src/search/scope/composite.py b/src/search/scope/composite.py index afa0d5001..1fc9dc401 100644 --- a/src/search/scope/composite.py +++ b/src/search/scope/composite.py @@ -68,7 +68,7 @@ TimeWindow, omitted_observations, ) -from search.observations import provenance_refs +from search.observations import composite_facet_ref, provenance_refs from search.render.labels import join_labels from search.scope.tables import ScopeFacts from series_coverage.coverage_types import BoundRoles, Role @@ -425,8 +425,9 @@ def assemble_composite( headline counts the latest of those years over every component serving it (`_headline`); its StatVar counts are the components' summed, and it states whether a groupby sum adds the components (`_stat_vars`); its - provenance facet names the composite, with the composite's label and the account its - component tables give; its sources are its component tables, whose labels its + provenance facet names the composite, with the composite's label, the source and Data + Commons its component tables share (`search.observations.composite_facet_ref`) and the + account they give; its sources are its component tables, whose labels its `published_tables` joins, and its measure label reads the declaration's `measure_noun` (`_measure_label`), so its sentence cites both tables and names the composite's measure; its open role axes (`ResolvedSpec.scope`, `role_axis_facts`) list and count the parties @@ -446,12 +447,15 @@ def assemble_composite( observations = sum(item.years.counts.get(year, 0) for item in served for year in item.served) window = TimeWindow(start_year=years[0], end_year=years[-1]) if years else None sources = _component_sources(index, declaration) + provenance = composite_facet_ref(declaration) facets = { **spec.resolved.facets, "provenance": ResolvedFacet( basis="bound", - ref=declaration.provenance, - label=declaration.label, + ref=provenance.ref, + label=provenance.label, + source=provenance.source, + data_commons=provenance.data_commons, account=provenance_account(index, declaration.provenance), ), } diff --git a/tests/engine/test_dac2a_destination_populations.py b/tests/engine/test_dac2a_destination_populations.py index 77aaa7be9..9500553e7 100644 --- a/tests/engine/test_dac2a_destination_populations.py +++ b/tests/engine/test_dac2a_destination_populations.py @@ -25,7 +25,7 @@ from code_index.declarations import DAC2A_INCLUSIVE from code_index.shared import get_index from dc_sdmx import SdmxClient as MaterialiseSdmxClient -from origin_catalogue import current_catalogue +from origin_catalogue import DFO, current_catalogue from search.engine.clients.graph import Facet from search.engine.concurrency import scoped_concurrency from search.engine.core import ResolveRecord @@ -94,6 +94,7 @@ _graph, ) from tests.materialise.conftest import FakeGraphReader +from tests.origin_catalogue.fake_catalogue import FakeCatalogue, provenance_row _BILATERAL = "provenance/ONEDAC2aBilateral" _AGENCY = "EuropeanCommission" @@ -510,6 +511,34 @@ def test_a_fresh_dac2a_answer_reads_the_composite_its_destinations_allow( assert (pin.ref if isinstance(pin, IdentityBindingValue) else None) == source +def test_a_total_read_as_the_bilateral_composite_keeps_its_table_s_origin_and_source( + bind_catalogue: Callable[[FakeCatalogue], None], +) -> None: + bind_catalogue(FakeCatalogue(provenances={_DAC2A: provenance_row(_DAC2A, DFO)})) + + def answer(scenario: str) -> DefinitePart: + return _part( + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + plain = answer("H5").spec.resolved.facets["provenance"] + fresh = answer("H1") + resubmitted = _resolve_resubmit(fresh.spec.definition.identity) + + assert plain.ref == _DAC2A + assert plain.data_commons is not None + assert plain.source is not None + for part in (fresh, resubmitted): + facet = part.spec.resolved.facets["provenance"] + assert facet.ref == _BILATERAL + assert (facet.data_commons, facet.source) == (plain.data_commons, plain.source) + + @pytest.mark.parametrize("scenario", ["H1", "H2"]) def test_a_fresh_bilateral_reading_lists_no_agency_in_its_scope(scenario: str) -> None: part = _part( diff --git a/tests/engine/test_provenance.py b/tests/engine/test_provenance.py index 58ac811b3..79811c1ec 100644 --- a/tests/engine/test_provenance.py +++ b/tests/engine/test_provenance.py @@ -2,15 +2,19 @@ from __future__ import annotations -from origin_catalogue import BASE, UN_SYSTEM, ProvenanceRow, SourceRef +import pytest + +from code_index.composites import DECLARED_COMPOSITES +from code_index.declarations import CRS, CRS_INCLUSIVE +from origin_catalogue import BASE, DFO, UN_SYSTEM, ProvenanceRow, SourceRef from search.engine.region.refs import provenance_labels from search.models import DataCommonsRef, ProvenanceRef from search.models import SourceRef as WireSourceRef -from search.observations import provenance_refs +from search.observations import composite_facet_ref, provenance_refs from tests.engine._fake_sdmx import FakeSdmxClient from tests.engine._harness import make_request, offline_resolve from tests.fixtures import FakeGraph -from tests.origin_catalogue.fake_catalogue import restrictive_catalogue +from tests.origin_catalogue.fake_catalogue import provenance_row, restrictive_catalogue # --------------------------------------------------------------------------- # Integration test: resolved_sources populated end-to-end @@ -101,6 +105,56 @@ def test_un_provenance_is_its_own_source(self, bind_catalogue): assert graph.label_reads == [] +_IMPUTED = "provenance/ONEImputedMultilateralSectors" + + +@pytest.mark.parametrize( + ("rows", "expected"), + [ + pytest.param( + [ + provenance_row(CRS, DFO, source="dc/s/OECD"), + provenance_row(_IMPUTED, DFO, source="dc/s/OECD"), + ], + ("dfo", "dc/s/OECD"), + id="agree", + ), + pytest.param( + [ + provenance_row(CRS, DFO, source="dc/s/OECD"), + provenance_row(_IMPUTED, BASE, source="dc/s/ONE"), + ], + (None, None), + id="disagree", + ), + pytest.param( + [ + provenance_row(CRS, DFO, source="dc/s/OECD"), + provenance_row(_IMPUTED, BASE, source="dc/s/OECD"), + ], + (None, "dc/s/OECD"), + id="one-publisher-two-origins", + ), + pytest.param([provenance_row(CRS, DFO, source="dc/s/OECD")], (None, None), id="unlisted"), + ], +) +def test_a_composite_facet_keeps_what_its_component_tables_share( + bind_catalogue, rows: list[ProvenanceRow], expected: tuple[str | None, str | None] +): + """A composite's provenance facet keeps the Data Commons and source only where every + component table's catalogue row names the same one, and carries no document route.""" + bind_catalogue(restrictive_catalogue(provenances=rows)) + composite = DECLARED_COMPOSITES[CRS_INCLUSIVE] + + ref = composite_facet_ref(composite) + + assert (ref.ref, ref.label, ref.documentation) == (CRS_INCLUSIVE, composite.label, None) + assert ( + ref.data_commons.id if ref.data_commons else None, + ref.source.ref if ref.source else None, + ) == expected + + class _LabelSpyGraph(FakeGraph): """Record every `node_labels_batch` and `node_arc_targets_many` call.""" From cdb678dc11ba25a6ef7237c34c4dbc130412feec Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 14:33:26 +0100 Subject: [PATCH 03/12] fix(eval): count a composite's reads within each component's population The materialisation check counted a composite with only the roles bound in resolved.entities, so an open destination under the bilateral composite counted agencies and DACAllRecipients (11,263 against the answer's 7,842 on dq-crs-053). _CountedRead.within carries each component's population into the series count and the latest-year election. The DAC2a cases that now answer the bilateral composite expect provenance/ONEDAC2aBilateral with origin dfo. --- corpus/crs.yaml | 24 +++++++ corpus/dac.yaml | 7 +++ corpus/datasets.yaml | 16 +++-- corpus/exclusions.yaml | 14 +++++ corpus/mls.yaml | 3 + corpus/stability.yaml | 1 + corpus/standard.yaml | 15 +++++ src/search/eval/checks/materialisation.py | 28 ++++++--- tests/eval/checks/test_materialisation.py | 77 +++++++++++++++++++++++ 9 files changed, 171 insertions(+), 14 deletions(-) diff --git a/corpus/crs.yaml b/corpus/crs.yaml index fb9b9d58b..1905a23a8 100644 --- a/corpus/crs.yaml +++ b/corpus/crs.yaml @@ -6,6 +6,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/DEU, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: all} operations: [] @@ -18,6 +19,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/BGD, role: destination}] window: {kind: all} operations: [] @@ -69,6 +71,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - {dcid: EuropeanUnionInstitutions, role: origin} - {dcid: country/ETH, role: destination} @@ -83,6 +86,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - {dcid: GlobalFund, role: origin} - {dcid: country/MWI, role: destination} @@ -140,6 +144,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/TUR, role: origin}, {dcid: country/SYR, role: destination}] window: {kind: all} @@ -151,6 +156,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/TUR, role: origin}, {dcid: country/SYR, role: destination}] window: {kind: all} @@ -193,6 +199,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: DACSouthOfSahara, role: destination}] window: {kind: all} operations: [] @@ -205,6 +212,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/FRA, role: origin}, {dcid: DACAfrica, role: destination}] window: {kind: all} @@ -218,6 +226,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: DACAfricaUnallocated, role: destination}] - id: dq-crs-020 @@ -380,6 +389,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/USA, role: origin}, {dcid: country/PAK, role: destination}] window: {kind: range, start: 2020, end: 2020} @@ -423,10 +433,12 @@ parts: - definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/GBR, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2022, end: 2022} - definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/GBR, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2022, end: 2022} @@ -468,6 +480,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/FRA, role: origin}, {dcid: country/SEN, role: destination}] unit: {currency: EUR, prices: current} @@ -485,6 +498,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2020, end: 2020} unit: {currency: {role: destination}, prices: current} @@ -570,6 +584,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} unit: {currency: USD, prices: constant} no_data: {reason: unit_not_resolved, category: unresolved} @@ -581,6 +596,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] unit: {currency: USD, prices: constant} @@ -618,6 +634,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/DEU, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2015, end: 2020} operations: [] @@ -630,6 +647,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/SWE, role: origin}, {dcid: country/TZA, role: destination}] - id: dq-crs-052 @@ -640,6 +658,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/DEU, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 1975, end: 1975} @@ -651,6 +670,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/USA, role: origin}] operations: [{verb: rank, over: destination}] @@ -663,6 +683,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - {dcid: country/DEU, role: origin} - {dcid: country/FRA, role: origin} @@ -677,6 +698,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/CAN, role: origin}, {dcid: country/HTI, role: destination}] operations: [{verb: growth}] @@ -732,6 +754,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2020, end: 2020} @@ -800,6 +823,7 @@ parts: - definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/NOR, role: origin}, {dcid: country/ETH, role: destination}] - {definite: {measure: {family: base.gdp}, entities: [{dcid: country/ETH, role: observationAbout}]}} diff --git a/corpus/dac.yaml b/corpus/dac.yaml index 46d925d32..2722c586e 100644 --- a/corpus/dac.yaml +++ b/corpus/dac.yaml @@ -156,6 +156,7 @@ expect: definite: measure: {family: [dac2a.oda_net_received, dac2a.oda_gross_received]} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KOR, role: destination}] window: {kind: range, start: 1980, end: 1980} @@ -317,6 +318,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/ETH, role: destination}] window: {kind: range, start: 1965, end: 1965} @@ -338,6 +340,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2022, end: 2022} @@ -469,6 +472,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: GroupOfSeven, role: origin}, {dcid: DACAfrica, role: destination}] - id: dq-dac-036 @@ -487,6 +491,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/FRA, role: origin}, {dcid: DACAsia, role: destination}] - id: dq-dac-038 @@ -496,6 +501,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: DACFarEastAsia, role: destination}] - id: dq-dac-039 @@ -516,6 +522,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: DACAsia, role: destination}] window: {kind: range, start: 1990, end: 1999} diff --git a/corpus/datasets.yaml b/corpus/datasets.yaml index 7b019e75f..99c911ad0 100644 --- a/corpus/datasets.yaml +++ b/corpus/datasets.yaml @@ -94,7 +94,7 @@ tags: {theme: aid, entity: country, direction: destination_only, door: raw_text, dataset: none, language: fr} # Extraction drops "imputée", and "multilateral" without "imputed" refuses qualifier_not_bound # on the CRS answer instead of answering from the imputed table. - known_failure: {issue: 1156, checks: [measure]} + known_failure: {issue: 1156, checks: [measure, provenance]} expect: definite: measure: {family: dac2a.imputed_multilateral} @@ -163,6 +163,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/DEU, role: origin}, {dcid: country/KEN, role: destination}] operations: [] @@ -173,6 +174,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2024, end: 2024} @@ -190,6 +192,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACMultilaterals, role: origin}, {dcid: country/KEN, role: destination}] operations: [] @@ -330,7 +333,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} - provenance: {bound: false} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral, bound: true} entities: [{dcid: country/KEN, role: destination}] operations: [] @@ -341,7 +344,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} - provenance: {bound: true} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral, bound: true} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] operations: [] @@ -384,6 +387,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACMultilaterals, role: origin}, {dcid: country/KEN, role: destination}] operations: [] @@ -391,7 +395,7 @@ query: inputed aid to Kenya why: '"Inputed" is a misspelling of "imputed", and imputed aid to Kenya names no sector, so DAC2a''s imputed multilateral line answers it.' tags: {theme: aid, entity: country, direction: destination_only, door: raw_text, dataset: none, phrasing: misspelt} - known_failure: {issue: 1164, checks: [measure]} + known_failure: {issue: 1164, checks: [measure, provenance]} expect: definite: measure: {family: dac2a.imputed_multilateral} @@ -417,7 +421,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} - provenance: {bound: true} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral, bound: true} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] operations: [] @@ -492,7 +496,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} - provenance: {bound: true} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral, bound: true} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] operations: [] diff --git a/corpus/exclusions.yaml b/corpus/exclusions.yaml index c11cec872..54ec9da14 100644 --- a/corpus/exclusions.yaml +++ b/corpus/exclusions.yaml @@ -10,6 +10,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-oda-kenya-not-united-states @@ -19,6 +20,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-oda-africa-except-kenya @@ -28,6 +30,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: DACAfrica, role: destination}] - id: exclusion-oda-france-except-recipients @@ -46,6 +49,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-gross-not-net @@ -55,6 +59,7 @@ expect: definite: measure: {family: dac2a.oda_gross_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-not-net-leading @@ -64,6 +69,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-no-humanitarian-aid @@ -73,6 +79,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-everything-but-humanitarian @@ -107,6 +114,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-oda-france-standard-grants @@ -126,6 +134,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: DACAfrica, role: destination}] - id: exclusion-bilateral-oda-bilateral-loans @@ -135,6 +144,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-education-multilateral-donors @@ -169,6 +179,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-control-only-grants @@ -187,6 +198,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-not-including-multilateral-aid @@ -196,6 +208,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] - id: exclusion-control-bilateral-not-including-multilateral @@ -205,4 +218,5 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] diff --git a/corpus/mls.yaml b/corpus/mls.yaml index 5537d4826..44743a27c 100644 --- a/corpus/mls.yaml +++ b/corpus/mls.yaml @@ -226,6 +226,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/NOR, role: origin}, {dcid: country/KEN, role: destination}] - id: dq-mls-023 @@ -268,6 +269,7 @@ parts: - definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/NOR, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2022, end: 2022} - definite: @@ -283,6 +285,7 @@ parts: - definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/ETH, role: destination}] - definite: measure: {family: dac2a.imputed_multilateral} diff --git a/corpus/stability.yaml b/corpus/stability.yaml index d9a13d58d..c371d2350 100644 --- a/corpus/stability.yaml +++ b/corpus/stability.yaml @@ -140,6 +140,7 @@ expect: definite: measure: { family: dac2a.oda_net_received } + provenance: { origin: dfo, dataset: provenance/ONEDAC2aBilateral } entities: [ { dcid: country/KOR, role: origin }, diff --git a/corpus/standard.yaml b/corpus/standard.yaml index b07ecae4f..e42bb6fb6 100644 --- a/corpus/standard.yaml +++ b/corpus/standard.yaml @@ -2057,6 +2057,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/DEU, role: origin}, {dcid: country/KEN, role: destination}] - id: ent-abbreviation-uk-aid query: aid from the UK to Kenya @@ -2065,6 +2066,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/GBR, role: origin}, {dcid: country/KEN, role: destination}] - id: ent-04 query: ODA from the World Bank @@ -2110,6 +2112,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - dcid: DACOfficialDonors role: origin @@ -2288,6 +2291,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - dcid: DACOfficialDonors role: origin @@ -2304,6 +2308,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - dcid: DACOfficialDonors role: origin @@ -3883,6 +3888,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/ETH, role: destination}] window: {kind: range, start: 2021, end: 2021} - id: whole-total-oda-ethiopia-all-donors @@ -3894,6 +3900,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - {dcid: DACOfficialDonors, role: origin} - {dcid: DACPrivateDonors, role: origin} @@ -3908,6 +3915,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/ETH, role: destination}] - id: whole-total-oda-ethiopia-all-donors-total-label query: 'parsed: official development assistance, to Ethiopia, from All Donors, Total, 2021' @@ -3918,6 +3926,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/ETH, role: destination}] - id: whole-total-oda-ethiopia-summed-open-donors query: 'parsed: official development assistance, to Ethiopia, open donors, aggregate over origin, 2021' @@ -3933,6 +3942,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/ETH, role: destination}] - id: whole-total-oda-kenya-range-total query: 'parsed: official development assistance received, to Kenya, total from all donors, 2015 to 2022' @@ -3947,6 +3957,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: DACOfficialDonors, role: origin}, {dcid: country/KEN, role: destination}] window: {kind: range, start: 2015, end: 2022} - id: whole-total-oda-ethiopia-including-private @@ -3958,6 +3969,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: - {dcid: DACOfficialDonors, role: origin} - {dcid: DACPrivateDonors, role: origin} @@ -4011,6 +4023,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/KEN, role: destination}] window: {kind: range, start: 2020, end: 2024} operations: [] @@ -4021,6 +4034,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/KEN, role: destination}] window: {kind: range, start: 2022, end: 2022} operations: [] @@ -4031,6 +4045,7 @@ expect: definite: measure: {family: dac2a.oda_net_received} + provenance: {origin: dfo, dataset: provenance/ONEDAC2aBilateral} entities: [{dcid: country/KEN, role: destination}] window: {kind: range, start: 2022, end: 2022} operations: [] diff --git a/src/search/eval/checks/materialisation.py b/src/search/eval/checks/materialisation.py index 0023c73b3..24a059d25 100644 --- a/src/search/eval/checks/materialisation.py +++ b/src/search/eval/checks/materialisation.py @@ -30,7 +30,8 @@ An answer served by a declared composite provenance, which no series carries, is the sum of its components' counts, each derived from the definition by the bound release's code index (``code_index.composites.component_identities``) and counted under its own table's provenance -inside its own years (``_counted_reads``). +inside its own years, with an open role read within the population the component binds it +to (``_counted_reads``). A part the check cannot count is n/a, with the reason in the detail: a role-bearing answer confirmed against another coverage stamp, or mixing ``observationAbout`` with a flow role; a @@ -47,7 +48,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from code_index.composites import ( @@ -58,6 +59,7 @@ from code_index.members import SourceNotInFamily, identity_family_query, index_family from code_index.shared import get_index from dc_graph import Facet +from dfo_platform.roles import role_key_axis from graph_release.context import ReleaseNotBound from materialise.facets import Cadence, at_cadence, cadences_by_member, continuation from materialise.window import RequestedWindow, latest_period @@ -457,11 +459,14 @@ def _coverage_window(spec: SpecView, ctx: ScoreContext) -> TimeWindow | None: @dataclass(frozen=True) class _CountedRead: """One read an exact count sums: the served StatVars it counts, the facets it admits on - each axis, and the window it counts inside.""" + each axis, the window it counts inside, and the parties each coverage role it leaves open + is read within: a composite component's population + (`code_index.composites.ComponentIdentity.populations`), as Search reads it.""" stat_vars: list[str] stated: Mapping[str, _Stated] window: TimeWindow | None + within: Mapping[CoverageRole, frozenset[str]] = field(default_factory=dict) def _counted_reads( @@ -477,9 +482,9 @@ def _counted_reads( (`code_index.composites.component_identities`, as Search and the Datasets mint derive them). No graph series carries a composite's provenance, so each component reads the served StatVars of its own family under its own table's provenance, inside the answer's - window clipped to the component's. A composite whose components the index cannot derive, - or a component whose family it does not decide, is n/a with the reason; an unbound - release is an infrastructure result.""" + window clipped to the component's, and within the population each open role binds. A + composite whose components the index cannot derive, or a component whose family it does + not decide, is n/a with the reason; an unbound release is an infrastructure result.""" composite = (stated.get("provenance") or (None, False))[0] if composite not in DECLARED_COMPOSITES: return [_CountedRead(stat_vars, stated, window)] @@ -507,11 +512,17 @@ def _counted_reads( f"the code index decides no family for {composite}'s {component.role} component", ) members = {row.dcid for row in family.rows} + within = { + _COVERAGE_ROLE_FOR_AXIS[axis]: frozenset(population.members) + for key, population in component.populations.items() + if (axis := role_key_axis(key)) is not None + } reads.append( _CountedRead( [sv for sv in stat_vars if sv in members], {**stated, "provenance": (component.recipe.dataset, False)}, _clipped(window, component.identity.get("requested_window")), + within, ) ) return reads @@ -584,12 +595,13 @@ def _exact(spec: SpecView, response: Mapping[str, Any], ctx: ScoreContext) -> Re bound[coverage_role] = bound.get(coverage_role, frozenset()) | {ref} totals = 1 for read in reads: - facets_by_series = _role_series_facets(read.stat_vars, bound, snapshot) + read_bound = {**bound, **read.within} + facets_by_series = _role_series_facets(read.stat_vars, read_bound, snapshot) # The rows a conversion leaves out of the election make the answer unchecked above. elected = ( snapshot.index.elected_years( variables=read.stat_vars, - bound=bound, + bound=read_bound, admits=_stated_admits(read.stated, snapshot), ) if latest diff --git a/tests/eval/checks/test_materialisation.py b/tests/eval/checks/test_materialisation.py index 8f3e9c620..cf19a6644 100644 --- a/tests/eval/checks/test_materialisation.py +++ b/tests/eval/checks/test_materialisation.py @@ -7,6 +7,7 @@ import pytest +from code_index.shared import get_index from dc_graph import Facet from search.eval.checks import CHECKS, MODAL_ONLY_CHECKS, Result, ScoreContext, Verdict, WireSchema from series_coverage.loader import CoverageArtifactError @@ -962,6 +963,82 @@ def test_materialisation_composite_counts_its_components( assert detail in ev.detail +_DAC2A = "provenance/OECDDAC2A" +_DAC2A_BILATERAL = "provenance/ONEDAC2aBilateral" +_DAC2A_GROSS_ODA = next( + row.dcid + for row in get_index().statvars + if row.provenances == (_DAC2A,) + and row.measured_property == "amount" + and row.constraint_properties.get("dac2aAidType") == "DAC2aAidType206" + and row.constraint_properties.get("nettingBasis") == "Gross" +) +# France's DAC2a total ODA to Ethiopia, a bilateral-side destination, to the European +# Commission, an agency, and to DACAllRecipients, the total spanning both sides, 2019 to 2021. +_DAC2A_ROWS = tuple( + (_DAC2A_GROSS_ODA, "dac2a", "country/FRA", recipient, range(2019, 2022)) + for recipient in ("country/ETH", "EuropeanCommission", "DACAllRecipients") +) + + +def _bilateral_response(count: int) -> dict: + """France's total ODA on the DAC2a bilateral composite to an open destination, stating + `count` observations over 2019 to 2021.""" + resp = _role_response( + count, + (("country/FRA", "origin"),), + stat_vars=(_DAC2A_GROSS_ODA,), + window={"start_year": 2019, "end_year": 2021}, + ) + spec = resp["parts"][0]["spec"] + spec["resolved"]["facets"] = {"provenance": {"basis": "bound", "ref": _DAC2A_BILATERAL}} + spec["definition"]["identity"] = { + "identity_version": "spec1", + "shape": { + "kind": "statvar_schema", + "population_type": "FinancialFlow", + "measured_property": "amount", + "stat_type": "measuredValue", + "measurement_qualifier": "Nominal", + }, + "slots": [ + {"property": "dac2aAidType", "binding": {"state": "value", "ref": "DAC2aAidType206"}}, + {"property": "financialFlowType", "binding": {"state": "value", "ref": "Disbursement"}}, + {"property": "nettingBasis", "binding": {"state": "value", "ref": "Gross"}}, + ], + "entities": {"origin": ["country/FRA"]}, + "open_roles": ["destination"], + "requested_window": {"kind": "range", "start_year": 2019, "end_year": 2021}, + "facets": {"provenance": {"state": "value", "ref": _DAC2A_BILATERAL}}, + } + return resp + + +@pytest.mark.parametrize( + ("count", "verdict"), + [ + pytest.param(3, PASS, id="the-bilateral-destination-alone"), + pytest.param(9, FAIL, id="agency-and-spanning-rows-are-outside-the-population"), + ], +) +def test_materialisation_composite_counts_an_open_role_within_its_population( + count: int, verdict: Verdict +): + """The bilateral composite reads an open destination within its population, as Search + does, so of France's three recipients only Ethiopia's three years count: the European + Commission and DACAllRecipients are no member of the bilateral population.""" + coverage = functools.partial( + _ids_coverage, + rows=_DAC2A_ROWS, + methods={"dac2a": None}, + provenance=_DAC2A, + roles=("financingProvider", "financingRecipient"), + ) + ev = check(_bilateral_response(count), coverage=coverage) + assert ev.verdict == verdict, ev.detail + assert "observed=3," in ev.detail + + def _failing_coverage(): raise CoverageArtifactError("coverage view unreadable") From 92b69ed6ee6a1220b0e59d8f747e1c1b8f40c2d5 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 14:33:48 +0100 Subject: [PATCH 04/12] test(search): place core_contribution_read on the data side of the departure gate core_contribution_read describes the data (a multilateral agency named as a recipient reads total ODA to it), so it joins DATA_DESCRIBING_DEPARTURES. Test doubles of ResolveState gain from_request. --- src/search/catalogue/check.py | 4 +++- tests/engine/test_degraded_read_sites.py | 2 ++ tests/engine/test_window_answer.py | 8 +++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/search/catalogue/check.py b/src/search/catalogue/check.py index 9b891a65d..47846bd9e 100644 --- a/src/search/catalogue/check.py +++ b/src/search/catalogue/check.py @@ -76,6 +76,7 @@ "direction_inferred", "organisation_read_as_party", "scope_limited", + "core_contribution_read", } ) """Departures that describe the data the answer shows without changing what @@ -83,7 +84,8 @@ flow side was read as the total its source publishes for its whole population, or as every party its source holds there where it publishes no such total, a place named without a direction was read in the one direction holding data there, an organisation was -read as a party to the flow, a check covered only some values. An example may carry them.""" +read as a party to the flow, a check covered only some values, a multilateral agency named +as a recipient was read as total ODA to it. An example may carry them.""" REFUSING_DEPARTURES: Final[frozenset[DepartureCode]] = frozenset( { diff --git a/tests/engine/test_degraded_read_sites.py b/tests/engine/test_degraded_read_sites.py index 845b709ed..6fdb1dd73 100644 --- a/tests/engine/test_degraded_read_sites.py +++ b/tests/engine/test_degraded_read_sites.py @@ -141,6 +141,7 @@ def _relaxation_ctx(**changes: object) -> SimpleNamespace: "date_request": None, "deadline": None, "role_class_walks": object(), + "from_request": lambda *_warnings: None, **changes, } ) @@ -761,6 +762,7 @@ def _nearest_real(monkeypatch: pytest.MonkeyPatch, recorder: _NearestRealRecorde rate_cache=None, warnings=[], departure_facts={}, + from_request=lambda *_warnings: None, ) return materialise._nearest_real_for(cast(ResolveState, state)) diff --git a/tests/engine/test_window_answer.py b/tests/engine/test_window_answer.py index 694450913..6faaa6c8f 100644 --- a/tests/engine/test_window_answer.py +++ b/tests/engine/test_window_answer.py @@ -951,6 +951,7 @@ def test_a_trailing_answer_states_the_years_its_words_cover_once( monkeypatch.setattr(phase_answer, "deflator_omission_departure", lambda *_a, **_k: None) departed: list[DepartureFact] = [] state = SimpleNamespace( + from_request=lambda *_warnings: None, unit_request=object(), unit_elected_selections={}, date_request=settled.date_request, @@ -988,6 +989,7 @@ def start(_state: SimpleNamespace, *, date_request: DateRequest, **_kwargs: obje monkeypatch.setattr(related_read_module, "_elected_measure_family", lambda *_a: (None, ())) monkeypatch.setattr(related_read_module, "start_related_read", start) state = SimpleNamespace( + from_request=lambda *_warnings: None, unit_request=object(), unit_elected_selections={}, date_request=_trailing("the last five years"), @@ -1035,7 +1037,10 @@ async def decide(_state: object, _mat: object) -> RegionResult: else DateRequest(window=TimeWindow(start_year=2015, end_year=2020), latest=False) ) state = SimpleNamespace( - mat_result=_two_flows(range(2000, 2025)), date_request=request, related_read=None + from_request=lambda *_warnings: None, + mat_result=_two_flows(range(2000, 2025)), + date_request=request, + related_read=None, ) assert asyncio.run(phase_answer._ground_elected_answer(cast(Any, state))) is stop @@ -1074,6 +1079,7 @@ async def decide(_state: object, _mat: object) -> RegionResult: related_read_module, "start_related_read", lambda *_a, **_k: started.append(_k) ) state = SimpleNamespace( + from_request=lambda *_warnings: None, mat_result=dataclasses.replace(_two_flows(range(2000, 2025)), sv_dcids=[_FLOW]), date_request=request, related_read=held, From 58968bfc86aa1b0879b6e9fcda41ba483f4f5e63 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 15:30:36 +0100 Subject: [PATCH 05/12] test(eval): record two cases' measure failures that dev shares dq-crs-054 and dataset-aid-kenya-not-imputed answer the net netting basis where they expect gross, and the second also adds DACOfficialDonors as an origin. Both fail the same way on dev; editing their provenance made the eval treat them as new cases, so their known failures now name these checks. --- corpus/crs.yaml | 2 +- corpus/datasets.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/corpus/crs.yaml b/corpus/crs.yaml index 1905a23a8..4585c13f7 100644 --- a/corpus/crs.yaml +++ b/corpus/crs.yaml @@ -679,7 +679,7 @@ why: Compare is two origins in one part with no operation; DAC2a answers the aid measure. reuse: panel/crs-046 tags: {theme: aid, entity: two_countries, direction: both, unit: default, time: none, operation: compare, breakdown: none, door: raw_text, phrasing: casual} - known_failure: {issue: 1156, checks: [disposition]} + known_failure: {issue: 1156, checks: [disposition, measure]} expect: definite: measure: {family: dac2a.oda_gross_received} diff --git a/corpus/datasets.yaml b/corpus/datasets.yaml index 99c911ad0..a28eb1115 100644 --- a/corpus/datasets.yaml +++ b/corpus/datasets.yaml @@ -329,7 +329,7 @@ query: aid to Kenya, not imputed why: '"Imputed" words are the DAC2a measure, not a source, so "not imputed" reaches bind as a dropped qualifier and DAC2a answers with its provenance unbound.' tags: {theme: aid, entity: country, direction: destination_only, door: raw_text, dataset: excluded} - known_failure: {issue: 1156, checks: [disposition]} + known_failure: {issue: 1156, checks: [disposition, measure, entities]} expect: definite: measure: {family: dac2a.oda_gross_received} From 94a46a4b7a6c8021c27a2d2be6624d308915c503 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 19:43:44 +0100 Subject: [PATCH 06/12] refactor(search): elect the provider reading before the read A DAC2a part now elects its provider reading from pre-read state over the request's read window (region/reading.py), reads once within the bilateral destination population, and mints the Spec as the composite through build_spec(composite=CompositeMint). The restatement layer that resolved the population a second time and reconciled it onto the table answer is deleted: with_population_read, restated_reading, with_composite_answer, with_restated_keys, recounted_scope, assemble_composite, with_bilateral_readings, as_merged_composite and ResolveState.request_warnings. decide_scope splits into choose_provider before the read and build_part_scope after it, and within is required on materialise, run_materialise and graph_confirm_resolve. The bilateral_reading_unavailable warning carries only the params.why reasons spec-kit's warning-reason registry lists. Closes #311 --- docs/conformance-search-api-v1.md | 6 +- .../src/code_index/composites.py | 23 +- .../src/materialise/bundle_sources.py | 18 +- src/datasets_api/kinds/spec_id.py | 6 +- src/search/README.md | 6 +- src/search/engine/entry/dataset_entries.py | 32 +- src/search/engine/entry/dataset_resubmit.py | 97 +- .../engine/region/candidate_admission.py | 2 + .../engine/region/candidate_collapse.py | 265 ++-- .../engine/region/candidate_grounding.py | 134 ++- src/search/engine/region/candidates.py | 53 +- src/search/engine/region/composite_answer.py | 739 +----------- src/search/engine/region/grounding.py | 168 ++- src/search/engine/region/materialise_step.py | 2 +- src/search/engine/region/phase_answer.py | 326 +++-- .../engine/region/phase_place_materialise.py | 170 ++- src/search/engine/region/phase_shared.py | 121 +- src/search/engine/region/pipeline.py | 240 ++-- src/search/engine/region/place_options.py | 5 +- src/search/engine/region/promote_finalize.py | 10 +- src/search/engine/region/promote_ground.py | 31 +- src/search/engine/region/promote_state.py | 4 + src/search/engine/region/reading.py | 761 ++++++++++++ src/search/engine/region/resubmit.py | 33 +- src/search/engine/region/resubmit_promote.py | 6 +- src/search/engine/region/result.py | 143 +-- src/search/engine/region/scope_decision.py | 376 ++++-- src/search/engine/region/state.py | 136 ++- src/search/engine/region/whole_totals.py | 14 +- src/search/engine/response_cache_keys.py | 5 +- src/search/engine/retrieval/retrieve.py | 2 +- src/search/engine/retrieval/role_resolve.py | 2 +- src/search/engine/same_shape.py | 208 +++- src/search/engine/spec_build.py | 71 +- src/search/eval/checks/materialisation.py | 7 +- src/search/models/envelope.py | 10 +- src/search/models/request.py | 14 + src/search/models/scope.py | 3 +- src/search/observations.py | 16 +- src/search/render/heading.py | 15 +- src/search/scope/composite.py | 403 ++++--- src/search/scope/provider.py | 119 +- .../code_index/test_destination_population.py | 14 + tests/engine/_assemble_fixtures.py | 2 + tests/engine/_builders.py | 48 + tests/engine/test_arc_tables_serving.py | 9 +- tests/engine/test_assemble_spec_params.py | 3 + tests/engine/test_bound_property_assertion.py | 1 + .../test_candidate_collapse_rule_one.py | 9 +- tests/engine/test_composite_answer.py | 471 +------- .../test_composite_decided_before_preview.py | 144 ++- tests/engine/test_composite_mint.py | 157 +++ tests/engine/test_composite_resubmit_mint.py | 329 +++++ .../test_conjoin_assemble_and_enrichment.py | 2 + .../test_conjoin_entities_and_facets.py | 12 + .../test_conjoin_five_tuple_and_slots.py | 3 + tests/engine/test_conjoin_measure_merge.py | 1 + .../test_conjoin_warnings_and_primary.py | 4 + tests/engine/test_conjunction.py | 1 + tests/engine/test_coverage_facet_dates.py | 1 + tests/engine/test_dac2a_candidate_readings.py | 697 +++++++++++ .../test_dac2a_destination_populations.py | 1071 ++++++++--------- .../engine/test_dac2a_suggestion_readings.py | 336 ++++++ tests/engine/test_degraded_read_sites.py | 11 +- tests/engine/test_discover_graph_confirm.py | 1 + tests/engine/test_discover_role_resolve.py | 5 + tests/engine/test_entity_resolution.py | 1 + .../test_grounding_identity_entities.py | 31 + tests/engine/test_index_member_source.py | 2 + tests/engine/test_member_population.py | 4 + tests/engine/test_merged_composite_mint.py | 120 ++ tests/engine/test_observability.py | 1 + tests/engine/test_provenance.py | 16 +- ...st_qualifier_suppression_and_provenance.py | 5 +- tests/engine/test_read_plan_grid.py | 4 + tests/engine/test_reading_decision.py | 869 +++++++++++++ tests/engine/test_region_residuals_e2e.py | 2 +- tests/engine/test_request_warning_sites.py | 282 ----- tests/engine/test_resolved_scope.py | 6 +- tests/engine/test_role_axis_facts.py | 3 + .../engine/test_role_bearing_consolidation.py | 2 + tests/engine/test_role_bearing_resolve.py | 25 + tests/engine/test_role_class_shapes.py | 3 + tests/engine/test_role_currentness.py | 3 + tests/engine/test_role_grid_search.py | 1 + .../test_role_resolve_financing_roles.py | 6 + tests/engine/test_role_scope_discovery.py | 10 + tests/engine/test_role_twin_resolution.py | 8 + tests/engine/test_role_twin_unit_roles.py | 18 +- tests/engine/test_same_shape.py | 5 + tests/engine/test_set_ref.py | 1 + tests/engine/test_setaxes_multi.py | 3 + .../test_shape_label_measure_agreement.py | 6 + tests/engine/test_skills.py | 1 + tests/engine/test_slot_expansion.py | 3 + tests/engine/test_slot_expansion_wire.py | 1 + tests/engine/test_slot_summable.py | 1 + tests/engine/test_spec_build.py | 2 + tests/engine/test_standard_resolve.py | 29 + tests/engine/test_streaming.py | 2 + tests/engine/test_window_answer.py | 4 - tests/materialise/test_bundle.py | 17 +- tests/models/test_edge_cases.py | 2 + tests/platform/test_registry_binding.py | 32 + tests/render/test_render_byline.py | 21 +- tests/scope/test_basis.py | 4 +- tests/scope/test_parsed_door.py | 4 +- tests/scope/test_provider.py | 18 +- 108 files changed, 6471 insertions(+), 3241 deletions(-) create mode 100644 src/search/engine/region/reading.py create mode 100644 tests/engine/test_composite_mint.py create mode 100644 tests/engine/test_composite_resubmit_mint.py create mode 100644 tests/engine/test_dac2a_candidate_readings.py create mode 100644 tests/engine/test_dac2a_suggestion_readings.py create mode 100644 tests/engine/test_merged_composite_mint.py create mode 100644 tests/engine/test_reading_decision.py delete mode 100644 tests/engine/test_request_warning_sites.py diff --git a/docs/conformance-search-api-v1.md b/docs/conformance-search-api-v1.md index ae9aa1ea2..8becf91fb 100644 --- a/docs/conformance-search-api-v1.md +++ b/docs/conformance-search-api-v1.md @@ -159,8 +159,8 @@ Gap rows state what is needed to close the clause and name the open issue that t | C66c | resolved.stat_vars.sums_components is true when the definition binds a declared composite whose components sum (`code_index.composites.components_sum`, the rule the Datasets API's groupby adds a composite's components by, `datasets_api.groupby.sums_components`): a groupby whose `by` keeps no `stat_var` adds the members of its different components into one series per cell, and a cell holding one component's member serves that member. True for the stacked composites; false, and omitted from the wire, for the spliced one and for every answer binding no composite. It says nothing about slots; slots[].summable still governs a sum over each slot. Additive only; spec identity does not hash it | implemented | | | C66d | A composite component that binds its destinations to one side of a donor's aid (`ComponentRecipe.destinations`, set on `provenance/ONEDAC2aBilateral` and on both components of `ONEDAC2aBilateralAndImputedMultilateral`) reads a role open within a declared population (`code_index.scope_conventions.RolePopulation`). `code_index.composites.destination_population` builds it once per code index from the parties the index publishes as destinations of the component's dataset, each placed by `code_index.scope_conventions.destination_side`: `members` holds the bilateral-side parties and residuals and no total, as an open read holds none; `named` holds every bilateral-side destination, totals included; `total` is the dataset's published total of that side, `DACDevelopingCountries` (`BilateralSource.rebound`). The population is bound where a component's identity derives (`code_index.composites._derived_identity`), so every reader of rows, every cache key and the mint read it from the identity without a rule of their own. An open destination binds `members`, and its `entities` key is listed in `ComponentIdentity.populations`. A named destination passes when it is in `named` (a ref the dataset does not publish as a destination passes and reads no rows) and otherwise raises `CompositeNotServable` at `/entities/destination`, with `refs` naming the destinations at fault. An identity that neither names nor opens a destination binds `[total]` and lists no population, so the answer states the DAC's total of developing countries as it states any bound destination. `table_identity` returns a `TableIdentity` (`identity`, `populations`) built from the composite's first component, so Search elects within the population the mint reads. That read holds the first component's rows alone, so a reader counting what the composite serves reads every component. Only a reader of the names a caller asked for reads `populations`. `composite_digest` takes the code index its caller reads and folds each bound population's dataset, side, members and total into the digest, so a stored frame re-mints when the population changes, and leaves out a `destinations` field that is `None`, so a composite declared without it keeps its digest | implemented | Pinned by `tests/code_index/test_destination_population.py` (`test_the_population_holds_bilateral_parties_and_residuals_and_names_its_totals`, `test_an_open_destination_binds_the_population_and_tags_its_key`, `test_a_named_destination_on_the_bilateral_side_passes_untagged`, `test_a_named_destination_off_the_bilateral_side_is_refused`, `test_an_absent_destination_binds_the_population_s_total_untagged`, `test_the_table_identity_carries_the_first_component_s_population`, `test_the_table_identity_of_an_identity_binding_no_composite_is_itself`, `test_the_composite_digest_moves_with_the_population`, `test_every_declared_component_binding_destinations_has_a_declared_source`), `tests/code_index/test_destination_side.py` (`test_a_dac2a_destination_is_placed_on_its_side`). | | C66e | The Datasets mint reads a population through a typed filter. `datasets_api.adapter.to_materialisation_request` sends each role in `ComponentIdentity.populations` as `dc_sdmx.Among(members, side)`, which the SDMX client sends as the role's OR-list, as it sends a list of dcids. Each reader of an entity filter chooses between rows and names. Readers of rows use `members`: `dc_sdmx.client._entity_params`, the coverage and facet reads in `materialise.facets`, `datasets_api.admission.bound_roles`, and the cache keys `facets.entity_filter_key`, `materialise_sdmx_cache` and `bulk_pair_reads._bound_key`. Readers of the names a caller gave (`materialise.finish._plain_role_bindings`) skip it, as they skip `ContainedIn`, so the manifest states no population member as unserved (`coverage_below_scope`). A frame minted for an open destination holds no agency, `DACAllMultilateralRecipients` or `DACAllRecipients` row, the bilateral composite's sum is the bilateral-side rows alone, and the inclusive composite adds the imputed line (aid type 106) to it. The row ceiling, the admission estimate and the sync limits charge the population's rows as they charge a named list, and the bound read sends the whole population in one SDMX request within the client's URL budget. A scope's entities and window apply to a composite mint: its components derive from the scope-applied identity, for the CRS inclusive composite and both DAC2a composites, whether the definition is posted beside the mint or registered before it. A definition or scope that binds a population composite to a destination off its side is refused 422 `facet_not_available` at the mint and options doors, before any read and before the registry write, one issue per destination (`datasets_api.adapter._off_side_refusal`): `DACDevelopingCountries` on the destination role for `DACAllRecipients`, and the plain DAC2a table on the provenance axis for an agency or the DAC's total of them. The refusal's `facet_pointer` is `/entities/destination` when every such destination is `DACAllRecipients` and `/facets/provenance` otherwise | implemented | Pinned by `tests/datasets_api/test_mint_dac2a_destinations.py` (`test_an_open_destination_mints_no_agency_or_spanning_row`, `test_an_open_destination_sums_the_country_row_alone_and_inclusive_adds_106`, `test_the_bound_recipient_list_fits_one_request_under_the_url_budget`, `test_an_open_destination_states_no_population_member_as_unserved`, `test_the_mint_door_refuses_a_composite_bound_to_a_destination_off_its_side`, `test_the_definition_door_refuses_a_composite_bound_to_a_destination_off_its_side`, `test_the_off_side_refusal_points_where_its_fix_is`, `test_a_scope_binding_the_destination_reads_that_destination_alone`, `test_a_scope_binding_an_agency_refuses_before_any_read`, `test_a_scope_narrowing_the_window_reads_that_year_alone`), `tests/datasets_api/test_sync_estimate.py` (`test_a_population_bound_role_prices_only_its_members_rows`), `tests/materialise/test_facets.py` (`test_probe_facets_role_bearing_with_no_bound_role_covers_every_coverage_row`). | -| C66f | Search discovers an open role within its population, and a fresh answer reads the population as a resubmit of its definition does. At the resolver's entry a role in `ResubmitIdentityPayload.populations` reads as open `within` its members (`identity_edit.open_within`), and `retrieval.role_scope.discover_role_scope(within=...)` adds the members to the one bound every coverage read on that path takes, so `resolved.scope`, `role_scope_defaulted`, coverage, headline, plan and preview count the population alone. The resubmit door and the options door read the posted identity through `identity_edit.composite_table_identity`, which returns `TableRead(identity, populations, sides)`, and each component preview reads its component's populations (`composite_answer._resolve_component`). The part memo and the options cache key the bound list, so a population read and a caller naming the same parties do not collide. The options door adds the side's `named` destinations to its bound, so no agency, `DACAllMultilateralRecipients` or `DACAllRecipients` is offered on a destination axis a population composite reads, and its origin options list the donors holding rows at the population. A whole-total offer names `RolePopulation.total` in place of the side's whole totals (`whole_totals.whole_total_warnings`), so the comparable figure is the DAC's total of developing countries. `scope.composite._headline` and `_stat_vars` read each served component's own bound. A fresh answer that settles on a composite binding a population is read again within it before the pass judges the answer (`pipeline._within_population`, `composite_answer.population_table_answer`) and takes that read's scope, headline, coverage, plan and warnings, with the request's own words, the warnings raised before the read (`ResolveState.request_warnings`) and their departures (`composite_answer.with_population_read`). The fresh answer and a resubmit of its definition then agree on `resolved.scope`, headline, coverage and warnings. A composite whose identity names no destination is read at the population's total (`RolePopulation.total`, `DACDevelopingCountries`). The read supplies the answer's figures, members and warnings alone: the answer keeps the identity, spec_id and scope decided once on the request's own identity, so it names no destination and keeps its provider options. A composite that refuses the read answers `no_data` `facet_not_available` and logs the refusal at ERROR. A population holding no row in the window answers `no_data` under the request's words, and a candidate set drops a reading whose population holds none; a set left with one reading answers it definite, as a resubmit of that reading does. A candidate set's readings are read again concurrently, every part's reads together at most the request's concurrency limit (`concurrency.bounded_thread_permits`). An inclusive composite's scope, role axis facts and `role_scope_defaulted` count the recipients of both lines together, as its headline and the mint's frame do (`scope.composite._open_axes`, `result.recounted_scope`); where the coverage snapshot cannot list a component's parties it keeps its table read's counts and logs a WARNING naming the component. A same-shape merge of composites and each candidate restated as a composite read their table within the population. A posted identity that binds either DAC2a composite to an agency or to `DACAllRecipients` is refused 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `multilateral_destination` with the destinations in the cause's `entities`, at the identity's `/entities/destination`, with the plain DAC2a table as the alternative; no Search door settles it to another identity | implemented | Pinned by `tests/engine/test_dac2a_population_reads.py` (`test_a_resubmit_discovers_its_open_destination_within_the_population`, `test_a_resubmit_offers_the_population_total_in_place_of_the_spanning_one`, `test_a_composite_posted_with_a_destination_off_its_side_refuses`, `test_a_population_read_and_a_named_list_of_its_members_key_apart`, `test_each_component_preview_reads_its_destination_within_the_population`, `test_the_destination_options_offer_no_agency_or_spanning_total`, `test_the_origin_options_list_the_donors_holding_rows_at_the_population`, `test_a_composite_s_options_key_apart_from_its_table_identity_s`, `test_a_population_filter_keys_search_s_sdmx_reads_by_its_members`, `test_the_candidate_door_reads_each_composite_s_table_identity_once`), `tests/engine/test_dac2a_destination_populations.py` (`test_a_resubmit_counts_no_agency_or_spanning_series_for_an_open_destination`, `test_a_composite_preview_holds_the_country_row_and_inclusive_adds_106`, `test_a_fresh_answer_equals_a_resubmit_of_its_definition`, `test_a_fresh_composite_answer_reads_every_field_a_resubmit_reads`, `test_a_fresh_bilateral_reading_lists_no_agency_in_its_scope`, `test_a_merged_composite_reads_its_table_within_the_population`, `test_a_candidate_restated_as_the_bilateral_composite_lists_no_agency_in_its_scope`, `test_a_composite_whose_population_holds_no_row_answers_no_data_with_the_request_s_words`, `test_a_composite_answer_keeps_the_warnings_raised_before_its_read`, `test_a_candidate_whose_population_holds_no_row_is_dropped_not_restated`, `test_a_composite_answer_keeps_a_denominator_its_words_name_unverified`, `test_an_inclusive_answer_counts_the_recipients_of_both_lines`, `test_an_inclusive_answer_whose_component_parties_cannot_be_listed_says_so`, `test_a_composite_naming_no_destination_is_read_again_at_its_population_total`, `test_a_fresh_answer_naming_no_destination_states_the_scope_of_its_own_identity`, `test_a_population_read_the_composite_refuses_answers_no_data`, `test_a_candidate_set_left_with_one_reading_answers_as_its_resubmit`, `test_population_reads_take_the_request_s_limit_and_report_their_lost_reads`), `tests/engine/test_composite_answer.py` (`test_a_population_read_keeps_the_request_s_warnings_and_replaces_the_read_s`), `tests/engine/test_whole_total_bind_e2e.py` (`test_a_recipients_reading_under_total_oda_states_the_total_beside_its_recipients`, `test_ranked_recipients_under_total_oda_state_the_total_beside_them`). | -| C66g | DAC2a's plain aid type 206 table is total ODA, the contributions scope (`provider:contributions`, rule `contribution_recipient_total`), and counts the donor's core contributions at every multilateral destination. Its bilateral option binds `provenance/ONEDAC2aBilateral` (rule `bilateral_destinations`), aid type 206 over the bilateral population, and its inclusive option binds `ONEDAC2aBilateralAndImputedMultilateral`, total ODA plus DAC2a's imputed multilateral line (aid type 106), both lines over the population, so a core contribution is counted once as a flow or once as an imputed share and neither line holds a row at `DACAllRecipients`. Posted over a provider that holds the multilaterals' outflows, or over every donor, where the imputed line would count them twice, the inclusive composite serves the bilateral line. A fresh DAC2a total ODA reading settles on the bilateral composite (`provider.bilateral_default_reading`, `provider.with_bilateral_default`), with or without a provider axis, whatever donors it names, where it names no destination or only bilateral-side ones, so a ranking over all donors lists no agency. It keeps total ODA where the part's first provider entry, stated or carried, is `provider:contributions`, where the donor's total ODA holds no breakdown by recipient over the years asked, and where the population is unavailable (C66i). A fresh part with no provider axis has no option to state why, so wherever the default reaches it and its bilateral option is not available, it carries the info warning `bilateral_reading_unavailable`, stamped with its spec_id, with `params` `{why}`: the option's `why`, `destination_unclassified` for an unavailable population or the slot the composite refuses where it cannot serve what the part binds, else its reason code, `option_not_covered` where it does not cover every year the answer holds, and `scope_undecided` where the part's scope decision raised and the answer states no scope (`scope_decision.guarded_scope_decision`), and a message saying the figures include core contributions to multilateral agencies (`provider.bilateral_default_withheld`, `scope_decision.bilateral_withheld_warning`). Each reading a candidate set or refusal shows that keeps total ODA so carries it too (`composite_answer.with_bilateral_readings`). A part with a provider axis, or whose provider entry asks for total ODA, carries none. Where the part has no provider axis, no table can apply a `provider:contributions` or `provider:bilateral` entry, so the default serves it and the part does not report it in `dropped_entries`. A part holding a composite over bilateral-side destinations alone is not offered the total itself (`provider_not_offered`, `why` `destination_bilateral`), since those destinations hold no core contribution. The bilateral option is labelled by what it keeps and leaves out, the total by `BilateralSource.total_label`; the composite's measure noun reads "(bilateral flows)", its citation "OECD DAC2a", and its label counts the recipients its file holds. A `latest` window compiles to the one year the composite serves (`provider.elect_latest_window`). "Bilateral plus multilateral" on DAC2a reads the inclusive composite. A stored bare 206 identity reads back as total ODA under its own spec_id and offers the bilateral composite and the inclusive scope as switches | implemented | Pinned by `tests/scope/test_provider.py` (`test_a_fresh_dac2a_total_reads_its_bilateral_destinations_unless_it_names_an_agency`, `test_a_fresh_dac2a_part_switched_to_gross_from_its_base_keeps_its_bilateral_reading`, `test_a_fresh_dac2a_part_at_a_country_offers_no_switch_to_total_oda`, `test_bilateral_plus_multilateral_on_dac2a_reads_the_inclusive_composite`, `test_a_resubmitted_bare_dac2a_total_reads_as_total_oda`, `test_a_donor_with_no_breakdown_by_recipient_keeps_total_oda`, `test_the_latest_bilateral_reading_elects_the_year_its_composite_reads`, `test_a_dac2a_bilateral_answer_counts_the_recipients_its_file_holds`, `test_each_table_names_its_provider_options_by_what_they_count`, `test_a_dac2a_composite_over_bilateral_destinations_offers_no_total_oda`, `test_an_inclusive_composite_over_providers_it_counts_twice_serves_the_bilateral_line`), `tests/engine/test_dac2a_destination_populations.py` (`test_a_fresh_dac2a_answer_reads_the_composite_its_destinations_allow`, `test_a_ranking_over_every_donor_says_when_it_cannot_read_bilateral_destinations`, `test_a_part_with_a_provider_axis_states_an_unavailable_population_on_its_scope`, `test_a_candidate_over_every_donor_says_when_it_cannot_read_bilateral_destinations`), `tests/engine/test_composite_answer.py` (`test_a_candidate_set_restating_a_dac2a_reading_keeps_its_other_readings_options`). | +| C66f | Search discovers an open role within its population, and a fresh answer reads the population as a resubmit of its definition does. At the resolver's entry a role in `ResubmitIdentityPayload.populations` reads as open `within` its members (`identity_edit.open_within`), and `retrieval.role_scope.discover_role_scope(within=...)` adds the members to the one bound every coverage read on that path takes, so `resolved.scope`, `role_scope_defaulted`, coverage, headline, plan and preview count the population alone. The resubmit door and the options door read the posted identity through `identity_edit.composite_table_identity`, which returns `TableRead(identity, populations, sides)`, and each component preview reads its component's populations (`composite_answer._resolve_component`). The part memo and the options cache key the bound list, so a population read and a caller naming the same parties do not collide. The options door adds the side's `named` destinations to its bound, so no agency, `DACAllMultilateralRecipients` or `DACAllRecipients` is offered on a destination axis a population composite reads, and its origin options list the donors holding rows at the population. A whole-total offer names `RolePopulation.total` in place of the side's whole totals (`whole_totals.whole_total_warnings`), so the comparable figure is the DAC's total of developing countries. `scope.composite._headline` and `_stat_vars` read each served component's own bound. A fresh answer elects its provider reading before its read (`reading.elect_reading`, over `extract.read_window`) and, where the reading is a composite binding a population, reads its table once within that population (`reading.read_within`), so the pass judges the answer over the population's rows alone. Its Spec is minted as the composite (`spec_build.build_spec(composite=)`, `scope.composite.CompositeMint`) and its scope is built from the same election (`scope_decision.build_part_scope`), so the fresh answer and a resubmit of its definition agree on `resolved.scope`, headline, coverage and warnings. After the read a monitor (`reading._checked`) compares the election with the identity read: a bilateral reading that covers the range the answer settled on but not every year read states `bilateral_reading_unavailable` with `why: read_window_not_covered`, and any other difference logs at ERROR and notes a degraded read. A composite whose identity names no destination is read at the population's total (`RolePopulation.total`, `DACDevelopingCountries`). The read supplies the answer's figures, members and warnings alone: the answer keeps the identity, spec_id and scope decided once on the request's own identity, so it names no destination and keeps its provider options. A composite that refuses the read answers `no_data` `facet_not_available`, logs the refusal at ERROR and notes a degraded read so no cache stores the answer, once per region's resolution however many of its reads elect that composite, its reading with party words and without alike (`reading.noting_refusals_once`). Each answer is judged after its read by the reading its own rows were read under: the definite answer's by the elected shape's reading, a candidate set's survivor by its own (`phase_answer.finished_scope(reading=)`). A Spec read within a population is stated only as that population's composite, so a Spec read within one and not minted as it fails closed before any scope is decided again (`scope_decision.decide_after_read`): it logs at ERROR, notes a degraded read so no cache stores the answer, and answers `no_data` `facet_not_available`. A Spec read over the whole table that no decision made before its read states is decided again, chooses nothing, and reads as that table. A definite answer's scope is checked once (`phase_answer.finished_scope`), by its producer, the answer phase (`phase_answer._assemble_definite_answer`) or the candidates path (`candidate_collapse._assemble_definite`), before its related read is awaited or any of its options or preview launch. An answer that fails closed (`phase_answer.failed_closed`) launches none of them and contributes no warning, departure fact or judgement: its producer marks the part's warnings, departure facts, re-election warning and judgement where its answer becomes specific (`ResolveState.mark_answer`), the answer phase before it grounds the elected answer (`phase_answer._ground_elected_answer`) and the candidates path before it grounds the candidates (`candidate_grounding.ground_candidates`), and the refusal restores them to the mark, so a warning raised since is dropped, one reworded since reads as it was, and the answer's judgement is undone. The requirements the request names (`ResolveState.requirements`), which the candidates path records after its mark (`candidate_collapse._record_unheld_values`), lie outside the mark and stand. It also cancels the related-measures read already started for it (`ResolveState.related_read`): the answer's own on the answer phase, and on the candidates path the elected shape's, started before the set is grounded. The `no_data` answer that stands in its place describes the candidate set's survivor where the survivor fails closed, and the pass states its departures, warnings and timing on it. A definite answer carries the decision its check returned (`RegionResult.scope_decision`), which the pass states as its scope (`pipeline._with_scope_choices`); a definite result carrying none raises. Both producers read a Spec minted as a composite as that composite, its preview per component and none of its table's options, only where that decision compiles the composite (`ScopeDecision.composite`). A population holding no row in the window answers `no_data` under the request's words. A read within a population that returns no row takes the path a plain table's read returning no row takes: a candidate set's elected reading answers `no_data` `no_observations` from its one read, and no other reading of the set is read in its place, since a reading without data offers only status rule 1's stand-ins and the readings that ride on its refusal (`phase_place_materialise._offered`); a candidate set whose reading cannot be minted drops it, and a set left with one reading answers it definite, as a resubmit of that reading does. Each reading a candidate set, a refusal's alternate, a flagged match or a nearest-real suggestion shows elects its own provider reading at its own read (`reading.elect_at_read`) and is minted as the composite that reading names (`reading.decide_read_scope`). An inclusive composite's scope, role axis facts and `role_scope_defaulted` count the recipients of both lines together, as its headline and the mint's frame do (`scope.composite.CompositeMint.open_axes`, `CompositeMint.role_scope_warnings`); where the coverage snapshot cannot list a component's parties it keeps its table read's counts and logs a WARNING naming the component. A same-shape merge of composites is minted as the composite over the merged identity from its members' facets (`same_shape._merged_composite`), whose members each read within the population. A posted identity that binds either DAC2a composite to an agency or to `DACAllRecipients` is refused 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `multilateral_destination` with the destinations in the cause's `entities`, at the identity's `/entities/destination`, with the plain DAC2a table as the alternative; no Search door settles it to another identity | implemented | Pinned by `tests/engine/test_dac2a_population_reads.py` (`test_a_resubmit_discovers_its_open_destination_within_the_population`, `test_a_resubmit_offers_the_population_total_in_place_of_the_spanning_one`, `test_a_composite_posted_with_a_destination_off_its_side_refuses`, `test_a_population_read_and_a_named_list_of_its_members_key_apart`, `test_each_component_preview_reads_its_destination_within_the_population`, `test_the_destination_options_offer_no_agency_or_spanning_total`, `test_the_origin_options_list_the_donors_holding_rows_at_the_population`, `test_a_composite_s_options_key_apart_from_its_table_identity_s`, `test_a_population_filter_keys_search_s_sdmx_reads_by_its_members`, `test_the_candidate_door_reads_each_composite_s_table_identity_once`), `tests/engine/test_dac2a_destination_populations.py` (`test_a_resubmit_counts_no_agency_or_spanning_series_for_an_open_destination`, `test_a_composite_preview_holds_the_country_row_and_inclusive_adds_106`, `test_a_fresh_answer_equals_a_resubmit_of_its_definition`, `test_a_fresh_composite_answer_reads_every_field_a_resubmit_reads`, `test_a_fresh_bilateral_reading_lists_no_agency_in_its_scope`, `test_a_composite_whose_population_holds_no_row_answers_no_data_with_the_request_s_words`, `test_a_composite_answer_keeps_the_warnings_raised_before_its_read`, `test_a_composite_answer_keeps_a_denominator_its_words_name_unverified`, `test_an_inclusive_answer_counts_the_recipients_of_both_lines`, `test_an_inclusive_answer_whose_component_parties_cannot_be_listed_says_so`, `test_a_fresh_answer_naming_no_destination_states_the_scope_of_its_own_identity`, `test_a_part_settling_on_a_population_composite_reads_its_table_once_within_it`, `test_a_trailing_request_reads_the_reading_its_settled_range_allows`), `tests/engine/test_reading_decision.py` (`test_the_reading_elected_before_the_read_states_the_scope_decided_after_it`, `test_a_reading_the_read_window_withholds_and_the_settled_range_allows_says_so`, `test_a_composite_that_refuses_its_table_read_answers_facet_not_available`, `test_a_composite_that_cannot_be_minted_answers_no_data_not_the_table`, `test_a_finished_answer_decided_again_chooses_no_composite`, `test_a_finished_answer_that_fails_closed_raises_and_judges_nothing`, `test_a_finished_answer_is_judged_by_the_reading_its_own_rows_were_read_under`, `test_a_definite_answer_carries_the_scope_decision_it_states`), `tests/engine/test_composite_mint.py` (`test_a_minted_dac2a_composite_states_the_composite_s_identity_and_components`, `test_the_mint_states_the_defaulted_scope_count_its_role_axis_fact_counts`), `tests/engine/test_composite_resubmit_mint.py` (`test_a_composite_resubmit_answers_as_the_composite_with_no_restatement_layer`, `test_the_oecd_headline_resubmitted_serves_net_flows_then_grant_equivalents`), `tests/engine/test_dac2a_candidate_readings.py` (`test_each_dac2a_candidate_is_read_within_its_population_and_minted_as_its_composite`, `test_a_set_whose_elected_reading_s_read_holds_no_row_answers_no_data`, `test_a_set_left_with_one_composite_reading_answers_it_as_its_resubmit_does`, `test_a_survivor_no_scope_states_fails_closed_launching_no_task`, `test_a_survivor_that_fails_closed_raises_and_judges_nothing`, `test_a_survivor_stated_as_no_composite_reads_as_its_table`, `test_a_refusal_offering_a_composite_alternate_carries_its_mint`), `tests/engine/test_dac2a_suggestion_readings.py` (`test_a_flagged_match_settling_on_the_bilateral_composite_is_minted_as_it`, `test_a_nearest_real_suggestion_settling_on_the_bilateral_composite_is_minted_as_it`), `tests/engine/test_merged_composite_mint.py` (`test_a_merged_bilateral_composite_answers_as_a_resubmit_of_its_definition`), `tests/engine/test_whole_total_bind_e2e.py` (`test_a_recipients_reading_under_total_oda_states_the_total_beside_its_recipients`, `test_ranked_recipients_under_total_oda_state_the_total_beside_them`). | +| C66g | DAC2a's plain aid type 206 table is total ODA, the contributions scope (`provider:contributions`, rule `contribution_recipient_total`), and counts the donor's core contributions at every multilateral destination. Its bilateral option binds `provenance/ONEDAC2aBilateral` (rule `bilateral_destinations`), aid type 206 over the bilateral population, and its inclusive option binds `ONEDAC2aBilateralAndImputedMultilateral`, total ODA plus DAC2a's imputed multilateral line (aid type 106), both lines over the population, so a core contribution is counted once as a flow or once as an imputed share and neither line holds a row at `DACAllRecipients`. Posted over a provider that holds the multilaterals' outflows, or over every donor, where the imputed line would count them twice, the inclusive composite serves the bilateral line. A fresh DAC2a total ODA reading settles on the bilateral composite (`provider.bilateral_default_reading`, `provider.with_bilateral_default`), with or without a provider axis, whatever donors it names, where it names no destination or only bilateral-side ones, so a ranking over all donors lists no agency. It keeps total ODA where the part's first provider entry, stated or carried, is `provider:contributions`, where the donor's total ODA holds no breakdown by recipient over the years asked, and where the population is unavailable (C66i). A fresh part with no provider axis has no option to state why, so wherever the default reaches it and its bilateral option is not available, it carries the info warning `bilateral_reading_unavailable`, stamped with its spec_id, with `params` `{why}`, `why` one of the reasons dfo-spec-kit's `registries/warning-reasons.json` lists for the code and no other (`provider.WithheldWhy`): `destination_unclassified` for an unavailable population, `slot_not_served` with `params.slot` naming the slot's property where the composite cannot serve a slot the part binds, `option_not_covered` where it does not cover every year the answer holds or the window reaches none of its years, `read_window_not_covered` where it covers the range the answer settled on but not every year the read reads (`reading._checked`), and `scope_undecided` where the answer states no scope because it could not be decided: the part's scope decision raised (`scope_decision.guarded_scope_decision`), its reading election raised (`reading.elect_reading`), its composite refused its table read, so a flagged match built over the part is the table's Spec (`ReadingDecision.refused`), a reading read with no election would read the bilateral default (`reading._unelected_choice`), or a finished answer read over the whole table was decided again after its read and failed closed (`scope_decision.decide_after_read`), and a message saying the figures include core contributions to multilateral agencies (`provider.bilateral_default_withheld`, `scope_decision.bilateral_withheld_warning`). Each reading a candidate set, a refusal's alternate, a flagged match or a nearest-real suggestion shows that keeps total ODA so carries it too, keyed to its own spec_id (`reading.decide_read_scope`). A part with a provider axis, or whose provider entry asks for total ODA, carries none. Where the part has no provider axis, no table can apply a `provider:contributions` or `provider:bilateral` entry, so the default serves it and the part does not report it in `dropped_entries`. A part holding a composite over bilateral-side destinations alone is not offered the total itself (`provider_not_offered`, `why` `destination_bilateral`), since those destinations hold no core contribution. The bilateral option is labelled by what it keeps and leaves out, the total by `BilateralSource.total_label`; the composite's measure noun reads "(bilateral flows)", its citation "OECD DAC2a", and its label counts the recipients its file holds. A `latest` window compiles to the one year the composite serves (`provider.elect_latest_window`). "Bilateral plus multilateral" on DAC2a reads the inclusive composite. A stored bare 206 identity reads back as total ODA under its own spec_id and offers the bilateral composite and the inclusive scope as switches | implemented | Pinned by `tests/scope/test_provider.py` (`test_a_fresh_dac2a_total_reads_its_bilateral_destinations_unless_it_names_an_agency`, `test_a_fresh_dac2a_part_switched_to_gross_from_its_base_keeps_its_bilateral_reading`, `test_a_fresh_dac2a_part_at_a_country_offers_no_switch_to_total_oda`, `test_bilateral_plus_multilateral_on_dac2a_reads_the_inclusive_composite`, `test_a_resubmitted_bare_dac2a_total_reads_as_total_oda`, `test_a_donor_with_no_breakdown_by_recipient_keeps_total_oda`, `test_the_latest_bilateral_reading_elects_the_year_its_composite_reads`, `test_a_dac2a_bilateral_answer_counts_the_recipients_its_file_holds`, `test_each_table_names_its_provider_options_by_what_they_count`, `test_a_dac2a_composite_over_bilateral_destinations_offers_no_total_oda`, `test_an_inclusive_composite_over_providers_it_counts_twice_serves_the_bilateral_line`), `tests/engine/test_dac2a_destination_populations.py` (`test_a_fresh_dac2a_answer_reads_the_composite_its_destinations_allow`, `test_a_ranking_over_every_donor_says_when_it_cannot_read_bilateral_destinations`, `test_a_part_with_a_provider_axis_states_an_unavailable_population_on_its_scope`, `test_a_trailing_request_reads_the_reading_its_settled_range_allows`), `tests/engine/test_reading_decision.py` (`test_a_latest_request_narrowed_to_a_range_keeps_the_reading_it_read`, `test_an_election_that_raises_building_its_document_reads_total_oda_saying_so`, `test_a_finished_answer_decided_again_chooses_no_composite`), `tests/engine/test_dac2a_candidate_readings.py` (`test_each_dac2a_candidate_is_read_within_its_population_and_minted_as_its_composite`, `test_each_candidate_carries_its_own_withheld_reading_warning`), `tests/engine/test_dac2a_suggestion_readings.py` (`test_each_reading_the_bilateral_default_cannot_reach_says_so_under_its_own_spec_id`, `test_a_flagged_match_whose_composite_refuses_its_read_says_it_reads_total_oda`). | | C66h | A DAC2a part naming a multilateral-side destination reads total ODA: a named agency, `DACAllMultilateralRecipients` or one of the DAC's unnamed multilateral recipients. Its bilateral and inclusive options are `unavailable` with the scope reason `multilateral_destination`, the destination in `detail`, and a stated bilateral scope settles on total ODA. Any DAC2a part naming such a destination, with or without a provider axis, carries the `core_contribution_read` departure (`departures._core_contribution_facts`), which names the destination and is worded for an agency, for the DAC's total over agencies or for its line of agencies it does not name. A part bound to `DACAllRecipients`, a total that spans both sides, holds total ODA. Under the bilateral or inclusive scope its destination rebinds to `DACDevelopingCountries` (`scope.edits.rebound_destinations`), the bilateral side of all recipients, and the option labels say so (`OPTION_LABELS.provider_rebound`); a switch back reads all recipients again. A posted identity binding a composite to either destination is refused rather than settled (C66f) | implemented | Pinned by `tests/scope/test_provider.py` (`test_a_dac2a_total_to_an_agency_offers_no_bilateral_reading_and_says_why`, `test_a_dac2a_answer_to_an_agency_says_its_figures_are_core_contributions`, `test_every_donor_s_dac2a_aid_to_an_agency_says_its_figures_are_core_contributions`, `test_a_dac2a_total_to_all_recipients_reads_developing_countries_when_cut`), `tests/render/test_render_departures.py` (`test_departure_text`), `tests/engine/test_dac2a_population_reads.py` (`test_a_composite_posted_with_a_destination_off_its_side_refuses`). | | C66i | A population is available only while every destination it could leave out is placed on a side. `destination_population` raises `PopulationUnavailable` (`found_in` `code_index`), a `CompositeNotServable` at `/facets/provenance` naming the composite and carrying the destinations in `unclassified`, so every handler of `CompositeNotServable` catches it, when the code index publishes a destination `destination_side` cannot place: a party that is neither a country nor an organisation the entity resolver knows, a total other than `DACAllRecipients`, `DACAllMultilateralRecipients` and the declared bilateral totals (`DACDevelopingCountries`, the DAC regions of `REGION_NAMES`, `DAC_BILATERAL_GROUP_TOTALS`), and a residual other than `DAC_MULTILATERAL_RECIPIENTS` and the declared bilateral residuals (the unallocated lines of `TOTAL_GROUPS`, `DAC_BILATERAL_UNALLOCATED`). `code_index.composites.population_check(index, coverage)` also reads the coverage snapshot: every recipient coverage holds for the population's variables must be a destination the code index publishes, else the population is unavailable with `found_in` `coverage`, since a list of members would leave that recipient out without a word. The check is memoised per code index, then per coverage index, each entry dropped with its own index. The release service runs the check as a release opens (`graph_release.service.build_release_views`), logs `destination_population_unavailable` for each failing population, and still serves the release. A head's freshness check (`materialise.dependencies.refreshed_dependencies`) gives a composite whose population is unavailable no digest and logs `composite_digest_unavailable`, so the head rebuilds and the mint answers its refusal. While the check fails, the mint refuses a composite reading the population before any read with `DestinationUnclassifiedError`, a server fault (500 `internal_error`) whose detail names the destinations (`datasets_api.serve._materialise_components`, `datasets_api.adapter.composite_components`), and Search reports the bilateral and inclusive options `provider_not_offered` with `why` `destination_unclassified` and keeps total ODA on a fresh part (`provider._population_unavailable`), which warns `bilateral_reading_unavailable` where it has no provider axis (C66g). A posted identity binding the composite then refuses 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `provider_not_offered` naming the unplaced destinations, with the plain DAC2a table as the alternative, and a reader of the composite's table, its pins or its family query refuses it alike | implemented | Pinned by `tests/code_index/test_destination_side.py` (`test_an_unknown_party_that_is_no_country_is_unclassified`, `test_a_total_or_residual_no_declaration_places_is_unclassified`, `test_a_party_no_dataset_publishes_as_a_recipient_is_unclassified`, `test_every_dac2a_destination_of_the_recorded_index_is_classified`, `test_the_organisation_records_read_from_a_zipped_package_without_running_its_init`), `tests/code_index/test_destination_population.py` (`test_a_destination_the_code_index_cannot_place_refuses_the_composite_reading_it`, `test_coverage_whose_recipients_the_population_places_leaves_it_available`, `test_a_covered_recipient_the_code_index_does_not_publish_makes_the_population_unavailable`, `test_the_population_check_memo_holds_one_index_finalizer_however_many_coverages`), `tests/materialise/test_dependencies.py` (`test_a_composite_whose_population_is_unavailable_rebuilds_without_raising`), `tests/graph_release/test_service.py` (`test_opening_a_release_checks_its_destination_populations_against_its_coverage`), `tests/datasets_api/test_mint_dac2a_destinations.py` (`test_a_composite_whose_population_coverage_cannot_place_refuses_before_any_read`, `test_a_composite_whose_code_index_cannot_place_a_destination_is_a_server_fault`), `tests/datasets_api/test_mint_error_classification.py` (`test_an_unclassified_destination_is_a_server_fault_whose_detail_names_it`), `tests/scope/test_provider.py` (`test_a_dac2a_total_naming_a_destination_on_no_side_keeps_total_oda`, `test_a_dac2a_total_naming_a_destination_on_no_side_offers_neither_cut_reading`), `tests/engine/test_dac2a_population_reads.py` (`test_a_composite_whose_coverage_holds_a_destination_on_no_side_refuses_at_resubmit`, `test_a_composite_whose_code_index_places_no_side_refuses_at_resubmit`, `test_a_composite_whose_coverage_holds_a_destination_on_no_side_refuses_its_options`, `test_a_reader_of_a_composite_s_table_refuses_an_unavailable_population`). | | C67 | A singleton set normalises to the value form; empty sets are invalid everywhere | implemented | | @@ -267,7 +267,7 @@ Gap rows state what is needed to close the clause and name the open issue that t | C107 | Named codes: date_coverage_partial, binding_folded, observations_omitted | partial | `date_coverage_partial` is minted when a unit conversion's tables cover only part of the served years (C178), and when a span counted back from now answers from data whose newest year lies more than `SEARCH_TRAILING_STALE_YEARS` before the current one (C178a). `binding_folded` and `observations_omitted` are minted and reachable. The registry keeps `default_not_declared` active on search.resolve, and nothing emits it, because unit completion reads declared sources or refuses (C100). Issue: none | | C107a | Origin catalogue warnings on search.resolve: `un_system_unavailable` on a part the UN pass could change, one whose own pass ends no_data for a reason other than its entities or answers doubted (C51e), when the UN resolve or its shape derivation failed upstream, the two were not in within SEARCH_UN_FIRST_PASS_WAIT_S of the start of derivation, or the UN pass does not complete, and with SEARCH_UN_ON_FIRST_MENU on any part whose weak first pass waited for the UN leg and found it failed or not in, and on any part when the UN front door refuses places a read named, such as `country/GRC`, `country/SVN` or `country/ESH` (the catalogue's `un.json` lists them) or its part of a read that also names our variables fails while our part serves, including in a trailing observations or slot-options read, whose warning carries its spec_id in the done event, and whose `params` hold the refused place dcids as `places`, the failed reads' variables as `variables` and each variable's name as `variable_labels`, parallel to `variables`, taken from the members of the Specs the part holds with no graph read, and "" where it holds none (a member the graph names no label for, and every variable on search.options), so a client tells a refusal from a failed read and names the variables without the message, and in search.options diagnostics; `withheld_by_registry` on the request's first part that ends no_data with reason `no_observations` and gate `no_member_has_data_at_place` after electing a subject-only shape, naming the provenances the catalogue withholds that hold data for its elected StatVars at the places the part names (one unfiltered facet read per request at most); no other no-data reason triggers it; `origin_catalogue_stale` in diagnostics when the loaded catalogue is more than 14 days old, added after the response cache so a cached answer reports the catalogue's age when served | implemented | Search has no door that pins a variable dcid (a posted `stat_var` shape refuses), so the pinned-variable trigger applies to Datasets only. Pinned by `tests/engine/test_origin_wire.py` (`test_trailing_observations_report_places_the_un_refused`, `test_refused_places_alone_name_no_variable`) and, for a failed, slow or unreadable UN pool and the UN pass's failures, `tests/engine/test_pool_election.py` and `tests/engine/test_recall_transfer.py` (`test_a_un_resolve_that_never_answers_leaves_a_transferred_answer_intact`). | | C107b | Artifact warnings on search.resolve, one per response at most each: `artifact_pair_stale` in diagnostics when the bound origin catalogue and coverage builds record different `ingestion.workflow_execution_id` values (a build recording none never warns), naming both builds and their ingestions, added after the response cache, whose key holds both builds; `catalogue_index_drift` in diagnostics when, for a StatVar a definite or candidate Spec serves that the packaged code index lists and the catalogue's arc tables hold, the two disagree on the five-tuple, the constraint properties and their values, or the roles (`observationProperties`), naming each such StatVar with the aspects, added before the response cache stores the answer, whose key holds the catalogue build and the build commit that packages the code index. Each also logs one WARNING line, `artifact_pair_stale` or `catalogue_index_drift` followed by a JSON object. Neither fails the request or readiness. `ORIGIN_CATALOGUE_WARN_PAIR_STALE=false` and `ORIGIN_CATALOGUE_WARN_INDEX_DRIFT=false` silence the warning and keep the log line | implemented | Pinned by `tests/engine/test_artifact_checks.py`. | -| C149 | A warning carries `params` only for `observations_series_truncated` (C150), `elected_shape_no_data` (C151), `un_system_unavailable` (C107a), `whole_total_available`, `bilateral_reading_unavailable` (C66g), `currency_unmapped_omitted`, `deflator_missing_omitted` and `deflator_projected`, holding the counts and labels its message states so a client composes its own text without parsing the message; for `currency_unmapped_omitted` and `deflator_missing_omitted` the first ten entities the message counts, with their labels (`""` where the conversion tables hold none, as for most entities no currency names) and, for `deflator_missing_omitted`, the years left out; for `deflator_projected` `{count, examples}`, the rows that took a projected rate or deflator, and at most ten distinct `entity@year` keys among them, sorted, where `count` counts rows, not keys, and `examples` may leave keys out; every other code's warning omits the key | implemented | Pinned by the C150 and C151 tests. | +| C149 | A warning carries `params` only for `observations_series_truncated` (C150), `elected_shape_no_data` (C151), `un_system_unavailable` (C107a), `whole_total_available`, `bilateral_reading_unavailable` (C66g), `currency_unmapped_omitted`, `deflator_missing_omitted`, `deflator_projected` and `role_scope_defaulted`, holding the counts and labels its message states so a client composes its own text without parsing the message; for `currency_unmapped_omitted` and `deflator_missing_omitted` the first ten entities the message counts, with their labels (`""` where the conversion tables hold none, as for most entities no currency names) and, for `deflator_missing_omitted`, the years left out; for `deflator_projected` `{count, examples}`, the rows that took a projected rate or deflator, and at most ten distinct `entity@year` keys among them, sorted, where `count` counts rows, not keys, and `examples` may leave keys out; for `role_scope_defaulted` `{axis}`, the open role axis the answer covers in full, `origin` or `destination`; every other code's warning omits the key | implemented | Pinned by the C150 and C151 tests. | | C150 | `observations_series_truncated` carries `params: {shown, total, order}`: the series the preview ships and the confirmed series behind them, both integers, `shown` < `total`, and `order`, `"ranked"` when the preview read every series and ships the most recent and largest (latest year, then latest value, then identity) or `"fixed"` when the observations the answer keeps in each member's years exceed `role_bulk_rank_max_rows` or the read converts into each entity's own currency and the preview ships a selection made before any value was read, StatVars newest year first and taken round-robin. The message states the same order. A fixed selection takes only series with data in their member's years, StatVars ordered by their newest year with data there. The observations themselves carry the same `order` whenever they show fewer series than the answer holds, and omit it otherwise. The Datasets observations door selects a snapshot share's series by the same rule over its frame's rows and states its own `order` on the observations it serves | implemented | Pinned by `tests/engine/test_bulk_role_series_observations.py`, `tests/engine/test_population_headline.py` and, for the door against the live read on each window, `tests/datasets_api/test_observations_door.py` (`test_the_door_and_the_live_role_bulk_preview_agree_on_each_series_axes`). | | C151 | A part carries at most one `elected_shape_no_data` per election, however many empty readings it dropped, with `params: {no_data, elected}`: the shape labels of the readings with no data and of the readings answered instead, each listed once in first-seen order. A single-sibling re-election names the elected shape and its sibling; a candidates answer of several siblings names the elected shape and each candidate's shape label. A same-shape merge folds its parts' warnings that share a message into one, whose `no_data` and `elected` are the first-seen unions over the parts | implemented | Pinned by `tests/engine/test_zero_observation_election.py`, `tests/engine/test_candidates_measure_sibling_collapse.py`, and `tests/engine/test_conjoin_warnings_and_primary.py`. | | C194 | A price-basis requirement the query names on `measurementQualifier` ("constant", "real", "at constant prices") is met by an answer served under a unit request whose price basis reads as one of the requirement's codes (`requirements._serves_price_basis`): constant prices meet `RealValue` and `InflationAdjusted`, current prices meet `Nominal`, and no unit request meets `PurchasingPowerParity`. A definite answer, or a candidate set every member of which is served so, carries no `query_qualifier_dropped` warning, `measure_not_applied` departure or sentence fact for it. The requirement still elects a publisher's series at that basis where one exists. An answer served under no unit request, or at the other basis, leaves it unmet, so "constant prices" over a series holding only current prices still warns | implemented | Pinned by `tests/engine/test_pinned_code_refusal.py`. | diff --git a/packages/dfo-data-tools/src/code_index/composites.py b/packages/dfo-data-tools/src/code_index/composites.py index 3ce3ebfea..a7f72839c 100644 --- a/packages/dfo-data-tools/src/code_index/composites.py +++ b/packages/dfo-data-tools/src/code_index/composites.py @@ -643,11 +643,22 @@ def refusal(self, composite: str) -> UnclassifiedDestinations | None: ) +class _Checks(dict[int, PopulationCheck]): + """The population checks of one code index built so far, by the `id` of the coverage index + each was built against (`population_check`). A subclass, since a plain `dict` cannot be + weakly referenced.""" + + @per_index -def _checks(index: CodeIndex) -> dict[int, PopulationCheck]: - """The population checks of `index` built so far, by the `id` of the coverage index each - was built against (`population_check`).""" - return {} +def _checks(index: CodeIndex) -> _Checks: + """The population checks of `index` built so far (`_Checks`).""" + return _Checks() + + +def _forget(checks: weakref.ReferenceType[_Checks], key: int) -> None: + """Drop the entry `key` from `checks`, unless its code index has gone with it.""" + if (held := checks()) is not None: + held.pop(key, None) def population_check(index: CodeIndex, coverage: CoverageIndex) -> PopulationCheck: @@ -664,7 +675,7 @@ def population_check(index: CodeIndex, coverage: CoverageIndex) -> PopulationChe Built once per pair of indexes: memoised per code index (`per_index`), then per coverage index within it by `id`. Each coverage index carries one finalizer per code index it was checked against, which drops its entry when the coverage index is collected; the code - index's own memo goes with the code index. The release + index's own memo goes with the code index, which the finalizer holds weakly. The release service builds it when a release opens (`graph_release.service.build_release_views`), so a request reads it from the memo. The mint and Search refuse a composite reading an unavailable population (`PopulationCheck.refusal`).""" @@ -674,7 +685,7 @@ def population_check(index: CodeIndex, coverage: CoverageIndex) -> PopulationChe if found is None: found = _population_check(index, coverage) checks[key] = found - weakref.finalize(coverage, checks.pop, key, None) + weakref.finalize(coverage, _forget, weakref.ref(checks), key) return found diff --git a/packages/dfo-data-tools/src/materialise/bundle_sources.py b/packages/dfo-data-tools/src/materialise/bundle_sources.py index 6685fa6a1..69559a4de 100644 --- a/packages/dfo-data-tools/src/materialise/bundle_sources.py +++ b/packages/dfo-data-tools/src/materialise/bundle_sources.py @@ -61,8 +61,10 @@ def composite_summary(index: CodeIndex, composite: CompositeDeclaration) -> str: the codes it binds and the words of the declared basis it sets (`BASIS_MEASURES`), with the years its periods hold. A component binding its destinations to the bilateral side (`ComponentRecipe.destinations`) states what that reading counts - (`BILATERAL_SOURCES`), once for the whole composite where every component binds them.""" - shared = all(recipe.destinations is not None for recipe in composite.components) + (`BILATERAL_SOURCES`) where it binds the source's published total, once for the whole + composite where its one component does, and beside its own component where there are + several, so a component that adds an imputed share is not worded as the bilateral total.""" + shared = len(composite.components) == 1 and composite.components[0].destinations is not None phrases = [ _component_phrase(index, recipe, reading=not shared) for recipe in composite.components ] @@ -80,10 +82,18 @@ def composite_summary(index: CodeIndex, composite: CompositeDeclaration) -> str: def _destination_reading(recipe: ComponentRecipe) -> str | None: """What a component binding its destinations to the bilateral side counts, as the - `BILATERAL_SOURCES` row of its dataset words it.""" + `BILATERAL_SOURCES` row of its dataset words it, where the component binds that row's + published total (`code` on `code_system`).""" if recipe.destinations is None: return None - return next((row.label for row in BILATERAL_SOURCES if row.dataset == recipe.dataset), None) + return next( + ( + row.label + for row in BILATERAL_SOURCES + if row.dataset == recipe.dataset and recipe.bind.get(row.code_system) == row.code + ), + None, + ) def _component_phrase(index: CodeIndex, recipe: ComponentRecipe, *, reading: bool) -> str: diff --git a/src/datasets_api/kinds/spec_id.py b/src/datasets_api/kinds/spec_id.py index 5d19b7fc4..49c06215d 100644 --- a/src/datasets_api/kinds/spec_id.py +++ b/src/datasets_api/kinds/spec_id.py @@ -82,6 +82,7 @@ from datasets_api.timings import phase from dfo_platform.problem import Refusal from materialise import MaterialisationRequest +from materialise.errors import FacetNotAvailableError from materialise.graph import GraphReader from search.models import TimeWindow from search.role_kinds import RoleKindRefused, check_role_kinds, graph_types @@ -134,7 +135,8 @@ def _resolve_members( scope: Scope | None = None, ) -> tuple[StatVarShape, ...]: """Resolves an identity and attaches `/input/spec_id` to `MemberNotResolvable` and to the - composite refusals (`CompositeUnservable`, `CompositeWindowUnsupported`). + composite refusals (`CompositeUnservable`, `CompositeWindowUnsupported`, and the + off-side `FacetNotAvailableError` of `adapter.composite_components`). Accepts a registered identity or an unregistered sidecar. A `scope` that supplies `facets` replaces the identity's own before resolution, as scope surgery does, and one that supplies @@ -173,7 +175,7 @@ def _resolve_members( except MemberNotResolvable as exc: exc.issue_pointer = _POINTER raise - except (CompositeUnservable, CompositeWindowUnsupported) as exc: + except (CompositeUnservable, CompositeWindowUnsupported, FacetNotAvailableError) as exc: exc.pointer = _POINTER raise return resolved.shapes diff --git a/src/search/README.md b/src/search/README.md index 15a798302..7619886c1 100644 --- a/src/search/README.md +++ b/src/search/README.md @@ -193,7 +193,7 @@ A part starts its pass with the scope its request posts (`stated_scope.part_scop The pass's terminal builds the scope in memory from the records, the release's code index and the request's coverage snapshot (`pipeline._with_scope_choices`). A part with no record carries none. Only the entries the binders applied compile (`AxisRecord.entry`), while the answer's identity holds the option the binder settled (`scope.build.build_part_scope`), so every choice states the option the answer serves. The provider axis is settled over the answer's identity once the binders have run (`scope.provider.settle_provider`). A stated or carried entry applies where its option is available, or for a stated entry partial, and the answer can serve it: the scope it holds, the inclusive composite, or the bilateral scope, which removes that composite (`scope.edits.applies_in_pass`). Otherwise the declared convention applies where the answer can serve it, else the scope the answer holds, else the bilateral scope, else the first available option, so the record holds only an option it marks available. A choice names the convention's rule only where the request matched the convention and the part holds the convention's scope. Otherwise it names the rule of the option it holds (`scope.tables.AxisTable.rules`), which describes that option's own scope: on a part the CRS serves, `bilateral_flows` for the bilateral scope and `bilateral_plus_imputed` for the inclusive one (`code_index.scope_conventions.PROVIDER_SCOPE_RULES`), and on a published total the row's rule. A resubmit matches no convention, so it names the option's own rule. `code_index.scope_conventions.PROVIDER_CONVENTIONS` counts a named donor's total of a CRS sector or recipient part as bilateral flows plus the imputed multilateral share when both tables cover every year the bilateral answer holds. Every convention reads the CRS, so a published donor total matches none and holds the provider scope its DAC1 aid type declares (`code_index.scope_conventions.PUBLISHED_PROVIDER_SCOPES`), under the row's rule, where every provider the part names is a donor: a country, the EU institutions (`DONOR_ORGANISATIONS`), or a total of a group whose membership holds countries alone (the G7, the DAC countries). A part naming any other organisation or total, the DAC's total of the multilaterals among them, or no provider, has no provider record. DAC1's total ODA counts bilateral flows plus core contributions to multilaterals in the year they are paid, so it holds `provider:contributions`, labelled "bilateral plus core contributions to multilaterals", under the rule `contribution_total`. Its `provider:bilateral` option binds DAC1's bilateral ODA (aid type 1015), judged per donor on its basis. A resubmit judges each explicit entry on the identity the other explicit entries' edits produce (`scope.build._requested_tables`), so a request switching to gross and to bilateral flows together judges bilateral ODA on gross and compiles as the two switches do one after the other. A year the total holds is covered when every donor holding the total in that year holds bilateral ODA in it, so a donor holding no total in a year places no demand on it. Where no donor holds the total in any year of the window, the option is available if every donor holds bilateral ODA in the window and unavailable otherwise. Otherwise the option is available where every year the total holds is covered, partial over the covered years where those are fewer, with the latest run of them as the window a switch reads (`SlotBinding.window`), and unavailable where none is, each with `option_not_covered`. On the OECD headline a year counts on the basis the headline reads it on, net flows before 2018 and grant equivalents from 2018. A `latest` read elects one year for the whole pool, so on a part asking for the latest year the option elects bilateral ODA's year among those every donor holding the total in its elected year holds it in, and a switch reads that one year. The option is available where that year is the total's elected year, partial over it, stated in the label, where it is another, and unavailable where the donors share no year or no donor holds the total in its elected year. One coverage scan per table reads every donor's years. Its `provider:inclusive` option is unavailable with `contributions_counted`, since it counts the contributions in place of an imputed share of the multilaterals' spending, with the total's dataset in `detail`. DAC1's bilateral ODA holds `provider:bilateral` under the rule `bilateral_total`, offers `provider:contributions` as total ODA under the same coverage check, and its inclusive option is not offered (`provider_not_offered`, why `dataset`). A pool naming the EU institutions beside an EU member state, or beside a total of a group holding one, has no provider record, since total ODA counts the member's core contributions to the institutions and the institutions' own aid. The members are those the membership policy holds, on any day of the part's window (today for a latest read), for the group `REDISTRIBUTED_PROVIDERS` declares as the institutions' funders, so the United Kingdom counts over any window holding a year before it left in 2020. Where the policy holds no members for that group, every pool with the institutions counts. A pool of the EU institutions and a donor outside the EU has a provider record. A fresh search answers the total it resolved, so a stated "bilateral" on a DAC1 total is reported in `dropped_entries` as `stated_choice_not_applied` with the cause `not_served`, and a switch to it reads bilateral ODA. A stated "including multilateral" is reported with `contributions_counted`. Words naming multilateral ODA state `provider:multilateral` only on a part whose provider table offers it, DAC1's. Elsewhere, as on a part the CRS or DAC2a serves, the entry is dropped before the axis settles (`scope.provider.without_unoffered_multilateral`), so it is neither applied nor reported. The other entries that attach to the part, those on an axis its binders settled and, of those made for a concept, the ones for the concept the axis was asked for, and the stated ones on an axis the part lacks (`ScopeState.attached`), are reported in `dropped_entries`. A carried one is reported as `carried_choice_not_applied` and a stated one as `stated_choice_not_applied`, with the cause (`entry_on_axis`, `axis_absent`, the option's partial or unavailable reason, or `not_served`). -A composite option (the inclusive provider scope, or the OECD headline per year for a donor window crossing 2018) binds a declared composite provenance as the identity's source (`code_index.composites`). When the part settles one, the scope's base is the composite, and the answer is restated as it (`scope.composite.assemble_composite`). The restated answer carries the composite's spec_id and identity, its coverage and headline over the years its components serve, its StatVar counts summed over the components, its provenance facet naming the composite with the source and Data Commons its components share, and `compiled.components` with each component's role, table, years, StatVar count and identity. Its `download` facts count the file a Datasets mint of the composite writes, from the coverage snapshot in memory (`scope.composite.composite_download`), with `rows_exact` false, since the count reads no facet the mint elects. Its observation preview reads each component identity with `resolve_spec_resubmit`, which elects the component's members as the mint does, in worker threads that run together, and composes the previews under the composite's rules (`scope.composite.compose_observations`). Each component keeps the points in the years it serves, under its own StatVar and `source_refs`, and the event or embedded preview carries the composite's spec_id. A streamed request reads it in a trailing task launched once the answer is restated (`region.composite_answer.CompositeRelaunch`), so the `response` event waits on nothing new, and a buffered request reads it inline. The answer phase decides the answer's scope from the identity the Spec will hold before any preview read launches (`region.scope_decision.decide_scope`, over `spec_build.answer_identity`), and the restatement reuses that decision, so a fresh composite answer reads no preview of its table identity, streamed or buffered. A dataset_resubmit resolves the composite's table identity with no preview and reads each component through the part memo. A composite whose components serve no point omits its preview with `observations_no_points`, or the first reason a component's own preview omits with, or `observations_probe_failed` when a component answered nothing. A component read that fails upstream, crosses the deadline or answers nothing leaves its years out and warns `observations_probe_failed`, or `observations_timed_out` for the deadline, naming the component's role, so the answer stands. Each component reads through the request's catalogue under its own pin alone, as the mint reads it, whatever source scope the part read under. A same-shape merge previews the read its merged Spec makes: composite answers, which merge only with each other, answer the composite over the merged identity (`region.composite_answer.as_merged_composite`), whose spec_id, components, years, counts, download facts and preview cover every member's entities and slots, and any other merge reads its preview from the read plan the merge rebuilds in memory, the plan its download counts, as a single answer reads its own, or, where the merge rebuilds no plan, with one resubmit of its merged identity. The inclusive option serves the years both tables cover, read from the coverage snapshot in memory. Over a window the imputed table covers in part it is `partial`, with the latest run of covered years as its own window (`ProviderBinding.window`), which a switch or the request's own words apply. A stated inclusive scope ("including imputed multilateral") is then served over that run, the scope's base keeps the years asked for, so a switch to bilateral answers them, and the part carries a `year_unavailable` departure on `when:`: "Imputed multilateral aid covers only 2011 to 2024, so this shows those years." A carried entry on a partial option is dropped, since only the user may change the period asked for. For a `latest` window it covers the latest year both tables cover: the scope's base keeps the `latest` window, and the composite compiles to that one year (`provider.elect_latest_window`), so a switch to bilateral answers the CRS's own latest year. An option a part cannot serve is not applied as its default, and a donor with no grant-equivalent series keeps net flows. Search elects and resolves a composite over the identity its option was set on, with its first component's population bound (`code_index.composites.table_identity`). That read holds the first component's rows alone, so the counts of what the composite serves read every component. A fresh search does not refuse its scope. Parts a same-shape merge folds state the scope of the merged identity, built as a resubmit of it builds it from the union of the members' entries (`pipeline.merged_scope_choices`). To switch an option, post the scope's `base.identity` as a `dataset_resubmit` identity entry with its `entries` as `scope_entries`, the entry on the switched axis replaced by the switch (an explicit entry whose origin names the choice), so each axis holds one entry. The engine compiles them over the identity, resolves the result, and refuses `requested_option_unavailable` or `conflicting_requested_options` (422) at the scope entry it cannot apply. A sector part's whole, the same dataset over every sector, is asked the same way: post `base.identity` with the slot its sector options bind (`SlotBinding.slot`) set `unbound`, and as `scope_entries` an explicit entry from each other choice holding its active option. The whole then keeps the part's donor, recipient, basis, prices, window and provider scope, or refuses `requested_option_unavailable` at the entry holding an option it cannot serve, such as the inclusive scope for a donor the imputed table carries no rows for. A part whose scope states no sector choice binds no sector and has no whole. +A composite option (the inclusive provider scope, or the OECD headline per year for a donor window crossing 2018) binds a declared composite provenance as the identity's source (`code_index.composites`). When the part settles one, the scope's base is the composite, and the answer's Spec is minted as it (`spec_build.build_spec(composite=)`, `scope.composite.CompositeMint`). The composite answer carries the composite's spec_id and identity, its coverage and headline over the years its components serve, its StatVar counts summed over the components, its provenance facet naming the composite with the source and Data Commons its components share, and `compiled.components` with each component's role, table, years, StatVar count and identity. Its `download` facts count the file a Datasets mint of the composite writes, from the coverage snapshot in memory (`scope.composite.composite_download`), with `rows_exact` false, since the count reads no facet the mint elects. Its observation preview reads each component identity with `resolve_spec_resubmit`, which elects the component's members as the mint does, in worker threads that run together, and composes the previews under the composite's rules (`scope.composite.compose_observations`). Each component keeps the points in the years it serves, under its own StatVar and `source_refs`, and the event or embedded preview carries the composite's spec_id. A streamed request reads it in a trailing task launched once the answer is minted (`region.composite_answer.CompositeRelaunch`), so the `response` event waits on nothing new, and a buffered request reads it inline. The answer phase elects the part's provider reading before the read (`region.reading.elect_reading`), reads the part's table once within the composite's population, mints the Spec as the composite, and builds the scope from the same election (`region.scope_decision.build_part_scope`), all before any preview read launches, so a fresh composite answer reads no preview of its table identity, streamed or buffered. A dataset_resubmit resolves the composite's table identity with no preview and reads each component through the part memo. A composite whose components serve no point omits its preview with `observations_no_points`, or the first reason a component's own preview omits with, or `observations_probe_failed` when a component answered nothing. A component read that fails upstream, crosses the deadline or answers nothing leaves its years out and warns `observations_probe_failed`, or `observations_timed_out` for the deadline, naming the component's role, so the answer stands. Each component reads through the request's catalogue under its own pin alone, as the mint reads it, whatever source scope the part read under. A same-shape merge previews the read its merged Spec makes: composite answers, which merge only with each other, are minted as the composite over the merged identity (`same_shape._merged_composite`), whose spec_id, components, years, counts, download facts and preview cover every member's entities and slots, and any other merge reads its preview from the read plan the merge rebuilds in memory, the plan its download counts, as a single answer reads its own, or, where the merge rebuilds no plan, with one resubmit of its merged identity. The inclusive option serves the years both tables cover, read from the coverage snapshot in memory. Over a window the imputed table covers in part it is `partial`, with the latest run of covered years as its own window (`ProviderBinding.window`), which a switch or the request's own words apply. A stated inclusive scope ("including imputed multilateral") is then served over that run, the scope's base keeps the years asked for, so a switch to bilateral answers them, and the part carries a `year_unavailable` departure on `when:`: "Imputed multilateral aid covers only 2011 to 2024, so this shows those years." A carried entry on a partial option is dropped, since only the user may change the period asked for. For a `latest` window it covers the latest year both tables cover: the scope's base keeps the `latest` window, and the composite compiles to that one year (`provider.elect_latest_window`), so a switch to bilateral answers the CRS's own latest year. An option a part cannot serve is not applied as its default, and a donor with no grant-equivalent series keeps net flows. Search elects and resolves a composite over the identity its option was set on, with its first component's population bound (`code_index.composites.table_identity`). That read holds the first component's rows alone, so the counts of what the composite serves read every component. A fresh search does not refuse its scope. Parts a same-shape merge folds state the scope of the merged identity, built as a resubmit of it builds it from the union of the members' entries (`pipeline.merged_scope_choices`). To switch an option, post the scope's `base.identity` as a `dataset_resubmit` identity entry with its `entries` as `scope_entries`, the entry on the switched axis replaced by the switch (an explicit entry whose origin names the choice), so each axis holds one entry. The engine compiles them over the identity, resolves the result, and refuses `requested_option_unavailable` or `conflicting_requested_options` (422) at the scope entry it cannot apply. A sector part's whole, the same dataset over every sector, is asked the same way: post `base.identity` with the slot its sector options bind (`SlotBinding.slot`) set `unbound`, and as `scope_entries` an explicit entry from each other choice holding its active option. The whole then keeps the part's donor, recipient, basis, prices, window and provider scope, or refuses `requested_option_unavailable` at the entry holding an option it cannot serve, such as the inclusive scope for a donor the imputed table carries no rows for. A part whose scope states no sector choice binds no sector and has no whole. On the SSE stream, every per-variable event (`shape`, `sentence`, `step`) carries `variable_index`, the same index the terminal `response` event's `parts` list is keyed by. Trailing per-spec events (`slot_options`, `entity_options`, `facet_options`, `observations`) join on `spec_id` instead, since a same-shape collapse can put two parts on the same spec. `sentence` events stay keyed by `variable_index`, one per part; the composed envelope sentence only ever rides the terminal `response` event, alongside the rest of the envelope. A part whose shape carries a measure family bind elected gets a first `sentence` event right after bind, naming only its measured thing from that election ("gross disbursements"), without the flow type and the year, followed by a second `sentence` event once materialise resolves the served rows, which adds both and refines that phrase when the served measure family differs from what bind elected. A part with no elected family (most indicators outside the CRS-style financial-flow axes) gets only the post-materialise `sentence` event, because materialise resolves its measured thing from the served StatVar. @@ -207,9 +207,9 @@ The provider axis offers up to three readings of a donor's aid, each a `provider A DAC2a composite reads its destinations as a role open within a population (`code_index.scope_conventions.RolePopulation`). A destination is placed on a side by `code_index.scope_conventions.destination_side`: countries, the organisations the DAC counts as bilateral recipients, the declared regional, income and DAC group totals and the DAC's unallocated lines are `bilateral`, multilateral agencies, the DAC's unnamed multilateral recipients and `DACAllMultilateralRecipients` are `multilateral`, and `DACAllRecipients` is `spanning`. The population's `members` are the bilateral-side parties and residuals, its `named` set adds the bilateral totals, and its `total` is `DACDevelopingCountries`. `code_index.composites._derived_identity` binds it where a component's identity derives, so every reader of rows reads the bound list from the identity and `ComponentIdentity.populations` tags the role for the few readers of names. An open destination binds the members, a named one passes when it is in `named` and refuses with `CompositeNotServable` otherwise, and an identity naming no destination binds the total. -The mint sends a bound population as `dc_sdmx.Among`, so the frame, the coverage, the row ceiling and the manifest count the population alone. Search discovers an open destination within it (`discover_role_scope(within=...)`), and the options door offers no agency and no total of both sides. A fresh answer that settles on the composite is read again within the population, or at the population's total where its identity names no destination (`pipeline._within_population`), so its scope, headline, coverage and warnings equal a resubmit's of its definition. That read supplies the figures, members and warnings alone: the answer keeps the identity, spec_id and scope decided on the request's own identity, so "top donors of ODA" names no destination and keeps its scope. A candidate set's readings are read again concurrently, every part's reads together within the request's concurrency limit (`concurrency.bounded_thread_permits`). A population that holds a destination no declaration places, in the code index or in the request's coverage snapshot, is unavailable: the mint refuses the composite as a server fault, and Search offers neither cut reading (`provider_not_offered`, why `destination_unclassified`) and keeps total ODA. A posted identity binding the composite then refuses 422 `requested_option_unavailable` at the resubmit and candidate doors and `facet_not_available` at the options door, cause `provider_not_offered` naming the unplaced destinations, with the plain DAC2a table as the alternative. +The mint sends a bound population as `dc_sdmx.Among`, so the frame, the coverage, the row ceiling and the manifest count the population alone. Search discovers an open destination within it (`discover_role_scope(within=...)`), and the options door offers no agency and no total of both sides. A fresh part elects its provider reading before it reads (`reading.elect_reading`), so a part that settles on the composite reads its table once, within the population, or at the population's total where its identity names no destination, and its Spec is minted as the composite (`phase_answer._decide_answer_scope`): its scope, headline, coverage and warnings equal a resubmit's of its definition. The composite's identity is the request's own with the elected choice applied, so "top donors of ODA" names no destination and keeps its scope. Each reading a candidate set, a refusal's alternate, a flagged match or a nearest-real suggestion shows elects its own provider reading at its own read (`reading.elect_at_read`) and is minted as the composite that reading names (`reading.decide_read_scope`). A population that holds a destination no declaration places, in the code index or in the request's coverage snapshot, is unavailable: the mint refuses the composite as a server fault, and Search offers neither cut reading (`provider_not_offered`, why `destination_unclassified`) and keeps total ODA. A posted identity binding the composite then refuses 422 `requested_option_unavailable` at the resubmit and candidate doors and `facet_not_available` at the options door, cause `provider_not_offered` naming the unplaced destinations, with the plain DAC2a table as the alternative. -A fresh DAC2a total settles on `provider:bilateral` when it names no destination or only bilateral-side ones and the composite serves every year its answer holds (`provider.bilateral_default_reading`), with or without a provider axis, so a ranking over all donors lists no agency. The scope's base is then the composite, so a switch of any other axis posted over the base keeps the bilateral reading. The composite replaces a provenance pin to DAC2a, so a part pinned to DAC2a and switched to total ODA from the composite reads the unpinned total ODA spec, with the same figures as the pinned one. It holds `provider:contributions` when a stated choice or the part's own destination asks for it. A part whose destination is a multilateral agency reads total ODA: its bilateral and inclusive options are unavailable with `multilateral_destination`, the destination in `detail`, and the answer departs `core_contribution_read`, worded for an agency, for `DACAllMultilateralRecipients` or for the DAC's lines of unnamed agencies. The departure reads the destination's side, so a part naming no donor, which has no provider axis, carries it too. A part bound to `DACAllRecipients` holds total ODA by default; under the bilateral or inclusive scope its destination rebinds to `DACDevelopingCountries`, the bilateral side of all recipients, and the option label says so. A destination no declaration places leaves both options `provider_not_offered`, why `destination_unclassified`. A fresh part with no provider axis that keeps total ODA because its bilateral option is not available warns `bilateral_reading_unavailable`, with the option's reason in `params.why`, since no option is there to say so. A part whose scope decision raised states no scope, so where the default reaches it it carries the warning with `why` `scope_undecided` (`scope_decision.guarded_scope_decision`). +A fresh DAC2a total settles on `provider:bilateral` when it names no destination or only bilateral-side ones and the composite serves every year its answer holds (`provider.bilateral_default_reading`), with or without a provider axis, so a ranking over all donors lists no agency. The scope's base is then the composite, so a switch of any other axis posted over the base keeps the bilateral reading. The composite replaces a provenance pin to DAC2a, so a part pinned to DAC2a and switched to total ODA from the composite reads the unpinned total ODA spec, with the same figures as the pinned one. It holds `provider:contributions` when a stated choice or the part's own destination asks for it. A part whose destination is a multilateral agency reads total ODA: its bilateral and inclusive options are unavailable with `multilateral_destination`, the destination in `detail`, and the answer departs `core_contribution_read`, worded for an agency, for `DACAllMultilateralRecipients` or for the DAC's lines of unnamed agencies. The departure reads the destination's side, so a part naming no donor, which has no provider axis, carries it too. A part bound to `DACAllRecipients` holds total ODA by default; under the bilateral or inclusive scope its destination rebinds to `DACDevelopingCountries`, the bilateral side of all recipients, and the option label says so. A destination no declaration places leaves both options `provider_not_offered`, why `destination_unclassified`. A fresh part with no provider axis that keeps total ODA because its bilateral option is not available warns `bilateral_reading_unavailable`, with the option's reason in `params.why`, since no option is there to say so. `params.why` is one of the reasons dfo-spec-kit's `registries/warning-reasons.json` lists for the code (`provider.WithheldWhy`), and `slot_not_served` adds `params.slot`, the property of the slot the composite cannot serve. A part whose scope decision raised states no scope, so where the default reaches it it carries the warning with `why` `scope_undecided` (`scope_decision.guarded_scope_decision`). A posted identity that binds either DAC2a composite to an agency or to `DACAllRecipients` is refused at Search's doors that read a posted identity's table (`identity_edit.composite_table_identity`), 422 `requested_option_unavailable` at the resubmit and candidate doors and 422 `facet_not_available` at the options door, cause `multilateral_destination`, with the plain DAC2a table as the alternative: a resubmit settles nothing. An identity that names no destination reads `DACDevelopingCountries`. `resolved.scope_choices.compiled.components[].identity` carries the bound list of members. diff --git a/src/search/engine/entry/dataset_entries.py b/src/search/engine/entry/dataset_entries.py index 0b6258873..162867aef 100644 --- a/src/search/engine/entry/dataset_entries.py +++ b/src/search/engine/entry/dataset_entries.py @@ -21,7 +21,6 @@ from entity_resolution import Admission, EntityResolver from materialise import GraphReader from materialise.rate_tables import RateTableCache -from origin_catalogue import current_catalogue from search.engine.binders import Binder from search.engine.clients.graph import EngineGraphClient from search.engine.clients.llm import SupportsLLM @@ -38,12 +37,10 @@ from search.engine.identity_edit import composite_table_identity from search.engine.options.population_memo import OptionsPopulationMemo from search.engine.region import RegionResult, resolve_spec_resubmit -from search.engine.region.composite_answer import ComponentReads, as_composite_answer from search.engine.region.place_options import PlaceOptionsPool from search.engine.region.result import any_degraded from search.engine.region.state import EngineOptions from search.engine.region.whole_totals import with_containing_total -from search.engine.spec_id import identity_spec_id from search.models import ( BaseIdentity, Entry, @@ -284,22 +281,12 @@ async def resolve_candidates_entry( the configured candidate cap before scheduling this coroutine. A candidate posting a composite, as a fresh candidate set's reading of DAC2a's bilateral - destinations is (`composite_answer.with_bilateral_readings`), resolves its table identity - with the populations its roles bind (`identity_edit.composite_table_identity`) and - answers the composite (`composite_answer.as_composite_answer`), over the one year it + destinations is (`reading.decide_read_scope`), resolves its table identity + with the populations its roles bind (`identity_edit.composite_table_identity`) and mints + its Spec as the composite (`ResubmitIdentityPayload.composite`), over the one year it serves for a `latest` window (`scope.provider.elect_latest_window`), as an identity entry posting it does. """ - reads = ComponentReads( - graph=graph, - sdmx=sdmx, - coverage=coverage, - materialise_graph=materialise_graph, - materialise_sdmx=materialise_sdmx, - rate_cache=rate_cache, - deadline=deadline, - catalogue=current_catalogue(), - ) facts = ScopeFacts(index=get_index(), coverage=coverage) def _promote(candidate: ResubmitCandidate) -> RegionResult: @@ -316,11 +303,13 @@ def _promote(candidate: ResubmitCandidate) -> RegionResult: # served bilateral line leave it unchanged. if identity != candidate.identity: table = composite_table_identity(identity, pointer="/input/identity", coverage=coverage) - region = resolve_spec_resubmit( + # A composite is minted as itself (`ResubmitIdentityPayload.composite`). + return resolve_spec_resubmit( inp=ResubmitIdentityPayload( identity=table.identity, entity_groups=candidate.entity_groups, populations=dict(table.populations), + composite=None if table.identity == identity else identity, ), graph=graph, sdmx=sdmx, @@ -335,15 +324,6 @@ def _promote(candidate: ResubmitCandidate) -> RegionResult: ), full_read_deadline=full_read_deadline, ) - if table.identity == identity or region.status != "definite": - return region - # A candidate reads no preview, so the composite's is not read either. - return dataclasses.replace( - as_composite_answer( - region, identity, identity_spec_id(identity), facts=facts, reads=reads - ), - observations_relaunch=None, - ) async def _promote_one(candidate_index: int, candidate: ResubmitCandidate) -> RegionResult: async with sem: diff --git a/src/search/engine/entry/dataset_resubmit.py b/src/search/engine/entry/dataset_resubmit.py index c045aa510..95662ba96 100644 --- a/src/search/engine/entry/dataset_resubmit.py +++ b/src/search/engine/entry/dataset_resubmit.py @@ -49,11 +49,14 @@ (`scope.build.build_part_scope`), resolves the identity they compile to, and carries the scope on its definite part (`_compiled_scope`, `_served_scope`). A composite identity, posted or compiled, resolves the identity its option was set on -(`identity_edit.composite_table_identity`), with no observation preview, and answers as the -composite (`_as_composite`). The composite's preview reads each of its components through the -part memo (`composite_answer`), in a trailing task where the request streams observations -and inline where it embeds them. A posted identity, as a posted candidate, is read as a fresh search of it would serve it -(`dataset_entries.served_posted_identity`): a flow side it leaves open binds its dataset's +(`identity_edit.composite_table_identity`) with no observation preview, mints its Spec as +the composite (`ResubmitIdentityPayload.composite`), and launches none of that table's +option or preview probes (`_as_composite`). The composite's preview reads each of its +components through the part memo (`composite_answer`), in a trailing task where the request +streams observations and inline where it embeds them. A definite part read as its own +identity states the `bilateral_reading_unavailable` warning a fresh answer of it states +(`_with_withheld_reading`). A posted identity, as a posted candidate, is read as a fresh +search of it would serve it (`dataset_entries.served_posted_identity`): a flow side it leaves open binds its dataset's containing total, and an inclusive composite whose providers a fresh search would not offer the inclusive scope answers its bilateral line, its scope reporting the inclusive scope dropped as a carried entry. A @@ -94,6 +97,7 @@ from search.engine.config import settings from search.engine.conjoin import assemble_region from search.engine.deadline import _await_bounded +from search.engine.departures import warning_key from search.engine.enrichment import _launch_trailing_tasks, _stream_slot_options from search.engine.entry.dataset_entries import ( check_candidates_ceiling, @@ -112,7 +116,7 @@ from search.engine.region import RegionResult, resolve_spec_resubmit from search.engine.region.composite_answer import ( ComponentReads, - as_composite_answer, + CompositeRelaunch, with_composite_observations, ) from search.engine.region.place_options import PlaceOptionsPool @@ -121,11 +125,11 @@ any_degraded, with_runnable_operations, ) +from search.engine.region.scope_decision import bilateral_withheld_warning, scope_facts from search.engine.region.sentence_events import answer_events, response_sentence_events from search.engine.region.state import EngineOptions from search.engine.response_cache_flow import PartMemoScope from search.engine.retrieval.bound_values import identity_dataset -from search.engine.spec_id import identity_spec_id from search.models import ( BaseIdentity, DatasetResubmitInput, @@ -145,6 +149,7 @@ from search.scope.compile import Refusal from search.scope.provider import elect_latest_window from search.scope.region import region_groups +from search.scope.state import ScopeState from search.scope.tables import PartFacts, ScopeFacts from series_coverage.shared import CoverageSnapshot @@ -158,13 +163,12 @@ def full_read_deadline_for(deadline: float) -> float: @dataclasses.dataclass(frozen=True, slots=True) class _Promoted: - """An identity entry's promote: the scope its scope entries compiled to, the facts it - read, the identity it answers, the table identity it read, the part memo key it stores + """An identity entry's promote: the scope its scope entries compiled to, the identity it + answers, the table identity it read, the part memo key it stores under, the region, which the part memo held when `stored`, and the ms `_served_entry` took.""" scope: PartScope | None - facts: ScopeFacts identity: BaseIdentity read: BaseIdentity key: str | None @@ -223,23 +227,46 @@ def _compiled_scope( def _as_composite( - region: RegionResult, - identity: BaseIdentity, - *, - scope: PartScope | None, - facts: ScopeFacts, - reads: ComponentReads, + region: RegionResult, identity: BaseIdentity, *, reads: ComponentReads ) -> RegionResult: - """`region`, resolved for composite `identity`'s table identity, answering the composite - (`composite_answer.as_composite_answer`) under its own spec_id, which is the compiled - scope's when `scope` compiled it. It carries the closure that reads the composite's - preview per component over `reads` (`composite_answer.CompositeRelaunch`), over the - components it was restated over (`composite_served`), which `_launch_trailing_tasks` - launches once every promote has returned, or `with_composite_observations` reads inline. - The closure reads the composite's identity alone, so a table identity the part memo held - answers with one too.""" - spec_id = scope.compiled.spec_id if scope is not None else identity_spec_id(identity) - return as_composite_answer(region, identity, spec_id, facts=facts, reads=reads) + """`region`, whose Spec `resolve_spec_resubmit` minted as composite `identity` + (`ResubmitIdentityPayload.composite`), carrying the closure that reads the composite's + preview per component over `reads` (`composite_answer.CompositeRelaunch`), which + `_launch_trailing_tasks` launches once every promote has returned, or + `with_composite_observations` reads inline. The slot-options, place-options and preview + probes of the table identity it read are dropped, so none launches and no table option + event streams under the composite's spec_id. The closure reads the composite's identity + alone, so a region the part memo held answers with one too.""" + assert region.spec.definition.identity == identity, "the resubmit minted another Spec" + return dataclasses.replace( + region, + slot_options_task=None, + slot_options_relaunch=None, + place_options_task=None, + place_options_relaunch=None, + observations_task=None, + observations_relaunch=CompositeRelaunch(spec=region.spec, reads=reads), + ) + + +def _with_withheld_reading( + region: RegionResult, entry: ResubmitIdentityEntry, *, coverage: CoverageSnapshot +) -> RegionResult: + """`region`, a definite answer read as its own identity, carrying the + `bilateral_reading_unavailable` warning a fresh answer of that identity carries where + the bilateral default reaches it and cannot apply on a part with no provider axis + (`scope_decision.bilateral_withheld_warning`), read over the facts its Spec states and + the entry's posted scope entries, so a provider entry asking for total ODA raises none.""" + spec = region.spec + warning = bilateral_withheld_warning( + spec.spec_id, + spec.definition.identity, + selection=ScopeState(entries=tuple(entry.scope_entries or ())), + facts=scope_facts(spec, operations=entry.operations, coverage=coverage), + ) + if warning is None or warning_key(warning) in set(map(warning_key, region.warnings)): + return region + return dataclasses.replace(region, warnings=(*region.warnings, warning)) def _served_scope(scope: PartScope | None, region: RegionResult) -> PartScope | None: @@ -454,11 +481,15 @@ def _promote(index: int, entry: ResubmitIdentityEntry) -> _Promoted: if scope is None or entry.entity_groups is None else region_groups(entry.entity_groups, read, index=get_index()) ) + # A composite is minted as itself, so the part memo keys it apart from its table. payload = ( spec_resubmit_input_for_identity(entry) if scope is None and read is identity else ResubmitIdentityPayload( - identity=read, entity_groups=groups, populations=dict(table.populations) + identity=read, + entity_groups=groups, + populations=dict(table.populations), + composite=None if read is identity else identity, ) ) key = memo.key(payload, read_options) if memo is not None and memo.active else None @@ -466,7 +497,6 @@ def _promote(index: int, entry: ResubmitIdentityEntry) -> _Promoted: if stored is not None: return _Promoted( scope=scope, - facts=facts, identity=identity, read=read, key=None, @@ -487,7 +517,6 @@ def _promote(index: int, entry: ResubmitIdentityEntry) -> _Promoted: raise _rewrite_pointer(exc, index) from exc return _Promoted( scope=scope, - facts=facts, identity=identity, read=read, key=key, @@ -512,6 +541,12 @@ async def _resolve_one(index: int, entry: ResubmitIdentityEntry) -> RegionResult memo.store(promoted.key, region) scope = promoted.scope steps = {"served_entry": promoted.served_entry_ms} + if promoted.read is promoted.identity and region.status == "definite": + # It reads coverage and scans families, so it runs off the event loop. The memo + # stores the region without it, since the posted scope entries decide it. + region = await asyncio.to_thread( + _with_withheld_reading, region, entry, coverage=params.coverage + ) if promoted.read is not promoted.identity and region.status == "definite": started = now_ms() # The components read to the deadline the entry's own reads ran to, the @@ -521,9 +556,7 @@ async def _resolve_one(index: int, entry: ResubmitIdentityEntry) -> RegionResult if region.reads_deadline is None else dataclasses.replace(component_reads, deadline=region.reads_deadline) ) - region = _as_composite( - region, promoted.identity, scope=scope, facts=promoted.facts, reads=reads - ) + region = _as_composite(region, promoted.identity, reads=reads) steps["as_composite"] = now_ms() - started if params.include_observations and not params.stream_observations: region = await with_composite_observations(region, semaphore=sem) diff --git a/src/search/engine/region/candidate_admission.py b/src/search/engine/region/candidate_admission.py index bc9667418..ae716f7e2 100644 --- a/src/search/engine/region/candidate_admission.py +++ b/src/search/engine/region/candidate_admission.py @@ -124,6 +124,8 @@ async def _materialise_candidates( date_request=state.date_request, deadline=state.deadline, coverage=state.coverage, + selection=state.scope_selection, + operations=state.operations, ) merge_presence(state, read.presence) # Every alternate the admission reads shapes the offer. diff --git a/src/search/engine/region/candidate_collapse.py b/src/search/engine/region/candidate_collapse.py index 812bf835a..a204a4766 100644 --- a/src/search/engine/region/candidate_collapse.py +++ b/src/search/engine/region/candidate_collapse.py @@ -56,6 +56,7 @@ _elect_no_data_reason, _ObsProbeResult, ) +from search.engine.region.composite_answer import CompositeRelaunch from search.engine.region.equivalent_series import shared_cluster from search.engine.region.materialise_step import lone_breakdown_absent from search.engine.region.observations.dispatch import ( @@ -63,6 +64,7 @@ PreviewReader, _attach_observations, ) +from search.engine.region.phase_answer import component_reads, failed_closed, finished_scope from search.engine.region.phase_election import judge_answer_coverage from search.engine.region.phase_place_materialise import ( note_denominator_unverified, @@ -92,18 +94,19 @@ settle_flagged, ) from search.engine.region.result import RegionResult -from search.engine.region.scope_decision import decide_guarded from search.engine.region.state import ( + AnswerMark, + MintedComposite, Requirement, ResolveState, ValueRule, raised_facts, unapplied_departure, + whole_totals_as_read, with_requirement, ) from search.engine.region.un_system import member_labels from search.engine.region.value_labels import draft_value_labels, model_only_values -from search.engine.region.whole_totals import whole_total_warnings from search.engine.retrieval.read_plan import ReadPlan, plan_coverage, serves_nothing from search.engine.retrieval.retrieve import Materialised, NoDataDraft from search.engine.retrieval.standard import Narrow @@ -126,14 +129,11 @@ def _offer_place_warnings( ) -> None: """Stamp each spec in `offered`, in menu order, with the place warnings of its own context that the shape in play's do not hold (`stamp_offered_place_warnings`).""" - before = len(state.warnings) stamp_offered_place_warnings( state.warnings, [(spec.spec_id, grounded[spec.spec_id].candidate.place) for spec in offered], departure_facts=state.departure_facts, ) - # Each reading's places are read from the request's words. - state.from_request(*state.warnings[before:]) def _apply_bound_unit_filter( @@ -229,9 +229,11 @@ def _handle_sparse_survivors( # Flagged-unavailable carriers belong in no-data responses. _offer_place_warnings(state, grounded, unique_specs) _settle_lone_survivor(state, grounded, unique_specs[0]) + matched = _shown(state, grounded, unique_specs[0]) return state.no_data( "no_observations", flagged_match=(unique_specs[0],), + matched=matched, gate=NO_MEMBER_HAS_DATA_AT_PLACE, llm_usage=state.var_usage, ) @@ -243,7 +245,6 @@ def _handle_sparse_survivors( state, served_twin.resolved.shape.label ) state.warnings.append(substituted) - state.from_request(substituted) return [served_twin] # The elected reading of a family with no total is one of its breakdowns, so the others # with data answer in its place: one alone, several as candidates. @@ -265,7 +266,6 @@ def _handle_sparse_survivors( no_data=[state.shape_draft.label], elected=[stand_ins[0].resolved.shape.label] ) state.warnings.append(stand_in) - state.from_request(stand_in) return stand_ins if stand_ins: return stand_ins @@ -279,9 +279,11 @@ def _handle_sparse_survivors( ) _offer_place_warnings(state, grounded, unique_specs[:1]) _settle_lone_survivor(state, grounded, unique_specs[0]) + matched = _shown(state, grounded, unique_specs[0]) return state.no_data( _elected_reason, alternate_match=unique_specs[0], + matched=matched, gate=ONLY_ALTERNATE_HAS_DATA, llm_usage=state.var_usage, ) @@ -493,6 +495,8 @@ async def _assemble_definite( unique_specs: list[Spec], grounded: _GroundedCandidates, rate_cache: RateTableCache | None, + *, + mark: AnswerMark, ) -> RegionResult: """Build the definite response for one surviving candidate. A requirement it leaves unmet returns no-data. A no-data elected shape reaches here only through measure-sibling @@ -508,6 +512,16 @@ async def _assemble_definite( unbound query terms other than its coverage records, its own coverage records (`_Candidate.uncovered`), and each code and value requirement its own answer leaves unmet. + + The scope the survivor states is checked once (`phase_answer.finished_scope`), before + its related read is awaited or any of its trailing tasks launch, and the result carries + it (`RegionResult.scope_decision`). A survivor minted as a composite, or read within a + composite's population, that no scope states answers no_data `facet_not_available` + (`phase_answer.failed_closed`), whose shape-derived fields describe the survivor + (`ResolveState.no_data(candidate=)`): the part's warnings, departure facts, + re-election warning and judgement are restored to `mark`, what it had raised and judged + before the candidates were grounded (`assemble_candidates`), and the related read + started for the elected shape is cancelled. """ _entry = grounded[unique_specs[0].spec_id] _survivor = _entry.candidate @@ -515,8 +529,6 @@ async def _assemble_definite( install_place_warnings( state.warnings, _survivor.place, departure_facts=state.departure_facts ) - # The survivor's places are read from the request's words. - state.from_request(*_survivor.place.warnings) _survivor_unbound_qualifiers = settle_answer( state, _survivor.shape, @@ -525,9 +537,11 @@ async def _assemble_definite( uncovered=_survivor.uncovered, ) if _survivor_unbound_qualifiers: + matched = _shown(state, grounded, unique_specs[0]) return state.no_data( "qualifier_not_bound", flagged_match=(unique_specs[0],), + matched=matched, gate=None, llm_usage=state.var_usage, unbound_qualifiers=_survivor_unbound_qualifiers, @@ -536,6 +550,25 @@ async def _assemble_definite( if not unique_specs[0].resolved.members: return state.no_data("no_observations", llm_usage=state.var_usage) _survivor_spec = unique_specs[0] + # The survivor's scope was decided when its Spec was minted, under the reading its own + # read elected (`candidate_grounding._candidate_answer`), so a Spec minted as a composite + # reads its preview per component and none of the table's. The survivor is judged by + # that reading once (`phase_answer.finished_scope`), before its related read, options or + # preview launch, and the result carries the decision for `pipeline._with_scope_choices`. + # A survivor no scope states answers no_data in its place, describing it, with what it + # raised dropped and the elected shape's related read cancelled. + _decision = await finished_scope( + state, + _survivor_spec, + decision=_entry.decision, + minted=_entry.minted, + reading=_survivor.reading, + ) + if _decision is None: + return failed_closed(state, mark, candidate=_survivor) + # Only a decision compiling the composite reads the survivor as it, as on the answer + # phase (`phase_answer._finalize_definite_result`). + _minted = _entry.minted if _decision.composite else None note_denominator_unverified(state, _survivor.shape, spec_id=_survivor_spec.spec_id) if (omitted := _unit_omitted_warning(grounded, _survivor_spec.spec_id)) is not None: state.warnings.append(omitted) @@ -560,94 +593,78 @@ async def _assemble_definite( deadline=state.deadline, coverage=state.coverage, ) - # The scope is decided from the survivor's Spec beside the reads below and before its - # preview launches, so an answer restated as a composite reads no preview of the one - # table it resolved. `pipeline._with_scope_choices` states the decision. - _decision_task = asyncio.create_task( - asyncio.to_thread( - decide_guarded, - _survivor_spec, - selection=state.scope_selection, - operations=state.operations, - coverage=state.coverage, - ) + # The related reads run beside the streamed preview and the options below. + _related_read = related_read_for( + state, + stat_vars=_entry.sv_dcids, + measure_partition=_entry.measure_partition, + places=_survivor.place.roles.roles, + arcs=_survivor.shape.sv_arc_facts, + date_request=_survivor.date_request, ) - try: - # The related reads run beside the streamed preview and the options below. - _related_read = related_read_for( - state, - stat_vars=_entry.sv_dcids, - measure_partition=_entry.measure_partition, - places=_survivor.place.roles.roles, - arcs=_survivor.shape.sv_arc_facts, - date_request=_survivor.date_request, - ) - if state.bound_unit_outcome is not None: - # State the unit lexicon version. A streamed answer also states its preview's - # facets and warning with its immediate response. A flagged-unavailable carrier - # holds nothing to read and states `_UNIT_CHECK_NOT_PROBED`, and a composite - # reads its components' preview once restated. - _survivor_spec = with_unit_lexicon_version(_survivor_spec, state.bound_unit_outcome) - if state.stream_observations: - # An inline preview states its own warning, below. - _streamed = ( - _UNIT_CHECK_NOT_PROBED - if _survivor_spec.resolved.available_at_place is False - or (await _decision_task).composite - else await asyncio.to_thread(_reader.observations) - ) - _survivor_spec = _attach_observations( - _survivor_spec, _streamed, state.warnings, include_points=False - ) - # Keep this timing key identical to the single-shape path. - _options_t0 = now_ms() - _survivor_spec = await _attach_candidate_slot_options(state, grounded, _survivor_spec) - (_survivor_spec,) = await _attach_candidate_facet_options(state, grounded, [_survivor_spec]) - _survivor_spec, _entity_options_raw = await _attach_candidate_entity_options( - state, grounded, _survivor_spec - ) - state.timing["options"] = now_ms() - _options_t0 - _family = [ - *state.family_total_absent_departures((_survivor_spec,)), - *grounded.stale_trailing_departures((_survivor_spec,)), - ] - # Place-option tasks run only when `materialise_graph` is available. - _place_options_materialise_graph = state.materialise_graph - # Each launch reads the identity of the Spec it is given, a merged one included. - _relaunch_place_options_bound = functools.partial( - _relaunch_place_options, - state=state, - place_options_materialise_graph=cast(GraphReader, _place_options_materialise_graph), - ) - _stream_place_options = ( - state.stream_slot_options - and _place_options_materialise_graph is not None - and state.place_options_pool is not None - ) - _place_options_task = ( - _relaunch_place_options_bound(_survivor_spec) if _stream_place_options else None - ) - _survivor_spec = await with_related(_survivor_spec, _related_read) - # No await follows the observation task's launch. Passing it live to `RegionResult` - # keeps it reachable if cancellation follows. - _obs_task: asyncio.Task[_ObsProbeResult] | None = None - _obs_relaunch: Callable[[str], asyncio.Task[_ObsProbeResult]] | None = None - # A composite's preview reads each component once the answer is restated - # (`pipeline._with_composite_preview`). - state.scope_decision = await _decision_task - composite = state.scope_decision.composite - if not composite and state.stream_observations: - _obs_relaunch = _reader.relaunch(member_labels((_survivor_spec,))) - _obs_task = _obs_relaunch(_survivor_spec.spec_id) - elif not composite and state.include_observations: + if state.bound_unit_outcome is not None: + # State the unit lexicon version. A streamed answer also states its preview's + # facets and warning with its immediate response. A flagged-unavailable carrier + # holds nothing to read and states `_UNIT_CHECK_NOT_PROBED`, and a composite + # reads its components' preview once minted. + _survivor_spec = with_unit_lexicon_version(_survivor_spec, state.bound_unit_outcome) + if state.stream_observations: + # An inline preview states its own warning, below. + _streamed = ( + _UNIT_CHECK_NOT_PROBED + if _survivor_spec.resolved.available_at_place is False or _minted is not None + else await asyncio.to_thread(_reader.observations) + ) _survivor_spec = _attach_observations( - _survivor_spec, await asyncio.to_thread(_reader.observations), state.warnings + _survivor_spec, _streamed, state.warnings, include_points=False ) - finally: - # Cancel and await the decision so cancellation cannot detach it. - if not _decision_task.done(): - _decision_task.cancel() - await asyncio.gather(_decision_task, return_exceptions=True) + # Keep this timing key identical to the single-shape path. + _options_t0 = now_ms() + _survivor_spec = await _attach_candidate_slot_options(state, grounded, _survivor_spec) + (_survivor_spec,) = await _attach_candidate_facet_options(state, grounded, [_survivor_spec]) + _survivor_spec, _entity_options_raw = await _attach_candidate_entity_options( + state, grounded, _survivor_spec + ) + state.timing["options"] = now_ms() - _options_t0 + _family = [ + *state.family_total_absent_departures((_survivor_spec,)), + *grounded.stale_trailing_departures((_survivor_spec,)), + ] + # Place-option tasks run only when `materialise_graph` is available. + _place_options_materialise_graph = state.materialise_graph + # Each launch reads the identity of the Spec it is given, a merged one included. + _relaunch_place_options_bound = functools.partial( + _relaunch_place_options, + state=state, + place_options_materialise_graph=cast(GraphReader, _place_options_materialise_graph), + ) + # A composite streams no option of the table it read. + _stream_place_options = ( + state.stream_slot_options + and _minted is None + and _place_options_materialise_graph is not None + and state.place_options_pool is not None + ) + _place_options_task = ( + _relaunch_place_options_bound(_survivor_spec) if _stream_place_options else None + ) + _survivor_spec = await with_related(_survivor_spec, _related_read) + # No await follows the observation task's launch. Passing it live to `RegionResult` + # keeps it reachable if cancellation follows. + _obs_task: asyncio.Task[_ObsProbeResult] | None = None + _obs_relaunch: Callable[[str], asyncio.Task[_ObsProbeResult]] | None = None + # A composite's preview reads each component (`pipeline._with_composite_preview`). + if _minted is not None: + _obs_relaunch = CompositeRelaunch( + spec=_survivor_spec, reads=component_reads(state), served=_minted.mint.served + ) + elif state.stream_observations: + _obs_relaunch = _reader.relaunch(member_labels((_survivor_spec,))) + _obs_task = _obs_relaunch(_survivor_spec.spec_id) + elif state.include_observations: + _survivor_spec = _attach_observations( + _survivor_spec, await asyncio.to_thread(_reader.observations), state.warnings + ) return RegionResult( variable_text=state.variable, status="definite", @@ -656,7 +673,7 @@ async def _assemble_definite( warnings=( *state.warnings, *(warning for warning, _fact in _family), - *whole_total_warnings(_survivor_spec, coverage=state.coverage.index), + *whole_totals_as_read(_survivor_spec, _minted, coverage=state.coverage.index), ), departure_facts={**state.departure_facts, **raised_facts(_family)}, timing_by_step=dict(state.timing), @@ -688,6 +705,7 @@ async def _assemble_definite( # conversions over (`RegionResult.served_members`). served_members=_entry.served_members, served_plans=_entry.served_plans, + scope_decision=_decision, ) @@ -809,7 +827,11 @@ async def _assemble_candidate_set( departure (`assemble._with_readings_substituted`). Each candidate the search could not check against the denominator the question names warns under its own spec_id (`note_denominator_unverified`), as each one listing a side whose total its dataset - publishes does (`whole_totals.whole_total_warnings`). + publishes does (`whole_totals.whole_total_warnings`), read over the identity its rows + were read for (`state.whole_totals_as_read`). Each candidate carries the warning of the + scope decided for it (`ScopeDecision.warning`), as a definite answer of its reading does. + For a reading the bilateral default could not reach, that is + `bilateral_reading_unavailable`. """ unique_specs = [ spec.model_copy( @@ -825,7 +847,6 @@ async def _assemble_candidate_set( if not state.elected_has_data: elected_no_data = _elected_no_data_warning(state, unique_specs) state.warnings.append(elected_no_data) - state.from_request(elected_no_data) await _record_unheld_values( state, [ @@ -882,10 +903,13 @@ async def _assemble_candidate_set( warnings=( *state.warnings, *(warning for warning, _fact in _family), + *_decision_warnings(state.warnings, grounded, _candidate_specs), *( warning for spec in _candidate_specs - for warning in whole_total_warnings(spec, coverage=state.coverage.index) + for warning in whole_totals_as_read( + spec, grounded[spec.spec_id].minted, coverage=state.coverage.index + ) ), ), departure_facts={**state.departure_facts, **raised_facts(_family)}, @@ -902,6 +926,33 @@ async def _assemble_candidate_set( ) +def _shown( + state: ResolveState, grounded: _GroundedCandidates, spec: Spec +) -> tuple[MintedComposite, ...]: + """Record `spec`, the candidate a refusal shows as its flagged or alternate match, as the + reading its read elected: the warning of the scope decided for it + (`_decision_warnings`). Returns the composite it is minted as, which the refusal passes + to `ResolveState.no_data` as `matched`, so its whole-total warnings read the identity its + rows were read for.""" + state.warnings.extend(_decision_warnings(state.warnings, grounded, [spec])) + minted = grounded[spec.spec_id].minted + return () if minted is None else (minted,) + + +def _decision_warnings( + stated: Sequence[Warning], grounded: _GroundedCandidates, specs: Sequence[Spec] +) -> list[Warning]: + """The warning of the scope decided for each of `specs` (`ScopeDecision.warning`), keyed + to its own spec_id, leaving out any `stated` already carries. Two candidates whose + readings the bilateral default cannot reach for the same reason each carry their own.""" + found: list[Warning] = [] + for spec in specs: + warning = grounded[spec.spec_id].decision.warning + if warning is not None and warning not in stated and warning not in found: + found.append(warning) + return found + + async def assemble_candidates( state: ResolveState, admission: _AlternatesAdmission, @@ -923,6 +974,7 @@ async def assemble_candidates( state.place, state.walked, state.date_request, + reading=state.reading, ) ) # Each candidate beside the elected answer is judged on the members its own read @@ -953,9 +1005,16 @@ async def assemble_candidates( state.place, state.walked, state.date_request, + reading=state.reading, ) ) + # The answer is a candidate's from here: everything grounding and the collapse raise or + # judge is the lone survivor's, which a survivor that fails closed restores + # (`_assemble_definite`, `phase_answer.failed_closed`). The ledger records each value + # the request's words name (`_record_unheld_values`) in `state.requirements`, which the + # mark leaves alone, so that record stands. + mark = state.mark_answer() # Grounding reads the graph synchronously for every scope member, so it runs off the event # loop, where the request deadline and other requests keep running. grounded = await asyncio.to_thread(ground_candidates, state, candidates) @@ -1031,13 +1090,7 @@ async def assemble_candidates( unique_specs = [_elected_left] # The discovered-scope warning speaks for the candidates the response keeps. - restate_scope_warning( - state, - [ - (grounded[s.spec_id].candidate.mat, grounded[s.spec_id].scope_members) - for s in unique_specs - ], - ) + restate_scope_warning(state, [grounded[s.spec_id] for s in unique_specs]) if len(unique_specs) == 1: _note_lone_breakdown(state, grounded, unique_specs[0]) # The lone reading, a twin or a nominated rival the quantity or unit filter left @@ -1047,7 +1100,7 @@ async def assemble_candidates( # labels. lone = grounded[unique_specs[0].spec_id].candidate await _record_unheld_values(state, [(lone.shape, lone.bindings)]) - return await _assemble_definite(state, unique_specs, grounded, rate_cache) + return await _assemble_definite(state, unique_specs, grounded, rate_cache, mark=mark) return await _assemble_candidate_set( state, unique_specs, diff --git a/src/search/engine/region/candidate_grounding.py b/src/search/engine/region/candidate_grounding.py index b706b501b..33b0aac8b 100644 --- a/src/search/engine/region/candidate_grounding.py +++ b/src/search/engine/region/candidate_grounding.py @@ -74,6 +74,7 @@ role_served_filter, ) from search.engine.region.place_roles import request_named_groups +from search.engine.region.reading import decide_read_scope from search.engine.region.requirements import answer_terms from search.engine.region.result import FacetAxisFilter, answered_roles from search.engine.region.scope import ( @@ -82,7 +83,8 @@ scope_axis, stated_scope_members, ) -from search.engine.region.state import ResolveState +from search.engine.region.scope_decision import ScopeDecision +from search.engine.region.state import MintedComposite, ResolveState from search.engine.region.trailing import settle_trailing from search.engine.retrieval.bound_values import ( MeasureFamilyCacheKey, @@ -659,6 +661,11 @@ class _GroundedCandidate: `spec` is the Spec as minted, with its unit conversion offer. The collapse rewrites the Specs it is handed (basis, equivalent readings, options, lexicon version) under the same `spec_id`, so it reads only the facts here. + + A Spec minted as a composite (`minted`) reads its preview per component, so it carries + no probe inputs, since the table's slot options would stream under the composite's + spec_id. + Its scope lists the composite's parties (`CompositeMint.parties`). """ spec: Spec @@ -695,6 +702,10 @@ class _GroundedCandidate: unit_coverage: UnitCoverage # The bound-unit gate's verdict on the plan its answer serves (`_CandidateConversion`). unit_gate_reason: str | None + # The scope decided for the Spec and the composite it is minted as, where its scope + # compiles one (`_CandidateAnswer`). + decision: ScopeDecision + minted: MintedComposite | None @property def spec_id(self) -> str: @@ -767,11 +778,22 @@ def _carrier_flags(cand: _Candidate, *, elected: bool) -> _CarrierFlags: class _CandidateAnswer(NamedTuple): - """The identity a candidate's Spec is minted as (`spec_build.answer_identity`) and the - facet bindings it holds.""" + """The identity a candidate's Spec is minted from (`spec_build.answer_identity`), the + facet bindings it holds, the scope decided for it under the provider reading its read + elected (`reading.decide_read_scope`) and the composite its Spec is minted as, + where that scope compiles one. `decision` is None where the candidate was read within a + composite's population and no scope minting that composite could be decided.""" identity: AnswerIdentity facet_bindings: dict[str, IdentityFacetBinding] | None + decision: ScopeDecision | None + minted: MintedComposite | None + + @property + def spec_id(self) -> str: + """The composite's spec_id where the candidate's Spec is minted as one, else the + identity's.""" + return self.identity.spec_id if self.minted is None else self.minted.mint.spec_id def _candidate_answer( @@ -781,9 +803,14 @@ def _candidate_answer( groundwork: _CandidateGroundwork, prep: _CandidatePrep, ) -> _CandidateAnswer: - """The identity candidate `cand_index`'s Spec will be minted as, read before the rest - of its Spec is assembled, so a candidate whose `spec_id` an earlier one holds can be - skipped at no further cost.""" + """The identity candidate `cand_index`'s Spec will be minted from and the scope decided + for it, read before the rest of its Spec is assembled, so a candidate whose `spec_id` an + earlier one holds can be skipped at no further cost. + + The scope applies the provider reading the candidate's own read elected + (`_Candidate.reading`), as the definite answer's applies its read's + (`reading.decide_read_scope`), so a candidate read within a composite's population + is minted as that composite and labelled by the reading that read its rows.""" facet_bindings: dict[str, IdentityFacetBinding] | None = ( # A bound outcome guarantees a unit request. The type checker cannot carry that # attribute invariant through this access. @@ -810,7 +837,20 @@ def _candidate_answer( pin_served_provenance=state.pin_served_provenance, named_groups=request_named_groups(state, cand.place.roles), ) - return _CandidateAnswer(identity=identity, facet_bindings=facet_bindings) + scoped = decide_read_scope( + identity, + cand.reading, + selection=state.scope_selection, + operations=state.operations, + coverage=state.coverage, + catalogue=state.request_catalogue, + ) + return _CandidateAnswer( + identity=identity, + facet_bindings=facet_bindings, + decision=scoped.decision, + minted=scoped.minted, + ) def _finish_candidate( @@ -828,8 +868,9 @@ def _finish_candidate( rivalled: bool, answer: _CandidateAnswer, ) -> _GroundedCandidate: - """Assemble one candidate's Spec, minted as `answer` (`_candidate_answer`), and the - metadata used by candidate collapse. + """Assemble one candidate's Spec, minted as `answer` (`_candidate_answer`), as the + composite it names where its scope compiles one, and the metadata used by candidate + collapse. `rivalled` holds when the status rule offers a structural or nominated rival beside the elected reading. @@ -895,7 +936,10 @@ def _finish_candidate( cand_role_axis_facts = role_axis_facts_for_materialised( cand_mat, shape=cand_shape, plan=groundwork.plan ) + decision, minted = answer.decision, answer.minted + assert decision is not None, "a candidate with no decided scope is not finished" cand_spec = build_spec( + composite=None if minted is None else minted.mint, shape=groundwork.shape_model, slots=groundwork.slots, stat_vars=groundwork.stat_vars, @@ -960,10 +1004,7 @@ def _finish_candidate( plans=(groundwork.plan,) if groundwork.plan is not None else (), ) # `build_spec` mints the Spec as `answer`, so it holds the id the dedup checked. - assert cand_spec.spec_id == answer.identity.spec_id, ( - cand_spec.spec_id, - answer.identity.spec_id, - ) + assert cand_spec.spec_id == answer.spec_id, (cand_spec.spec_id, answer.spec_id) coverage = prep.unit_coverage[cand_index] unit_refusals = dict(prep.unit_refusals[cand_index]) if coverage.refusal is not None: @@ -981,7 +1022,7 @@ def _finish_candidate( # Retain inputs for both buffered and SSE probes. probe_slot_options=( tuple(groundwork.slot_options_raw) - if state.stream_slot_options or state.include_option_coverage + if minted is None and (state.stream_slot_options or state.include_option_coverage) else None ), role_axis_facts=tuple(cand_role_axis_facts), @@ -989,16 +1030,30 @@ def _finish_candidate( measure_siblings=cand_measure_siblings, open_axis_facts=cand_open_axis_facts, bulk_scope=bulk_scope_for(cand.place, cand_mat, sdmx=state.sdmx), - scope_members=( - tuple(cand_scope_member_dcids) if cand_scope_member_dcids is not None else None - ), + scope_members=_listed_members(cand_spec, cand_scope_member_dcids, minted=minted), elected=prep.elected_raw[cand_index], unit_refusals=unit_refusals, unit_coverage=coverage, unit_gate_reason=prep.unit_gate_reasons[cand_index], + decision=decision, + minted=minted, ) +def _listed_members( + spec: Spec, members: Sequence[str] | None, *, minted: MintedComposite | None +) -> tuple[str, ...] | None: + """The members `spec`'s scope lists: `members`, the scope its plan serves, else, for a + Spec minted as a composite, the composite's parties its scope lists + (`CompositeMint.open_axes`).""" + if members is None: + return None + scope = spec.resolved.scope + if minted is None or scope is None: + return tuple(members) + return tuple(member.ref for member in scope.members) + + @dataclass(frozen=True) class _GroundedCandidates: """The kept candidates' records (`_GroundedCandidate`) by `spec_id`, one per Spec, in @@ -1046,36 +1101,40 @@ def scopes_on_axis( } -def restate_scope_warning( - state: ResolveState, scopes: Sequence[tuple[Materialised, Sequence[str] | None]] -) -> None: +def restate_scope_warning(state: ResolveState, entries: Sequence[_GroundedCandidate]) -> None: """Restate the elected shape's discovered-scope warning for the candidates a response - states, each given as its carrier and the scope its Spec lists - (`restate_role_scope_warnings`). The warning states a count only when every scope on its - axis lists the same members (`scopes_on_axis`), and otherwise names every member with - data. Grounding restates it over every candidate, and the collapse over the candidates - the unit filter keeps, so a lone survivor's warning counts its own scope, as its - resubmit does.""" + states (`entries`), each with the scope its Spec lists (`restate_role_scope_warnings`). + The warning states a count only when every scope on its axis lists the same members + (`scopes_on_axis`), and otherwise names every member with data. Grounding restates it + over every kept candidate, and the collapse over the candidates the unit filter keeps, + so a lone survivor's warning counts its own scope, as its resubmit does. Where the + elected candidate's Spec is minted as a composite, the count is the composite's parties + (`CompositeMint.role_scope_warnings`), as the definite answer's is.""" if state.opposite_labels is None or not isinstance(state.mat_result, Materialised): return axis = scope_axis(state.mat_result) on_axis = [ - members - for candidate, members in scopes - if members is not None and scope_axis(candidate) == axis + entry.scope_members + for entry in entries + if entry.scope_members is not None and scope_axis(entry.candidate.mat) == axis ] if not on_axis: return distinct = scopes_on_axis( - [candidate for candidate, _members in scopes], - [members for _candidate, members in scopes], + [entry.candidate.mat for entry in entries], + [entry.scope_members for entry in entries], axis=axis, ) - own = next((members for candidate, members in scopes if candidate is state.mat_result), None) + own = next((entry for entry in entries if entry.candidate.mat is state.mat_result), None) + if own is not None and own.minted is not None and len(distinct) <= 1: + state.warnings[:] = own.minted.mint.role_scope_warnings(state.warnings, own.spec) + return restate_role_scope_warnings( state.warnings, state.mat_result, - members=own if own is not None else on_axis[0], + members=own.scope_members + if own is not None and own.scope_members is not None + else on_axis[0], boundary_roles=state.place.roles.roles, label_of=state.opposite_labels, unnamed_wholes=state.place.roles.unnamed_wholes, @@ -1098,10 +1157,6 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro """ prep = _prepare_candidate_context(state, candidates) candidates = prep.candidates - restate_scope_warning( - state, - [(cand.mat, members) for cand, members in zip(candidates, prep.scope_members, strict=True)], - ) answer_t0 = now_ms() measure_family_cache: dict[MeasureFamilyCacheKey, tuple[MeasurePartition, ...]] = {} # These forced qualifier, measure, and default bindings belong only to candidates that @@ -1120,7 +1175,9 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro answer = _candidate_answer(state, cand_index, cand, groundwork, prep) # A candidate whose Spec an earlier one holds adds no Spec or record, and its Spec is # not assembled. It follows the candidate it repeats, so it is never the elected one. - if answer.identity.spec_id in entries: + # One read within a composite's population that cannot be minted as it is dropped, + # since the table's label would state the population's rows (`reading.decide_read_scope`). + if answer.decision is None or answer.spec_id in entries: continue entry = _finish_candidate( state, @@ -1140,6 +1197,7 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro if cand.mat.family_total_absent is not None: state.family_total_absent[entry.spec_id] = cand.mat.family_total_absent + restate_scope_warning(state, list(entries.values())) _warn_for_candidates(state, prep, groundworks) return _GroundedCandidates(entries=entries, answer_t0=answer_t0) diff --git a/src/search/engine/region/candidates.py b/src/search/engine/region/candidates.py index 0f6393625..33a5615ac 100644 --- a/src/search/engine/region/candidates.py +++ b/src/search/engine/region/candidates.py @@ -11,7 +11,7 @@ import dataclasses import logging from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import NamedTuple from dc_graph import GraphCapExceededError, GraphDeadlineExceededError, GraphReadError @@ -38,6 +38,7 @@ unread_context, with_where, ) +from search.engine.region.reading import ElectedReads, ReadingDecision, Within from search.engine.region.result import RegionResult from search.engine.region.state import ( AlternateUnevaluatedReason, @@ -78,11 +79,13 @@ from search.engine.source_scope import SourceScope from search.models import ( CandidateBasis, + Operation, ResolvedQualifier, SpecObservations, Warning, omitted_observations, ) +from search.scope.state import ScopeState from series_coverage.shared import CoverageSnapshot logger = logging.getLogger(__name__) @@ -114,7 +117,12 @@ class _Candidate: `date_request` is the request its Spec states and its option coverage and related measures read: the part's, until candidate grounding settles a trailing span on the newest years its plan serves (`candidate_grounding._settled_trailing`), which also sets - `stale_departure` when that newest year is stale (`trailing.stale_departure`).""" + `stale_departure` when that newest year is stale (`trailing.stale_departure`). + + `reading` is the provider reading its read elected at the place it settled on + (`reading.elect_at_read`), None where it binds no total with a bilateral source. A + reading that names a composite was read within the composite's population, and the + candidate's Spec is minted as that composite (`candidate_grounding._candidate_answer`).""" shape: ShapeDraft bindings: list[SlotBindingDraft] @@ -125,6 +133,7 @@ class _Candidate: basis: CandidateBasis | None = None uncovered: tuple[ResolvedQualifier, ...] | None = None stale_departure: Departed | None = None + reading: ReadingDecision | None = field(kw_only=True) def _prewarm_places( @@ -410,11 +419,20 @@ def _materialise_alt_shapes( date_request: DateRequest | None, deadline: float | None = None, coverage: CoverageSnapshot, + selection: ScopeState, + operations: Sequence[Operation], over_bound: list[_Alternate] | None = None, presence_failure: AlternateUnevaluatedReason | None = None, ) -> _AlternatesRead: """Materialise alternates (`elect_alternate`), each in its own place context. + Each read elects its provider reading first, at the place it tries, under the part's + provider entries `selection` and `operations`, and reads the composite the reading names + within its population, as the elected shape's read does (`reading.ElectedReads`). The + candidate carries the election of the place it + settled on (`_Candidate.reading`), and a reading whose composite refuses its table read + reads nothing, so the alternate is dropped as one without data. + ``places`` holds each alternate's places, in ``alternates`` order, for the shape the alternate holds. An alternate whose place sets are unsettled settles them on its own read (`place_context.settle_sets`): its where slot, role entities and probe places come @@ -508,34 +526,45 @@ def _materialise_alt_shapes( presences = [presence] def read( - place: PlaceContext, + read_at: PlaceContext, facet_batch: FacetBatch | None, + within: Within, alternate: _Alternate = alternate, bindings: list[SlotBindingDraft] = bindings, presences: list[ConsultedPresence | None] = presences, ) -> Materialised | NoDataDraft | MaterialisedCandidates | UnsupportedDraft: return materialise( alternate.shape, - with_where(bindings, place), - place.roles.resolved_entities, + with_where(bindings, read_at), + read_at.roles.resolved_entities, graph=graph, sdmx=sdmx, date_request=date_request, deadline=deadline, - place_dcids=place.probe_places, - default_presence=place.presence_for(presences[-1]), + place_dcids=read_at.probe_places, + default_presence=read_at.presence_for(presences[-1]), discover_scope=True, - open_axes=place.roles.open_axes, + open_axes=read_at.roles.open_axes, coverage=coverage, facet_batch=facet_batch, + within=within, ) + reads = ElectedReads( + alternate.shape, + bindings, + date_request=date_request, + selection=selection, + coverage=coverage, + operations=operations, + read=read, + ) try: # The alternate's values come from its representative StatVar, none from the # request, so values that match no member together are unbound silently, a # minimal set per round (`BoundValuesUnrealisable.relaxation`). while True: - place, result = settle_sets(unsettled, read) + place, result = settle_sets(unsettled, reads) if not isinstance(result, BoundValuesUnrealisable): break bindings[:] = without_values(bindings, result.relaxation()) @@ -585,14 +614,16 @@ def read( if isinstance(result, Materialised) and result.has_data: if alternate.unverified: result = dataclasses.replace(result, membership_unverified=True) + read_at, reading = reads.settled(place) results.append( _Candidate( settled_shape(alternate.shape, result), - with_where(bindings, place), + with_where(bindings, read_at), result, - place.settled(result), + read_at.settled(result), alternate.walked, date_request, + reading=reading, ) ) return _AlternatesRead(results, None, presence, tuple(unevaluated), consulted) diff --git a/src/search/engine/region/composite_answer.py b/src/search/engine/region/composite_answer.py index bb1fa0fb0..bc1591eb8 100644 --- a/src/search/engine/region/composite_answer.py +++ b/src/search/engine/region/composite_answer.py @@ -1,9 +1,9 @@ """The observation preview of a composite answer, read per component as the Datasets mint reads a composite. -A composite answer is resolved for its table identity and restated as the composite -(`scope.composite.assemble_composite`), so the one table that resolve read previews nothing of -the composite. Its preview reads each component identity +A composite answer is minted as the composite (`spec_build.build_spec(composite=)`, +`scope.composite.CompositeMint`) over the read its elected reading names, so the one table that +read resolved previews nothing of the composite. Its preview reads each component identity (`code_index.composites.component_identities`) with `resolve_spec_resubmit`, which elects its members as the mint does and scopes its reads to the provenance it binds, in worker threads that run together. Each component read embeds its preview @@ -15,34 +15,31 @@ composed under the composite's rules (`scope.composite.compose_observations`). A streamed request reads the preview in a trailing task (`CompositeRelaunch`), -launched once the answer is restated, so it adds nothing before the `response` event. A +launched once the answer is minted, so it adds nothing before the `response` event. A buffered request reads it inline (`with_composite_observations`). A same-shape merged answer previews the read of its merged identity -(`same_shape._with_merged_preview`). A merge of composite answers is restated as the -composite over that identity (`as_merged_composite`) and reads its components -(`CompositeRelaunch`). Any other merge whose read plan the merge rebuilds in memory reads that -plan (`PlanPreview`), so its preview and its download come from one plan, and the rest take -one resubmit of the merged identity (`IdentityPreview`). A component read and a resubmitted +(`same_shape._with_merged_preview`). A merge of composite answers is minted as the +composite over that identity and reads its components (`CompositeRelaunch`). Any other merge +whose read plan the merge rebuilds in memory reads that plan (`PlanPreview`), so its preview and +its download come from one plan, and the rest take one resubmit of the merged identity +(`IdentityPreview`). A component read and a resubmitted merged read are both `_resubmit_preview`, so they elect, cap and fail alike. """ from __future__ import annotations import asyncio -import contextvars import dataclasses import logging from collections.abc import Sequence -from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext from dataclasses import dataclass -from typing import TYPE_CHECKING, cast, get_args +from typing import TYPE_CHECKING, cast import dc_graph from code_index.shared import get_index from dc_sdmx import SdmxClient as MaterialiseSdmxClient -from dfo_platform.roles import role_key_axis from materialise import GraphReader from materialise.rate_tables import RateTableCache from origin_catalogue import Catalogue, bound_catalogue, recording_un_blocked_places @@ -50,21 +47,17 @@ from search.engine.clients.sdmx import EngineSdmxClient from search.engine.concurrency import ( bounded_semaphore, - bounded_thread_permits, create_owned_task, ) -from search.engine.config import settings from search.engine.degraded_reads import ( LOST_READ_ERRORS, note_degraded_read, note_lost_read, recording_degraded_reads, ) -from search.engine.departures import FactsByWarning, WarningKey, warning_key from search.engine.envelope import now_ms from search.engine.errors import EngineInputError from search.engine.events import ObservationsEvent -from search.engine.identity_edit import composite_table_identity from search.engine.region.codes import ( OBSERVATIONS_BULK_FETCH_FAILED, OBSERVATIONS_CLIENTS_UNAVAILABLE, @@ -78,7 +71,6 @@ OBSERVATIONS_TIMED_OUT, UN_SYSTEM_UNAVAILABLE, _ObsProbeResult, - _OptionProbeResult, ) from search.engine.region.observations.dispatch import ( PreviewRead, @@ -89,26 +81,14 @@ from search.engine.region.resubmit import resolve_spec_resubmit from search.engine.region.result import ( RegionResult, - recounted_scope, - with_composite_answer, - with_restated_keys, -) -from search.engine.region.scope_decision import ( - ScopeDecision, - bilateral_withheld_warning, - decide_guarded, ) from search.engine.region.state import EngineOptions from search.engine.region.un_system import un_unavailable_warning from search.engine.response_cache_keys import is_region_storable -from search.engine.spec_id import identity_spec_id from search.models import ( BaseIdentity, - Component, EntityGroup, EntityRoleKey, - NoDataReason, - Operation, ResubmitIdentityPayload, Spec, SpecObservations, @@ -122,13 +102,10 @@ ) from search.scope.composite import ( ServedComponent, - assemble_composite, compose_observations, served_components, ) -from search.scope.provider import bilateral_default_reading, elect_latest_window from search.scope.region import region_groups -from search.scope.state import ScopeState from search.scope.tables import ScopeFacts from series_coverage.shared import CoverageSnapshot @@ -137,7 +114,6 @@ from code_index.scope_conventions import RolePopulation from search.engine.response_cache_flow import PartMemoScope - from search.heading.departure_fact import DepartureFact logger = logging.getLogger(__name__) @@ -205,7 +181,7 @@ def composite_served(identity: BaseIdentity, *, reads: ComponentReads) -> list[S """Each component of composite `identity` with the years it serves (`scope.composite.served_components`), read through `reads.catalogue` with facts of its own, so no source scope a part's own reads ran under narrows them. The answer the - composite is restated as (`assemble_composite`) and its preview read these same + composite is minted as (`scope.composite.CompositeMint`) and its preview read these same components.""" facts = ScopeFacts(index=get_index(), coverage=reads.coverage) document = identity.model_dump(mode="json", exclude_none=True) @@ -411,7 +387,7 @@ class CompositeRelaunch: composed preview, each keyed to it. It reads `spec`'s identity and entity groups alone, so an answer the part memo held - builds it as a fresh resolve does. `served` holds the components the answer was restated + builds it as a fresh resolve does. `served` holds the components the answer was minted over (`composite_served`), so the preview filters by the years the answer states. With `served` None the preview reads `spec`'s components itself.""" @@ -561,697 +537,6 @@ async def _preview_events( return (event.stating_degraded(lost.degraded),), tuple(warnings) -def population_table_answer( - spec: Spec, identity: BaseIdentity, *, reads: ComponentReads -) -> RegionResult | None: - """The answer a resubmit of composite `identity` resolves for its table identity, where - that read binds a population (`identity_edit.composite_table_identity`, - `TableRead.populations`) or the population's total in place of a destination `identity` - leaves out (`RolePopulation.total`): `spec`, an answer resolved over the whole table, - read again with each population role open within its members (`resolve_spec_resubmit`), - posting `spec`'s entity groups, through `reads.catalogue`. A fresh answer discovers its - open roles before its scope settles on the composite, and a merge rebuilds its Spec from - its members', so its members, scope, headline, coverage and warnings count every party - the table holds; this read counts the composite's population alone, as the resubmit door - does (`dataset_resubmit`). It answers whatever the resubmit answers, no_data included, - where the population holds no row in the window. - - Where the composite refuses `identity`, as it does a population that is unavailable, - the read answers no_data, `facet_not_available`, and logs the refusal at ERROR: a fresh - search does not refuse its scope, and the whole table's read would state every party - under the composite's spec_id. A read that fails answers no_data too - (`_failed_population_read`), so no error of a read the request did not post reaches - its caller. None where the table read binds neither, so the caller reads `spec` as the - one table it resolved.""" - try: - table = composite_table_identity( - identity, pointer="/input/identity", coverage=reads.coverage - ) - except EngineInputError as exc: - logger.error( - "population read of %s as %s: the composite refuses it (%s): %s", - spec.spec_id, - identity_spec_id(identity), - exc.code, - exc, - ) - return _unread_population("facet_not_available") - if not table.populations and not _binds_left_out_role(identity, table.identity): - return None - groups = cast("Mapping[EntityRoleKey, EntityGroup] | None", spec.resolved.entity_groups) - payload = ResubmitIdentityPayload( - identity=table.identity, - entity_groups=None - if groups is None - else region_groups(groups, table.identity, index=get_index()), - populations=dict(table.populations), - ) - try: - with bound_catalogue(reads.catalogue): - return resolve_spec_resubmit( - inp=payload, - graph=reads.graph, - sdmx=reads.sdmx, - coverage=reads.coverage, - options=EngineOptions( - materialise_graph=reads.materialise_graph, - materialise_sdmx=reads.materialise_sdmx, - rate_cache=reads.rate_cache, - deadline=reads.deadline, - ), - ) - except (EngineInputError, *LOST_READ_ERRORS) as exc: - return _failed_population_read(spec, identity, exc) - - -def _failed_population_read(spec: Spec, identity: BaseIdentity, exc: BaseException) -> RegionResult: - """The no_data answer of `spec`'s population read as composite `identity` where the read - raised `exc`. A refusal of the posted read (`EngineInputError`) answers its own code where - that code is a no_data reason, else `facet_not_available`: the request never posted the - identity the refusal points into. A lost read (`degraded_reads.LOST_READ_ERRORS`) is noted - (`note_lost_read`) and answers degraded, `timed_out` where it crossed its deadline - (`TimeoutError`, `GraphDeadlineExceededError`), else `no_observations`.""" - about = f"population read of {spec.spec_id} as {identity_spec_id(identity)}" - if isinstance(exc, EngineInputError): - logger.warning("%s refused (%s): %s", about, exc.code, exc) - reason = exc.code if exc.code in _NO_DATA_REASONS else "facet_not_available" - return _unread_population(cast("NoDataReason", reason)) - note_lost_read(about, exc) - timed_out = isinstance(exc, TimeoutError | dc_graph.GraphDeadlineExceededError) - logger.warning("%s lost: %s: %s", about, type(exc).__name__, exc, exc_info=not timed_out) - return _unread_population("timed_out" if timed_out else "no_observations", degraded=True) - - -_NO_DATA_REASONS: frozenset[str] = frozenset(get_args(NoDataReason)) - - -def _unread_population(reason: NoDataReason, *, degraded: bool = False) -> RegionResult: - """A population read answering no_data for `reason`, `degraded` where it lost a read.""" - return RegionResult( - variable_text=None, - status="no_data", - specs=(), - no_data_reason=reason, - warnings=(), - timing_by_step={}, - degraded=degraded, - ) - - -def _binds_left_out_role(identity: BaseIdentity, read: BaseIdentity) -> bool: - """Whether `read`, the table identity of composite `identity`, binds a flow axis that - `identity` neither names nor opens: the population's total a component binds in place - of a destination the identity leaves out (`RolePopulation.total`).""" - asked = {role_key_axis(key) for key in identity.entities} | set(identity.open_roles or ()) - return any(role_key_axis(key) not in asked for key in read.entities) - - -def with_population_read( - fresh: RegionResult, answered: RegionResult, *, request_warnings: Sequence[Warning] -) -> RegionResult: - """`answered`, the population read of `fresh`'s definite answer - (`population_table_answer`), definite or not, carrying what `fresh` holds from the - request rather than from the rows it read, so every figure, member, plan and warning the - read derives is `answered`'s, and a field this leaves out costs wording, not figures: - - - `variable_text`, `earliest_index` and `llm_usage`: the request's words for the part, - its position among the request's parts, and the model calls its words took; - - `timing_by_step`: the steps the request's pass ran, the population read included; - - `warnings`: `request_warnings`, those the pass raised before it interpreted its read - (`ResolveState.request_warnings`), then `answered`'s, every one about the answer keyed - to `answered`'s Spec where `fresh`'s were keyed to its own (`_grafted_warnings`); - - `departure_facts`: those of `request_warnings`, then `answered`'s; - - `reads_deadline` and `promote_relaunch`: the request door's own deadline, and the - relaunch facts only a resubmit door's part memo binds, which a fresh part has none of; - - `degraded`: also set where the pass swallowed a read before the population read. - - Every field the request's pass sets once its phases return (its words, entity words, - judged requirements, scope) is set on this result by that pass. `fresh`'s trailing tasks - read the whole table, so the caller cancels them (`result.cancel_dropped_tasks`).""" - kept = {warning_key(warning) for warning in request_warnings} - reading = fresh.spec.spec_id - warnings = _grafted_warnings( - fresh.warnings, - answered, - reading=reading, - keyed_to=answered.spec.spec_id if answered.status == "definite" else reading, - request_warnings=request_warnings, - own=True, - ) - return dataclasses.replace( - answered, - variable_text=fresh.variable_text, - earliest_index=fresh.earliest_index, - llm_usage=fresh.llm_usage, - timing_by_step=fresh.timing_by_step, - warnings=warnings, - departure_facts={ - **{key: fact for key, fact in fresh.departure_facts.items() if key in kept}, - **answered.departure_facts, - }, - reads_deadline=fresh.reads_deadline, - promote_relaunch=fresh.promote_relaunch, - degraded=fresh.degraded or answered.degraded, - ) - - -def _grafted_warnings( - held: Sequence[Warning], - answered: RegionResult, - *, - reading: str, - keyed_to: str, - request_warnings: Sequence[Warning], - own: bool, -) -> tuple[Warning, ...]: - """`held`, a fresh region's warnings, with those about the reading under spec_id - `reading` replaced by `answered`'s, its read within its population, and every warning - about the reading keyed to `keyed_to`, so none names a Spec the response does not hold. - - A warning of `held` is about the reading when keyed to `reading`, or keyed to none where - the reading is the region's `own`, its definite answer or first candidate. Of those, the - ones in `request_warnings`, raised before the pass interpreted its read - (`ResolveState.request_warnings`), stand; the rest came from the whole table's rows. A - warning of `answered` is about the reading when keyed to the Spec it answers, whose - spec_id is its own table identity's and may differ from `reading`, or to `reading`, or to - none. A warning keyed to none on the region's own reading stays keyed to none.""" - request = set(map(warning_key, request_warnings)) - read = answered.spec.spec_id if answered.status == "definite" else None - - def about(warning: Warning) -> bool: - return warning.spec_id == reading or (own and warning.spec_id is None) - - def keyed(warning: Warning) -> Warning: - if warning.spec_id is None and own: - return warning - if warning.spec_id not in (None, reading, read) or warning.spec_id == keyed_to: - return warning - return warning.model_copy(update={"spec_id": keyed_to}) - - stood = ( - keyed(warning) if about(warning) else warning - for warning in held - if not about(warning) or warning_key(warning) in request - ) - return (*stood, *map(keyed, answered.warnings)) - - -@dataclass(frozen=True, slots=True, kw_only=True) -class RestatedReading: - """A reading restated as the composite it is served as (`restated_reading`).""" - - composite: Spec - # The composite's components (`CompiledScope.components`). - components: tuple[Component, ...] - # The closure that reads the composite's preview per component. - relaunch: CompositeRelaunch - - -def restated_reading( - table: Spec, - identity: BaseIdentity, - spec_id: str, - *, - facts: ScopeFacts, - reads: ComponentReads, -) -> RestatedReading: - """`table`, a reading resolved for composite `identity`'s table identity, restated as the - composite under `spec_id` (`scope.composite.assemble_composite`), over the components - `identity` serves (`composite_served`), read through `reads.catalogue` with the part's - `facts`, with the closure that reads its preview per component (`CompositeRelaunch`). - - Every reading a part shows is restated here: its definite answer (`with_decided_scope`), - each reading a candidate set or refusal shows (`with_bilateral_readings`), a set left - with one (`_collapsed`), a same-shape merge (`as_merged_composite`) and an identity - entry (`as_composite_answer`). A reading whose composite binds a population is read - within it first (`population_table_answer`), so `table` counts the population alone.""" - served = composite_served(identity, reads=reads) - with bound_catalogue(reads.catalogue): - composite, components = assemble_composite( - table, identity, spec_id, served=served, facts=facts - ) - return RestatedReading( - composite=composite, - components=tuple(components), - relaunch=CompositeRelaunch(spec=composite, reads=reads, served=served), - ) - - -def with_decided_scope( - result: RegionResult, decision: ScopeDecision, *, reads: ComponentReads -) -> RegionResult: - """`result`, a definite answer, stating the scope `decision` decided for it - (`RegionResult.scope_choices`) and carrying the decision's warning once - (`ScopeDecision.warning`). Where the decision compiles a composite, the answer is - restated as it (`restated_reading`, `result.with_composite_answer`) over `reads`, the - compiled scope listing the composite's components. A fresh definite answer - (`pipeline._with_scope_choices`) and a candidate set left with one reading (`_collapsed`) - both state their scope here.""" - if decision.warning is not None and warning_key(decision.warning) not in set( - map(warning_key, result.warnings) - ): - result = dataclasses.replace(result, warnings=(*result.warnings, decision.warning)) - built = decision.scope - if built is None: - return result - if not decision.composite: - return dataclasses.replace(result, scope_choices=built) - restated = restated_reading( - result.spec, - built.compiled.identity, - built.compiled.spec_id, - facts=decision.facts, - reads=reads, - ) - compiled = built.compiled.model_copy(update={"components": list(restated.components)}) - return dataclasses.replace( - with_composite_answer(result, restated.composite, observations_relaunch=restated.relaunch), - scope_choices=built.model_copy(update={"compiled": compiled}), - ) - - -def as_composite_answer( - region: RegionResult, - identity: BaseIdentity, - spec_id: str, - *, - facts: ScopeFacts, - reads: ComponentReads, -) -> RegionResult: - """`region`, a definite answer resolved for composite `identity`'s table identity, - answering the composite under `spec_id` (`restated_reading`).""" - restated = restated_reading(region.spec, identity, spec_id, facts=facts, reads=reads) - return with_composite_answer( - region, restated.composite, observations_relaunch=restated.relaunch - ) - - -def with_bilateral_readings( - region: RegionResult, - *, - reads: ComponentReads, - selection: ScopeState, - facts: ScopeFacts, - operations: Sequence[Operation] = (), - request_warnings: Sequence[Warning] = (), -) -> RegionResult: - """`region`, a fresh search's candidate set or refusal, with each reading it shows that - is not its definite answer read as a fresh definite answer is: a candidate, the alternate - a refusal offers in its place, each flagged match and each nearest real suggestion, - restated as the composite of DAC2a's bilateral destinations where the default settles it - there under the part's provider entries `selection` holds - (`scope.provider.bilateral_default_reading`), so a stated or carried - `provider:contributions` keeps total ODA, over the one year it serves for a `latest` - window (`scope.provider.elect_latest_window`). A client can fetch any of them by its - spec_id, so none serves the agencies the definite answer leaves out. `facts` are the - part's (`ScopeFacts`), which every reading's rules and restatement share. Every reading - is restated by `restated_reading`, through `reads.catalogue`, the request's catalogue - before any part's source scope narrowed it, as the definite answer is. - A reading whose composite binds a population is read again within it first - (`population_table_answer`), once per spec_id, the readings concurrently - (`_population_reads`), so its scope, headline, coverage and warnings count the - population alone: of the warnings about it, those in `request_warnings` - (`ResolveState.request_warnings`) stand and the read's replace the rest, keyed to the - reading (`_grafted_warnings`), and its `role_scope_defaulted` warning counts the - composite's parties (`result.recounted_scope`). A reading the default reaches that reads - total ODA because the default could not apply, on a part with no provider axis, carries - the `bilateral_reading_unavailable` warning a definite answer would - (`scope_decision.bilateral_withheld_warning`). A reading whose population holds no row in - the window, or whose composite refuses it, is dropped, never restated over the whole - table. A set left with no candidate refuses with the reason its first such read gives, - and a set left with one answers as a definite answer of that reading does (`_collapsed`). - - Each restated reading takes the composite's spec_id, and the warnings, unpublished-whole - facts and assumption echoes keyed to the table's spec_id are keyed to it - (`result.with_restated_keys`). A candidate set keeps its trailing slot options task, whose - events for a restated reading the caller withholds on the event loop - (`without_restated_options`). Where the region's own reading (`RegionResult.spec`) is - restated, the closures that would launch its slot options, place options and preview - again read the table under the composite's spec_id, so the region carries none of them, - nor the facts a part memo hit binds them from (`promote_relaunch`), as a composite answer - carries none (`result.with_composite_answer`). `region`, with any such warning, where no - reading is restated.""" - identities: dict[str, BaseIdentity | None] = {} - renamed: dict[str, Spec] = {} - tables: dict[str, Spec] = {} - - def served_as(spec: Spec) -> BaseIdentity | None: - """The composite identity `spec` is served as, None where it is served as itself.""" - if spec.spec_id not in identities: - identity, _ = bilateral_default_reading(spec.definition.identity, selection, facts) - identities[spec.spec_id] = ( - None - if identity is spec.definition.identity - else elect_latest_window(identity, facts) - ) - return identities[spec.spec_id] - - shown: list[Spec] = [ - *region.specs, - *([region.alternate_match] if region.alternate_match is not None else ()), - *(region.flagged_match or ()), - *(region.nearest_real or ()), - ] - with bound_catalogue(reads.catalogue): - pending = { - spec.spec_id: (spec, identity) - for spec in shown - if (identity := served_as(spec)) is not None - } - withheld = { - spec.spec_id: warning - for spec in shown - if served_as(spec) is None - and ( - warning := bilateral_withheld_warning( - spec.spec_id, spec.definition.identity, selection=selection, facts=facts - ) - ) - is not None - } - if withheld: - region = dataclasses.replace(region, warnings=(*region.warnings, *withheld.values())) - answered = _population_reads(pending, reads=reads) - - def restated[S: Spec](spec: S) -> S | None: - identity = served_as(spec) - if identity is None: - return spec - reading = spec.spec_id - read = answered[reading] - if read is not None: - if read.status != "definite": - return None - # The reading keeps its own class (`NearestRealSpec`) and the facts it carries. - spec = spec.model_copy( - update={"definition": read.spec.definition, "resolved": read.spec.resolved} - ) - composite = restated_reading( - spec, identity, identity_spec_id(identity), facts=facts, reads=reads - ).composite - renamed[reading] = composite - tables[reading] = spec - return cast("S", composite) - - def each[S: Spec](specs: Sequence[S]) -> tuple[S, ...]: - return tuple(kept for spec in specs if (kept := restated(spec)) is not None) - - if region.status == "candidates" and len(region.specs) > 1: - survivors = [ - spec - for spec in region.specs - if (read := answered.get(spec.spec_id)) is None or read.status == "definite" - ] - if len(survivors) == 1: - (survivor,) = survivors - return _collapsed( - region, - survivor, - served_as(survivor), - answered.get(survivor.spec_id), - reads=reads, - selection=selection, - operations=operations, - request_warnings=request_warnings, - ) - specs = each(region.specs) - alternate = region.alternate_match - if alternate is not None: - alternate = restated(alternate) - flagged = region.flagged_match - if flagged is not None: - flagged = each(flagged) or None - nearest = region.nearest_real - if nearest is not None: - nearest = each(nearest) or None - unread = [read for read in answered.values() if read is not None and read.status != "definite"] - if not renamed and not unread: - return region - dropped = frozenset(reading for reading, read in answered.items() if read in unread) - warnings, departure_facts = _without_dropped(region.warnings, region.departure_facts, dropped) - for reading, read in answered.items(): - if read is not None and reading not in dropped: - warnings = _grafted_warnings( - warnings, - read, - reading=reading, - keyed_to=reading, - request_warnings=request_warnings, - own=bool(region.specs) and region.spec.spec_id == reading, - ) - departure_facts.update(read.departure_facts) - for reading, composite in renamed.items(): - own = bool(region.specs) and region.spec.spec_id == reading - warnings = recounted_scope(warnings, tables[reading], composite, own=own) - read_again = dataclasses.replace(region, warnings=warnings, departure_facts=departure_facts) - restated_region = dataclasses.replace( - with_restated_keys(read_again, renamed), - specs=specs, - alternate_match=alternate, - flagged_match=flagged, - nearest_real=nearest, - ) - if region.specs and not specs: - # Every reading the set offered holds no row within its population. - restated_region = dataclasses.replace( - restated_region, status="no_data", no_data_reason=unread[0].no_data_reason - ) - if not region.specs or region.spec.spec_id not in renamed.keys() | answered.keys(): - return restated_region - return dataclasses.replace( - restated_region, - slot_options_relaunch=None, - place_options_relaunch=None, - observations_relaunch=None, - promote_relaunch=None, - ) - - -def _without_dropped( - warnings: Sequence[Warning], - departure_facts: FactsByWarning, - dropped: frozenset[str], -) -> tuple[tuple[Warning, ...], dict[WarningKey, DepartureFact]]: - """`warnings` without those keyed to a reading in `dropped`, and `departure_facts` - without the facts only those warnings disclosed (`departures.warning_key`).""" - kept = tuple(warning for warning in warnings if warning.spec_id not in dropped) - held = set(map(warning_key, kept)) - gone = {warning_key(w) for w in warnings if w.spec_id in dropped} - held - return kept, {key: fact for key, fact in departure_facts.items() if key not in gone} - - -def _population_reads( - pending: Mapping[str, tuple[Spec, BaseIdentity]], *, reads: ComponentReads -) -> dict[str, RegionResult | None]: - """The population read of each reading `pending` maps by spec_id to its table answer and - the composite identity it is served as (`population_table_answer`). Two or more run - concurrently, each in a worker thread over a copy of the caller's context, and every - part's reads together take at most the request's concurrency limit - (`SearchSettings.max_variable_concurrency`) at once (`concurrency.bounded_thread_permits`). - They take no permit of the request semaphore, which the part's own leg holds one of while - it resolves (`fanout`), so a request whose legs held every permit would wait on itself.""" - if len(pending) < 2: - return { - reading: population_table_answer(spec, identity, reads=reads) - for reading, (spec, identity) in pending.items() - } - permits = bounded_thread_permits() - - def read( - context: contextvars.Context, spec: Spec, identity: BaseIdentity - ) -> RegionResult | None: - with permits: - return context.run(lambda: population_table_answer(spec, identity, reads=reads)) - - workers = min(len(pending), settings().max_variable_concurrency) - with ThreadPoolExecutor(max_workers=workers) as pool: - futures = { - reading: pool.submit(read, contextvars.copy_context(), spec, identity) - for reading, (spec, identity) in pending.items() - } - return {reading: future.result() for reading, future in futures.items()} - - -def _collapsed( - region: RegionResult, - survivor: Spec, - served: BaseIdentity | None, - read: RegionResult | None, - *, - reads: ComponentReads, - selection: ScopeState, - operations: Sequence[Operation], - request_warnings: Sequence[Warning], -) -> RegionResult: - """`region`, a candidate set left with one reading, `survivor`, once the others' reads - held no row within their populations (`with_bilateral_readings`), answering as a fresh - definite answer of `survivor` does: its scope decided on its own identity under the - part's `selection` and `operations` (`scope_decision.decide_guarded`), and, where that - decision compiles a composite, read within the composite's population - (`with_population_read`) and restated as it (`with_decided_scope`). `read` is the - population read of the composite `served` the survivor was shown as, which serves where - the decision compiles that composite; any other composite is read again. A decision - compiling no composite states its scope on `survivor` itself. The warnings keyed to the - dropped readings go with them.""" - warnings, departure_facts = _without_dropped( - region.warnings, - region.departure_facts, - frozenset(spec.spec_id for spec in region.specs) - {survivor.spec_id}, - ) - definite = dataclasses.replace( - region, - status="definite", - specs=(survivor,), - alternate_match=None, - flagged_match=None, - nearest_real=None, - warnings=warnings, - departure_facts=departure_facts, - ) - decision = decide_guarded( - survivor, selection=selection, operations=operations, coverage=reads.coverage - ) - built = decision.scope - if decision.composite and built is not None: - identity = built.compiled.identity - if served != identity: - read = population_table_answer(survivor, identity, reads=reads) - if read is not None: - definite = with_population_read(definite, read, request_warnings=request_warnings) - if definite.status != "definite": - return definite - decision = dataclasses.replace(decision, spec_id=definite.spec.spec_id) - return with_decided_scope(definite, decision, reads=reads) - - -def without_restated_options(before: RegionResult, after: RegionResult) -> RegionResult: - """`after`, the restatement of `before` (`with_bilateral_readings`), with the trailing - slot options task `before` launched read without the events and warnings of each reading - it restated: they read the table the reading resolved, under a spec_id the set does not - show, and a composite reading carries none, as a composite answer carries none - (`result.with_composite_answer`). The other readings keep theirs. The task is wrapped on - the event loop, so the caller runs this there. `after` itself where it restated no - reading or holds another task.""" - task = after.slot_options_task - if task is None or task is not before.slot_options_task: - return after - withheld = frozenset(spec.spec_id for spec in before.specs) - { - spec.spec_id for spec in after.specs - } - if not withheld: - return after - return dataclasses.replace( - after, slot_options_task=create_owned_task(_without_readings(task, withheld)) - ) - - -async def _without_readings( - task: asyncio.Task[_OptionProbeResult], withheld: frozenset[str] -) -> _OptionProbeResult: - """The events and warnings `task` yields, without those of the spec_ids `withheld`.""" - events, warnings = await task - return ( - tuple(event for event in events if event.spec_id not in withheld), - tuple(warning for warning in warnings if warning.spec_id not in withheld), - ) - - -def as_merged_composite( - region: RegionResult, group: Sequence[RegionResult], *, reads: ComponentReads -) -> RegionResult: - """`region`, the same-shape merge of `group`, composite answers each, restated as the - composite over the merged identity, as a resubmit of that composite answers it - (`dataset_resubmit._as_composite`). - - The merge rebuilds the merged Spec from its members' slots, which bind no composite, so - its identity is the table identity over every member's entities. The composite's - identity binds there the members' facets, the composite provenance among them, which the - merge gate holds every member to. Its served components (`composite_served`), years, - counts, headline and download facts (`assemble_composite`) and its preview - (`CompositeRelaunch`) read that identity over `reads`, the request's component reads, - to the later of their deadline and the latest any member's reads ran to - (`RegionResult.reads_deadline`). The headline names the request's window as the members' - do, since the gate holds every member to that window. The warnings the merge keyed to - the table identity's spec_id are keyed to the composite's; the caller has dropped the - members' own preview warnings (`same_shape._with_merged_preview`). The observation task the merge carried over from its first member is left to - the combiner's reconciliation (`enrichment._reconcile_after_collapse`), which cancels - the members' tasks and launches the composite's where the request streams - observations. A composite binding a population restates the merged table identity read - within it (`population_table_answer`), as a resubmit of the merged composite reads it, - since the merged Spec is rebuilt from its members' slots and plans. Where that read - holds no row or refuses, the merge answers its no_data, as a reading the read drops - does (`with_bilateral_readings`), without the warnings keyed to the merged Spec.""" - member = group[0].spec - deadlines = [ - deadline - for deadline in (reads.deadline, *(r.reads_deadline for r in group)) - if deadline is not None - ] - if deadlines: - reads = dataclasses.replace(reads, deadline=max(deadlines)) - merged = region.spec - identity = merged.definition.identity.model_copy( - update={"facets": member.definition.identity.facets} - ) - spec_id = identity_spec_id(identity) - answered = population_table_answer(merged, identity, reads=reads) - if answered is not None and answered.status != "definite": - # Each member was read within the population, so a merged read holding no row there - # contradicts them; the merge answers it, never the whole table restated. - logger.warning( - "merged population read of %s answered %s (%s)", - merged.spec_id, - answered.status, - answered.no_data_reason, - ) - warnings, departure_facts = _without_dropped( - region.warnings, region.departure_facts, frozenset({merged.spec_id}) - ) - return dataclasses.replace( - region, - status="no_data", - specs=(), - no_data_reason=answered.no_data_reason, - warnings=warnings, - departure_facts=departure_facts, - degraded=region.degraded or answered.degraded, - slot_options_task=None, - slot_options_relaunch=None, - place_options_task=None, - place_options_relaunch=None, - observations_task=None, - observations_relaunch=None, - ) - if answered is not None: - table = answered.spec - else: - table = merged.model_copy( - update={ - "resolved": merged.resolved.model_copy( - update={"headline": member.resolved.headline} - ) - } - ) - restated = restated_reading( - table, - identity, - spec_id, - facts=ScopeFacts(index=get_index(), coverage=reads.coverage), - reads=reads, - ) - warnings = tuple( - warning.model_copy(update={"spec_id": spec_id}) - if warning.spec_id == merged.spec_id - else warning - for warning in region.warnings - ) - return with_composite_answer( - dataclasses.replace(region, warnings=warnings, observations_task=None), - restated.composite, - observations_relaunch=restated.relaunch, - ) - - async def with_composite_observations( region: RegionResult, *, semaphore: asyncio.Semaphore | None ) -> RegionResult: diff --git a/src/search/engine/region/grounding.py b/src/search/engine/region/grounding.py index e073c5463..5b2ea28d2 100644 --- a/src/search/engine/region/grounding.py +++ b/src/search/engine/region/grounding.py @@ -84,6 +84,7 @@ ) from search.engine.spec_build import ( NamedGroup, + answer_identity, bind_when_slot, build_shape_model, build_slot, @@ -102,6 +103,7 @@ GraphRef, IdentityUnitRequest, NearestRealSpec, + Operation, ProvenanceRef, Relaxation, RequestedWindow, @@ -114,10 +116,13 @@ Warning, ) from search.observations import provenance_refs +from search.scope.state import ScopeState if TYPE_CHECKING: + from origin_catalogue import Catalogue from search.engine.place import ConsultedPresence from search.engine.region.place_context import PlaceContext, UnsettledPlace + from search.engine.region.reading import Within from search.engine.retrieval.read_plan import ReadPlan from search.engine.retrieval.retrieve import Materialised from search.engine.retrieval.role_scope import DiscoveredScope @@ -343,7 +348,8 @@ def role_scope_defaulted_message( def role_scope_defaulted_warning(axis: EntityRoleKey, message: str) -> Warning: """The `role_scope_defaulted` warning on open `axis` stating `message`, carrying its axis - in `params`, by which a composite answer restates its count (`result.recounted_scope`).""" + in `params`, by which a composite counts the warning it states in place of the table's + (`scope.composite.CompositeMint.role_scope_warnings`).""" return Warning( code=ROLE_SCOPE_DEFAULTED, severity="warn", message=message, params={"axis": axis} ) @@ -487,10 +493,11 @@ def restate_role_scope_warnings( `mat` so it counts `members`, the scope the answer lists under the request's facet filter (`scope.stated_scope_members`), or with `state_count` false names every member with data, for a response whose answers list different scopes. The warning is the - `role_scope_defaulted` one opening with the same axis clause ("No recipient was named"), - whatever count it states, so a response restates it once its candidates are grounded - and again once they are filtered. It keeps its position, and `label_of`, the memoised - lookup that built it (`memoised_labels`), makes no new request. + `role_scope_defaulted` one on the same axis, found by its code and `params` axis + (`role_scope_defaulted_warning`), whatever count it states, so a response restates it + once its candidates are grounded and again once they are filtered. It keeps its + position, and `label_of`, the memoised lookup that built it (`memoised_labels`), makes + no new request. An empty `members` is an answer that lists no member on the axis: it covers none, so the warning is removed. An axis in `unnamed_wholes` built no warning @@ -512,9 +519,9 @@ def restate_role_scope_warnings( # The discovered axis's warning leads. A companion axis's warning states no count, so it # stands as built. restated: Warning | None = fresh[0] if members or not state_count else None - axis_clause = f"{_role_scope_axis_clause(mat.discovered_scope.axis)}," + axis = mat.discovered_scope.axis for index, old in enumerate(warnings): - if old.code == ROLE_SCOPE_DEFAULTED and old.message.startswith(axis_clause): + if old.code == ROLE_SCOPE_DEFAULTED and (old.params or {}).get("axis") == axis: if restated is None: del warnings[index] else: @@ -1060,6 +1067,11 @@ class _RelaxationProbeContext: # Whether the part posted source mentions, so a suggestion's identity binds the # provenances that serve it (`spec_build.build_spec`). pin_served_provenance: bool = False + # What each relaxation elects its provider reading from (`reading.elect_reading`), and + # the catalogue its composite is minted under (`reading.decide_read_scope`). + scope_selection: ScopeState = dataclasses.field(default_factory=ScopeState) + operations: Sequence[Operation] = () + request_catalogue: Catalogue | None = None @dataclasses.dataclass(slots=True, frozen=True) @@ -1072,6 +1084,11 @@ class SuggestionRead: plan: ReadPlan | None place: PlaceContext member_facts: Mapping[str, MemberFacts] | None + # Whether the suggestion is minted as a composite (`reading.decide_read_scope`), whose + # rows are its components', not the table this read holds. + composite: bool = False + # The suggestion's own warnings, keyed to its spec_id (`reading.ReadScope.warnings`). + warnings: tuple[Warning, ...] = () def _relaxation_place( @@ -1217,43 +1234,58 @@ def _relaxed_binding_suggestion( resolved response intact. """ from search.engine.region.place_context import settle_sets, settled_shape, with_where + from search.engine.region.reading import ElectedReads, decide_read_scope from search.engine.retrieval.retrieve import Materialised from search.engine.retrieval.role_resolve import graph_confirm_resolve ground_warnings: list[Warning] | None = None probe_warning: Warning | None = None - def read(place: PlaceContext, facet_batch: FacetBatch | None) -> object: - """The relaxed query's own resolve in `place`. A standard shape resolves as the + def read(read_at: PlaceContext, facet_batch: FacetBatch | None, within: Within) -> object: + """The relaxed query's own resolve at `read_at`, the context the provider reading the + relaxed bindings elect reads in (`reading.ElectedReads`), as the answer's read does: + a composite reads within its population, where a standard shape resolves as the standard route answers it, over the context's entities and places or its - default-scope presence read; a role-bearing shape confirms against coverage.""" + default-scope presence read, and a role-bearing shape confirms against coverage.""" if is_role_bearing(shape): return graph_confirm_resolve( shape=shape, bindings=relaxed, - subject_dcids=_relaxation_places(place), - origin_dcids=place.roles.origin_dcids, + subject_dcids=_relaxation_places(read_at), + origin_dcids=read_at.roles.origin_dcids, graph=ctx.graph, sdmx=ctx.sdmx, date_request=ctx.date_request, deadline=ctx.deadline, - open_axes=place.roles.open_axes, + open_axes=read_at.roles.open_axes, coverage_snapshot=ctx.coverage, + within=within, ) return standard_resolve( shape=shape, bindings=relaxed, - entities=place.roles.resolved_entities, + entities=read_at.roles.resolved_entities, graph=ctx.graph, date_request=ctx.date_request, - place_dcids=place.probe_places, - default_presence=place.presence_for(ctx.presence), + place_dcids=read_at.probe_places, + default_presence=read_at.presence_for(ctx.presence), facet_batch=facet_batch, ) + reads = ElectedReads( + shape, + relaxed, + date_request=ctx.date_request, + selection=ctx.scope_selection, + coverage=ctx.coverage, + operations=ctx.operations, + read=read, + ) + try: with fail_open_reads("nearest-real suggestion", budget_s=settings().nearest_real_budget_s): - bound, mat = settle_sets(_relaxation_place(ctx, shape), read) + tried_place, mat = settle_sets(_relaxation_place(ctx, shape), reads) + bound, reading = reads.settled(tried_place) if not isinstance(mat, Materialised): return None, None, None, None # A flagged-unavailable carrier represents the original binding. @@ -1396,7 +1428,52 @@ def read(place: PlaceContext, facet_batch: FacetBatch | None) -> object: ), ) + facets = answer_facets( + mat.read_plan, + labels, + bound_axes={}, + request_basis=None, + unit_request=None, + currency_of=functools.partial(unit_currency, ctx.graph, deadline=ctx.deadline), + ) + open_axis_facts = served_open_axes( + relaxed, + mat.sv_dcids, + mat.member_facts or {}, + value_labels=shape.value_labels or {}, + ) + requested_window = requested_window_from_date_request(ctx.date_request) + answer = answer_identity( + shape_model, + slots, + stat_vars_list, + entities_list, + identity_entities=None, + open_roles=mat.open_axes, + requested_window=requested_window, + facet_bindings=None, + facets=facets, + resolved_sources=source_refs, + plan=mat.read_plan, + pin_served_provenance=ctx.pin_served_provenance, + named_groups=ctx.named_groups, + ) + scoped = decide_read_scope( + answer, + reading, + selection=ctx.scope_selection, + operations=ctx.operations, + coverage=ctx.coverage, + catalogue=ctx.request_catalogue, + ) + if scoped.decision is None: + # Rows read within a composite's population belong to no table, so a + # relaxation whose scope compiles no composite has no label to state them under. + return None, None, ground_warnings, None + mint = None if scoped.minted is None else scoped.minted.mint spec = build_spec( + composite=mint, + answer=answer, shape=shape_model, slots=slots, stat_vars=stat_vars_list, @@ -1405,27 +1482,15 @@ def read(place: PlaceContext, facet_batch: FacetBatch | None) -> object: plan=mat.read_plan, # The relaxed query's own answer states this headline and these axes. headline=mat.headline, - open_axis_facts=served_open_axes( - relaxed, - mat.sv_dcids, - mat.member_facts or {}, - value_labels=shape.value_labels or {}, - ), + open_axis_facts=open_axis_facts, coverage_index=ctx.coverage.index, pin_served_provenance=ctx.pin_served_provenance, resolved_sources=source_refs, - facets=answer_facets( - mat.read_plan, - labels, - bound_axes={}, - request_basis=None, - unit_request=None, - currency_of=functools.partial(unit_currency, ctx.graph, deadline=ctx.deadline), - ), + facets=facets, slot_options=slot_opts, available_at_place=available_at_place, confidence=confidence, - requested_window=requested_window_from_date_request(ctx.date_request), + requested_window=requested_window, open_roles=mat.open_axes, inapplicable_slots=inapplicable_slots, unresolved_slots=unresolved_slots, @@ -1444,6 +1509,20 @@ def read(place: PlaceContext, facet_batch: FacetBatch | None) -> object: walked=walked, ) spec = apply_declaration_version(spec, relaxed) + # A suggestion minted as a composite counts its own read's defaulted scope over + # the composite's parties (`ReadScope.warnings`), and no other suggestion does. + labels = {entity.ref: entity.label or entity.ref for entity in spec.resolved.entities} + defaulted = ( + [] + if scoped.minted is None + else role_scope_defaulted_warnings( + mat, + boundary_roles=roles.roles, + label_of=lambda dcids: [labels.get(dcid, dcid) for dcid in dcids], + unnamed_wholes=place.roles.unnamed_wholes, + ) + ) + own_warnings = scoped.warnings(spec, defaulted) return ( NearestRealSpec( spec_id=spec.spec_id, @@ -1451,7 +1530,13 @@ def read(place: PlaceContext, facet_batch: FacetBatch | None) -> object: resolved=spec.resolved, relaxation=relaxation, ), - SuggestionRead(plan=mat.read_plan, place=place, member_facts=mat.member_facts), + SuggestionRead( + plan=mat.read_plan, + place=place, + member_facts=mat.member_facts, + composite=mint is not None, + warnings=own_warnings, + ), ground_warnings, probe_warning, ) @@ -1486,6 +1571,9 @@ def _suggest_nearest_real( unsettled: UnsettledPlace | None = None, pin_served_provenance: bool = False, level: tuple[ShapeDraft, Relaxation] | None = None, + scope_selection: ScopeState | None = None, + operations: Sequence[Operation] = (), + request_catalogue: Catalogue | None = None, ) -> tuple[ list[NearestRealSpec], list[Warning], dict[str, SuggestionRead], ConsultedPresence | None ]: @@ -1530,6 +1618,16 @@ def _suggest_nearest_real( exactly their members. With `pin_served_provenance` each suggestion's identity binds the provenances that serve it, as the part's answer would. + Each relaxation elects its provider reading for its relaxed bindings from + `scope_selection` and `operations` (`reading.elect_reading`) and reads within the + population the reading binds, as the part's answer does. A suggestion whose scope + compiles to a composite is minted as it, under `request_catalogue` + (`reading.decide_read_scope`, `spec_build.build_spec`), with no table preview + (`SuggestionRead.composite`). Each suggestion carries its own reading's warnings keyed + to its spec_id (`SuggestionRead.warnings`). These are a composite's + `role_scope_defaulted` count over its parties and the `bilateral_reading_unavailable` of + a reading the bilateral default could not reach. + With `level`, a shape and the relaxation that reaches it, the one relaxation probed is that shape under the same bindings: the level reading of a shape that divides by a denominator (`phase_place_materialise._level_relaxation`). @@ -1554,6 +1652,9 @@ def _suggest_nearest_real( defaulted_properties=defaulted_properties, named_groups=named_groups, pin_served_provenance=pin_served_provenance, + scope_selection=ScopeState() if scope_selection is None else scope_selection, + operations=operations, + request_catalogue=request_catalogue, ) suggestions: list[NearestRealSpec] = [] reads: dict[str, SuggestionRead] = {} @@ -1618,6 +1719,7 @@ def _suggest_nearest_real( suggestions.append(suggestion) unverified = unverified or walk.unverified reads[suggestion.spec_id] = read + probe_warnings.extend(read.warnings) if _entity_warnings: probe_warnings.extend(_entity_warnings) diff --git a/src/search/engine/region/materialise_step.py b/src/search/engine/region/materialise_step.py index 8c01cba34..ded79bac0 100644 --- a/src/search/engine/region/materialise_step.py +++ b/src/search/engine/region/materialise_step.py @@ -68,7 +68,7 @@ def run_materialise( open_axes: frozenset[RoleAxis], coverage: CoverageSnapshot, facet_batch: FacetBatch | None = None, - within: Mapping[RoleAxis, frozenset[str]] | None = None, + within: Mapping[RoleAxis, frozenset[str]] | None, ) -> Materialised | NoDataDraft | MaterialisedCandidates | UnsupportedDraft: """Materialise the chosen shape with scope discovery enabled. `default_presence` derives a default place scope in place of `place_dcids` (`standard_resolve`), `open_axes` diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index ac2a0a2ee..82ba7abe0 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -16,10 +16,11 @@ import functools import logging from collections.abc import Callable -from typing import cast +from typing import TYPE_CHECKING, cast from materialise import GraphReader from materialise.errors import DefaultFacetUndefinedError, FacetNotAvailableError +from origin_catalogue import current_catalogue from search.engine.envelope import _emit_step, now_ms from search.engine.errors import EngineInfraError from search.engine.extract import DateRequest @@ -34,6 +35,7 @@ NO_MEMBER_HAS_DATA_AT_PLACE, _ObsProbeResult, ) +from search.engine.region.composite_answer import ComponentReads, CompositeRelaunch from search.engine.region.convertibility import ( conversion_period_departure, convertible_answer, @@ -105,6 +107,7 @@ withdraw_typed_unit, ) from search.engine.region.place_roles import request_named_groups +from search.engine.region.reading import ReadingDecision, ReadScope, decide_read_scope from search.engine.region.related_read import ( cancel_related_reads, start_answer_related_read, @@ -121,15 +124,19 @@ ) from search.engine.region.scope_decision import ( ScopeDecision, - answer_scope_facts, - decide_guarded, - decide_scope, - guarded_scope_decision, + decide_after_read, + states_answer, +) +from search.engine.region.state import ( + AnswerMark, + MeasureFamily, + MintedComposite, + ResolveState, + raised_facts, + whole_totals_as_read, ) -from search.engine.region.state import MeasureFamily, ResolveState, raised_facts from search.engine.region.trailing import SettledTrailing, passed_over_departure, settle_trailing from search.engine.region.un_system import member_labels -from search.engine.region.whole_totals import whole_total_warnings from search.engine.retrieval.bound_values import ( MeasurePartition, OpenAxisFact, @@ -179,6 +186,9 @@ Warning, ) +if TYPE_CHECKING: + from search.engine.region.candidates import _Candidate + logger = logging.getLogger(__name__) @@ -288,9 +298,6 @@ def _over_conversion_period( if state.unit_request is None or state.unit_elected_selections is None: if stale is not None: state.depart(*stale) - # The years a trailing request's words asked, which the answer's identity no - # longer holds. - state.from_request(stale[0]) return mat_result, filtered catalogue = current_conversion_catalogue() period = unit_request_period( @@ -317,10 +324,8 @@ def _over_conversion_period( is not None ): state.depart(*passed) - state.from_request(passed[0]) elif stale is not None: state.depart(*stale) - state.from_request(stale[0]) omission = deflator_omission_departure( filtered.plan, state.unit_request, @@ -393,8 +398,6 @@ def _narrowed_to_period( ) warning, fact = conversion_period_departure(period, requested=requested, plan=filtered.plan) state.depart(warning, fact) - # The years asked, which the answer's identity, read over the period, no longer holds. - state.from_request(warning) assert fact.shown is not None return mat_result, filtered, fact.shown @@ -413,11 +416,14 @@ def _install_filtered_answer( state: ResolveState, mat_result: Materialised, filtered: FilteredAnswer ) -> None: """Record the answer's coverage, headline and read plan, and restate the discovered-scope - warning to count the members that plan serves, as the scope list and coverage do.""" + warning to count the members that plan serves, as the scope list and coverage do. A + part whose read elected a composite (`ResolveState.reading`) counts the composite's + parties instead, once its Spec is minted (`_build_definite_spec`).""" state.elected_coverage = filtered.coverage state.elected_headline = filtered.headline state.elected_plan = filtered.plan - if state.opposite_labels is not None: + reading = state.reading + if state.opposite_labels is not None and (reading is None or reading.table is None): restate_role_scope_warnings( state.warnings, mat_result, @@ -506,9 +512,7 @@ async def _ground_elected_answer(state: ResolveState) -> RegionResult | None: early = await _decide_bound_unit_convertibility(state, mat_result) if early is not None: # A refusal states no related measures, so the read started for them stops. - if state.related_read is not None: - cancel_related_reads((state.related_read,)) - state.related_read = None + _drop_related_read(state) return early t0 = now_ms() @@ -834,6 +838,21 @@ def _start_assemble_definite(state: ResolveState, mat_result: Materialised) -> _ ) +def component_reads(state: ResolveState) -> ComponentReads: + """The request clients a composite answer's component reads run on, with the request's + catalogue as the part found it, before its source scope narrowed it.""" + return ComponentReads( + graph=state.graph, + sdmx=state.sdmx, + coverage=state.coverage, + materialise_graph=state.materialise_graph, + materialise_sdmx=state.materialise_sdmx, + rate_cache=state.rate_cache, + deadline=state.deadline, + catalogue=state.request_catalogue or current_catalogue(), + ) + + def _preview_reader(state: ResolveState, read: PreviewRead) -> PreviewReader: """`read` bound to the request's clients.""" return PreviewReader( @@ -855,26 +874,17 @@ async def _fetch_definite_observations( return await asyncio.to_thread(_preview_reader(state, setup.preview).observations) -@dataclasses.dataclass(frozen=True) -class _AnswerScope: - """The identity `build_spec` mints for the answer (`answer`) and the scope decided over - it (`decision`).""" - - answer: AnswerIdentity - decision: ScopeDecision - - def _decide_answer_scope( state: ResolveState, mat_result: Materialised, slots: list[Slot], setup: _DefiniteSetup -) -> _AnswerScope: - """The scope the answer `_build_definite_spec` builds from `slots` states - (`scope_decision.decide_scope`), decided over the identity `build_spec` mints from the - same arguments (`spec_build.answer_identity`) before the options are counted, so a - composite answer launches no preview of the one table it resolved. `build_spec` takes - that identity as given, since every argument it reads (`state`, `slots`, - `mat_result.open_axes` and `setup.facet_bindings`) holds unchanged from here to there. - A decision that raises leaves the answer as a single table that states no scope - (`scope_decision.guarded_scope_decision`).""" +) -> ReadScope: + """The scope the answer `_build_definite_spec` builds from `slots` states, built over + the identity `build_spec` mints from the same arguments (`spec_build.answer_identity`) + before the options are counted, so a composite answer launches no preview of the one + table it read. `build_spec` takes that identity as given, since every argument it reads + (`state`, `slots`, `mat_result.open_axes` and `setup.facet_bindings`) holds unchanged + from here to there. The scope applies the provider choice the read elected + (`ResolveState.reading`) and mints the Spec as the composite it compiles, where it + compiles one (`reading.decide_read_scope`).""" answer = answer_identity( state.shape_model, slots, @@ -890,37 +900,24 @@ def _decide_answer_scope( pin_served_provenance=state.pin_served_provenance, named_groups=request_named_groups(state, state.place.roles), ) - - def decide() -> ScopeDecision: - facts = answer_scope_facts( - entity_groups=answer.entity_groups, - facets=answer.facets or {}, - sources=answer.resolved_sources, - operations=state.operations, - coverage=state.coverage, - ) - return decide_scope( - answer.spec_id, answer.identity, selection=state.scope_selection, facts=facts - ) - - decision = guarded_scope_decision( - answer.spec_id, - decide, - identity=answer.identity, + return decide_read_scope( + answer, + state.reading, selection=state.scope_selection, operations=state.operations, coverage=state.coverage, + catalogue=state.request_catalogue, ) - return _AnswerScope(answer=answer, decision=decision) async def _preview_unless_composite( - scope: asyncio.Task[_AnswerScope], setup: _DefiniteSetup, state: ResolveState + scope: asyncio.Task[ReadScope], setup: _DefiniteSetup, state: ResolveState ) -> tuple[SpecObservations, Warning | None] | None: """The answer's preview (`_fetch_definite_observations`), read once its scope is - decided. It is None for an answer restated as a composite, whose preview reads each - component (`pipeline._with_composite_preview`).""" - if (await scope).decision.composite: + decided. It is None for an answer whose scope compiles a composite, whose preview reads + each component (`pipeline._with_composite_preview`).""" + decision = (await scope).decision + if decision is None or decision.composite: return None return await _fetch_definite_observations(setup, state) @@ -933,6 +930,11 @@ class _DefiniteCoverage: counted: CountedSlotOptions # The identity the answer's scope was decided over, which `build_spec` mints the Spec from. answer: AnswerIdentity + # The scope decided over `answer` before the preview launched (`_decide_answer_scope`), + # which the finished answer is checked against (`finished_scope`). + decision: ScopeDecision + # The composite the Spec is minted as, where its scope compiles one. + minted: MintedComposite | None slot_options: list[SlotOptions] facet_options: list[FacetOptions] entity_options_raw: list[EntityOptions] @@ -975,17 +977,20 @@ async def _gather_definite_coverage( mat_result: Materialised, setup: _DefiniteSetup, count_task: asyncio.Task[CountedSlotOptions], - decision_task: asyncio.Task[_AnswerScope], + decision_task: asyncio.Task[ReadScope], + mark: AnswerMark, ) -> RegionResult | _DefiniteCoverage: """Check bound-unit availability, run eligible coverage probes, and await their tasks. `count_task` counts the answer's menus (`counted_slot_options`) beside the entity, facet and observation reads; the slot probe starts from its counted menus once it ends. `decision_task` decides the answer's scope (`_decide_answer_scope`), which the - preview read waits on. An answer restated as a composite reads no preview here, - buffered or streamed, and the decision is kept for `pipeline._with_scope_choices` - (`ResolveState.scope_decision`). Unit refusals return a terminal result before - slot/facet probes launch. Record option probe timing and return the coverage bundle + preview read waits on. An answer whose scope compiles a composite reads no preview + here, buffered or streamed, and the decision and the composite it mints are kept for + the finished answer's check (`finished_scope`, `_DefiniteCoverage.decision` and + `.minted`). A decision that mints no composite for rows read within one answers + `failed_closed` back to `mark`. Unit refusals return a terminal result before slot + and facet probes launch. Record option probe timing and return the coverage bundle otherwise. """ # A known carve-out guarantees no definite result, so the part reads no preview and @@ -1036,7 +1041,10 @@ async def _gather_definite_coverage( if state.stream_observations and not refuses: eager_obs_availability = await _preview_unless_composite(decision_task, setup, state) answer_scope = await decision_task - state.scope_decision = answer_scope.decision + if answer_scope.decision is None: + # Read within a composite's population, with no scope minting the composite. + return failed_closed(state, mark) + decision = answer_scope.decision counted = await count_task # Keep the counted options unless a coverage probe ran. slot_options = counted.options @@ -1076,6 +1084,8 @@ async def _gather_definite_coverage( return _DefiniteCoverage( counted=counted, answer=answer_scope.answer, + decision=decision, + minted=answer_scope.minted, slot_options=slot_options, facet_options=facet_options, entity_options_raw=entity_options_raw, @@ -1175,7 +1185,9 @@ def _build_definite_spec( served_member_facts, value_labels=state.shape_draft.value_labels or {}, ) + minted = coverage.minted spec = build_spec( + composite=None if minted is None else minted.mint, shape=state.shape_model, slots=slots, stat_vars=state.stat_vars, @@ -1245,6 +1257,10 @@ def _build_definite_spec( if state.bound_unit_outcome is not None: spec = with_unit_lexicon_version(spec, state.bound_unit_outcome) spec = apply_declaration_version(spec, state.bindings) + if minted is not None: + # A Spec minted as a composite counts its defaulted scope over the composite's + # parties (`CompositeMint.role_scope_warnings`). + state.warnings[:] = minted.mint.role_scope_warnings(state.warnings, spec) family_total_absent = mat_result.family_total_absent or lone_breakdown_absent( state.breakdown, shape=state.shape_draft, @@ -1263,13 +1279,19 @@ def _build_definite_spec( async def _assemble_carve_outs( - state: ResolveState, mat_result: Materialised, spec: Spec + state: ResolveState, + mat_result: Materialised, + spec: Spec, + *, + minted: MintedComposite | None = None, ) -> RegionResult | None: """Apply the remaining no-data gates after Spec construction. ``None`` means definite. Qualifier refusal precedes missing-slice and observation checks. Bound-unit availability - is already verified by ``bound_unit_gate_reason``. + is already verified by ``bound_unit_gate_reason``. `spec`, the flagged match each gate + names, is minted as `minted` where it is a composite. """ + matched = () if minted is None else (minted,) # Judge the final answer on every requirement (`requirements.settle_answer`). One it # leaves unmet refuses before any broader answer can pass as definite, naming the terms # its trace records. The candidates path judges its answers the same way. The answer is @@ -1281,6 +1303,7 @@ async def _assemble_carve_outs( return state.no_data( "qualifier_not_bound", flagged_match=(spec,), + matched=matched, gate=None, llm_usage=state.var_usage, unbound_qualifiers=unbound_qualifiers, @@ -1293,6 +1316,7 @@ async def _assemble_carve_outs( return state.no_data( "no_observations", flagged_match=(spec,), + matched=matched, gate=NO_MEMBER_HAS_DATA_AT_PLACE, nearest_real=await _nearest_real_for(state), llm_usage=state.var_usage, @@ -1304,6 +1328,7 @@ async def _assemble_carve_outs( return state.no_data( "no_matching_slice", flagged_match=(spec,), + matched=matched, gate=NO_MEMBER_ASSERTS_BOUND_PROPERTIES, llm_usage=state.var_usage, ) @@ -1314,27 +1339,74 @@ async def _assemble_carve_outs( return None -async def _finished_scope(state: ResolveState, spec: Spec) -> ScopeDecision: - """The scope `spec` states, as decided before its preview launched - (`ResolveState.scope_decision`). A decision made for another `spec_id` is decided again - from `spec`, guarded as every decision is (`scope_decision.decide_guarded`), and kept - for `pipeline._with_scope_choices`.""" - decision = cast(ScopeDecision, state.scope_decision) - if decision.spec_id == spec.spec_id: - return decision - logger.warning( - "scope decided for %s before the preview launched, but the answer is %s", - decision.spec_id, - spec.spec_id, - ) - state.scope_decision = await asyncio.to_thread( - decide_guarded, - spec, - selection=state.scope_selection, - operations=state.operations, - coverage=state.coverage, - ) - return state.scope_decision +def _drop_related_read(state: ResolveState) -> None: + """Cancel the related-measures read started for the answer (`ResolveState.related_read`), + whose result nothing will state.""" + if state.related_read is not None: + cancel_related_reads((state.related_read,)) + state.related_read = None + + +def failed_closed( + state: ResolveState, mark: AnswerMark, *, candidate: _Candidate | None = None +) -> RegionResult: + """The no_data `facet_not_available` answer of a definite answer read as a composite + that no scope states, describing `candidate` where the candidates door answers + (`ResolveState.no_data(candidate=)`). The refused answer contributes nothing: the part's + warnings, departure facts, re-election warning and judgement are restored to `mark`, + taken where its producer's answer became specific, before the elected answer is + grounded or the candidates are (`ResolveState.mark_answer`), and the related-measures + read started for it is cancelled.""" + state.restore(mark) + _drop_related_read(state) + return state.no_data("facet_not_available", candidate=candidate, llm_usage=state.var_usage) + + +async def finished_scope( + state: ResolveState, + spec: Spec, + *, + decision: ScopeDecision | None, + minted: MintedComposite | None, + reading: ReadingDecision | None, +) -> ScopeDecision | None: + """The scope the finished definite answer `spec` states. It is `decision`, decided + before the preview launched, where that decision states `spec` + (`scope_decision.states_answer`), judged by `reading`, the reading the answer's own rows + were read under, and `minted`, the composite its Spec was minted as. Any other is a + decision made for another answer, so `spec` is decided again + (`scope_decision.decide_after_read`), which chooses nothing and fails closed. That + decision reads coverage and scans families, so it alone runs off the event loop. It + returns None where `spec` is minted as a composite or its rows were read within a + composite's population, and the caller answers `failed_closed` in its place. Only a + returned decision keeps `minted` on the state (`ResolveState.minted`). + + Each producer of a definite answer, `_assemble_definite_answer` and + `candidate_collapse._assemble_definite`, calls it once, before its related read is + awaited or any of its trailing tasks launch, and carries the decision it returns on the + result (`RegionResult.scope_decision`).""" + minted_as_spec = minted is not None and minted.mint.spec_id == spec.spec_id + read_within = reading is not None and reading.table is not None + if decision is None or not states_answer( + decision, spec, minted=minted_as_spec, read_within=read_within + ): + logger.warning( + "scope decided for %s before the preview launched, but the answer is %s", + None if decision is None else decision.spec_id, + spec.spec_id, + ) + decision = await asyncio.to_thread( + decide_after_read, + spec, + minted=minted_as_spec, + read_within=read_within, + selection=state.scope_selection, + operations=state.operations, + coverage=state.coverage, + ) + if decision is not None: + state.minted = minted + return decision async def _finalize_definite_result( @@ -1343,12 +1415,21 @@ async def _finalize_definite_result( setup: _DefiniteSetup, coverage: _DefiniteCoverage, definite_spec: _DefiniteSpec, + decision: ScopeDecision, ) -> RegionResult: """Attach eager SSE data and related measures, launch relaunch tasks, and return the - definite result. The preview follows the scope decided for the finished answer - (`_finished_scope`).""" + definite result, which carries `decision`, the scope the finished answer states + (`finished_scope`, `RegionResult.scope_decision`). The preview follows it. + + A Spec minted as a composite (`MintedComposite`) launches none of its table's slot + options, place options or preview, which would stream under the composite's spec_id. It + carries the closure that reads the composite's preview per component + (`composite_answer.CompositeRelaunch`), which `pipeline._with_composite_preview` + launches. Its `whole_total_available` warnings are read over the identity its rows were + read for and the populations its roles were read within + (`state.whole_totals_as_read`).""" spec = await with_related(definite_spec.spec, state.related_read) - decision = await _finished_scope(state, spec) + minted = state.minted if decision.composite else None if not decision.composite and state.include_observations and not state.stream_observations: # A bound-unit check fetches observations whether or not a preview is asked for. # Without one the answer states nothing from that fetch, as its resubmit, which @@ -1381,17 +1462,17 @@ async def _finalize_definite_result( variable_labels=member_labels((spec,)), date_request=state.date_request, ) + # A composite streams no option of the table it read. + stream_slot_options = state.stream_slot_options and minted is None slot_options_task = ( - relaunch_slot_options(spec.spec_id, setup.slot_options_raw) - if state.stream_slot_options - else None + relaunch_slot_options(spec.spec_id, setup.slot_options_raw) if stream_slot_options else None ) # Place options require both the graph reader and their dedicated executor. Omit the # task when either dependency is absent. place_options_materialise_graph = state.materialise_graph stream_place_options = ( - state.stream_slot_options + stream_slot_options and place_options_materialise_graph is not None and state.place_options_pool is not None ) @@ -1405,9 +1486,12 @@ async def _finalize_definite_result( # Create the deferred SSE task only after the Spec has an ID for correlation. observations_task: asyncio.Task[_ObsProbeResult] | None = None observations_relaunch: Callable[[str], asyncio.Task[_ObsProbeResult]] | None = None - # A composite's preview reads each component once the answer is restated - # (`pipeline._with_composite_preview`). - if not decision.composite and state.stream_observations: + # A composite's preview reads each component (`pipeline._with_composite_preview`). + if minted is not None: + observations_relaunch = CompositeRelaunch( + spec=spec, reads=component_reads(state), served=minted.mint.served + ) + elif not decision.composite and state.stream_observations: observations_relaunch = _preview_reader(state, setup.preview).relaunch( member_labels((spec,)) ) @@ -1421,14 +1505,14 @@ async def _finalize_definite_result( warnings=( *state.warnings, *(warning for warning, _fact in family), - *whole_total_warnings(spec, coverage=state.coverage.index), + *whole_totals_as_read(spec, minted, coverage=state.coverage.index), ), departure_facts={**state.departure_facts, **raised_facts(family)}, timing_by_step=dict(state.timing), llm_usage=state.var_usage, slot_options_task=slot_options_task, facet_probe=setup.facet_probe, - slot_options_relaunch=relaunch_slot_options if state.stream_slot_options else None, + slot_options_relaunch=relaunch_slot_options if stream_slot_options else None, place_options_task=place_options_task, place_options_relaunch=relaunch_place_options if stream_place_options else None, observations_task=observations_task, @@ -1446,11 +1530,15 @@ async def _finalize_definite_result( role_axis_facts=tuple(setup.role_axis_facts), member_population=mat_result.population, answered_roles=answered_roles(state.shape_draft), + scope_decision=decision, ) -async def _assemble_definite_answer(state: ResolveState) -> RegionResult: - """Build the Spec, apply no-data gates, and return the terminal result.""" +async def _assemble_definite_answer(state: ResolveState, mark: AnswerMark) -> RegionResult: + """Build the Spec, apply no-data gates, and return the terminal result. `mark` is what + the part had raised and judged before the elected answer was grounded + (`_phase_outcome`), so an answer that fails closed restores the part to it + (`failed_closed`).""" mat_result = cast(Materialised, state.mat_result) # Bind the when-slot to the extracted window before assembling the Spec @@ -1485,7 +1573,9 @@ async def _assemble_definite_answer(state: ResolveState) -> RegionResult: decision_task = asyncio.create_task( asyncio.to_thread(_decide_answer_scope, state, mat_result, slots, setup) ) - gathered = await _gather_definite_coverage(state, mat_result, setup, count_task, decision_task) + gathered = await _gather_definite_coverage( + state, mat_result, setup, count_task, decision_task, mark + ) if isinstance(gathered, RegionResult): return gathered coverage = gathered @@ -1502,12 +1592,28 @@ async def _assemble_definite_answer(state: ResolveState) -> RegionResult: ) state.state_reelected_reading(definite_spec.spec) - early = await _assemble_carve_outs(state, mat_result, definite_spec.spec) + early = await _assemble_carve_outs( + state, mat_result, definite_spec.spec, minted=coverage.minted + ) if early is not None: return early + # The scope the answer states is checked once, before its related read is awaited or + # any of its trailing tasks launch. + decision = await finished_scope( + state, + definite_spec.spec, + decision=coverage.decision, + minted=coverage.minted, + reading=state.reading, + ) + if decision is None: + # Read as a composite, with no scope stating the composite it is. + return failed_closed(state, mark) note_denominator_unverified(state, state.shape_draft, spec_id=definite_spec.spec.spec_id) - return await _finalize_definite_result(state, mat_result, setup, coverage, definite_spec) + return await _finalize_definite_result( + state, mat_result, setup, coverage, definite_spec, decision + ) async def _phase_outcome(state: ResolveState) -> RegionResult: @@ -1522,10 +1628,14 @@ async def _phase_outcome(state: ResolveState) -> RegionResult: result = await resolve_candidates(state, rate_cache=state.rate_cache) if result is None: result = await _no_data_answer(state) + if result is not None: + return result + # The answer is the elected shape's from here: everything grounding and assembly raise + # or judge is its own (`ResolveState.mark_answer`, `failed_closed`). + mark = state.mark_answer() + result = await _ground_elected_answer(state) if result is None: - result = await _ground_elected_answer(state) - if result is None: - result = await _assemble_definite_answer(state) + result = await _assemble_definite_answer(state, mark) return result diff --git a/src/search/engine/region/phase_place_materialise.py b/src/search/engine/region/phase_place_materialise.py index dc4c6af2d..bbc2e67f7 100644 --- a/src/search/engine/region/phase_place_materialise.py +++ b/src/search/engine/region/phase_place_materialise.py @@ -7,6 +7,7 @@ import asyncio import dataclasses +import functools import itertools import logging import threading @@ -32,7 +33,11 @@ ) from search.engine.departures import direction_inferred from search.engine.envelope import _emit_step, now_ms -from search.engine.errors import FamilyNotResolvableError, GraphInfraError, RoleSignatureError +from search.engine.errors import ( + FamilyNotResolvableError, + GraphInfraError, + RoleSignatureError, +) from search.engine.interpret import PinnedResolved, Recall from search.engine.place import ConsultedPresence, earth_countries, read_default_presence from search.engine.quantity import ( @@ -128,6 +133,7 @@ undirected_place_name, whole_total_candidates, ) +from search.engine.region.reading import ElectedReads, ReadingDecision, Within from search.engine.region.refs import provenance_labels from search.engine.region.requirements import record_reading, settle_answer, stray_values from search.engine.region.result import FacetAxisFilter, RegionResult @@ -1631,44 +1637,89 @@ async def _phase_facet_requests(state: ResolveState) -> RegionResult | None: type _MaterialiseOutcome = Materialised | NoDataDraft | MaterialisedCandidates | UnsupportedDraft -def _materialise_task( +@dataclass(frozen=True) +class _ElectedRead: + """The elected shape's read (`_materialise_task`): the place context it was read in, its + outcome, and the provider reading elected before it (`reading.elect_reading`), None + where the part binds no total with a bilateral source.""" + + place: PlaceContext + outcome: _MaterialiseOutcome + reading: ReadingDecision | None + + +def _materialise_task(state: ResolveState) -> asyncio.Task[_ElectedRead]: + """Start materialising the elected shape over `state.bindings` off-thread + (`_elected_read`), and return its read. A shape whose place sets are not decided + settles them on this read; any other shape is read in the context it was installed + with.""" + unsettled = state.unsettled_place + target = unsettled if unsettled is not None and not unsettled.decided else state.place + read = functools.partial( + _elected_read, + state, + state.shape_draft, + list(state.bindings), + target, + presence=state.default_presence, + ) + # Match the observation fetch's ``needs_dates=True`` cache key when a preview is enabled, + # including dateless queries. The context scope applies only to this task. + with force_full_facets(state.include_observations or state.stream_observations): + return asyncio.create_task(asyncio.to_thread(read)) + + +def _elected_read( state: ResolveState, -) -> asyncio.Task[tuple[PlaceContext, _MaterialiseOutcome]]: - """Start materialising the elected shape over `state.bindings` off-thread, and return - the place context it is read in with its result. + shape: ShapeDraft, + bindings: Sequence[SlotBindingDraft], + target: PlaceContext | UnsettledPlace, + *, + presence: ConsultedPresence | None, +) -> _ElectedRead: + """`shape`'s read under `bindings` at `target` (`_ElectedRead`), on a worker thread. - A shape whose read decides a place set settles its sets on this read + A target whose place sets are unsettled settles them on this read (`place_context.settle_sets`): it is materialised at a set's node and, where the node - holds no data in the years asked for, at the set's members, from one facet read. Any - other shape is materialised in the context it was installed with. + holds no data in the years asked for, at the set's members, from one facet read. + + Each read elects its provider reading first, at the place it tries, and reads the + composite the reading names within its population (`reading.ElectedReads`), so a part + that settles on a composite is read once, as a resubmit of the composite reads it. The + read the sets settle on carries its own election. """ - shape, bindings, presence = state.shape_draft, list(state.bindings), state.default_presence - unsettled = state.unsettled_place - target = unsettled if unsettled is not None and not unsettled.decided else state.place - def read(place: PlaceContext, facet_batch: FacetBatch | None) -> _MaterialiseOutcome: + def read( + read_at: PlaceContext, facet_batch: FacetBatch | None, within: Within + ) -> _MaterialiseOutcome: return run_materialise( shape, - with_where(bindings, place), - place.roles.resolved_entities, + with_where(bindings, read_at), + read_at.roles.resolved_entities, graph=state.graph, sdmx=state.sdmx, date_request=state.date_request, deadline=state.deadline, - place_dcids=place.probe_places, - default_presence=place.presence_for(presence), - open_axes=place.roles.open_axes, + place_dcids=read_at.probe_places, + default_presence=read_at.presence_for(presence), + open_axes=read_at.roles.open_axes, coverage=state.coverage, facet_batch=facet_batch, + within=within, ) - def settle() -> tuple[PlaceContext, _MaterialiseOutcome]: - return settle_sets(target, read) - - # Match the observation fetch's ``needs_dates=True`` cache key when a preview is enabled, - # including dateless queries. The context scope applies only to this task. - with force_full_facets(state.include_observations or state.stream_observations): - return asyncio.create_task(asyncio.to_thread(settle)) + reads = ElectedReads( + shape, + bindings, + date_request=state.date_request, + selection=state.scope_selection, + coverage=state.coverage, + operations=state.operations, + read=read, + ) + place, outcome = settle_sets(target, reads) + read_at, reading = reads.settled(place) + return _ElectedRead(place=read_at, outcome=outcome, reading=reading) class _RelaxedPartRefused(Exception): @@ -1773,7 +1824,8 @@ async def _run_materialise_task(state: ResolveState) -> _MaterialiseOutcome: t0 = now_ms() materialise_task = _materialise_task(state) try: - place, mat_result = await materialise_task + elected = await materialise_task + place, mat_result = elected.place, elected.outcome while isinstance(mat_result, BoundValuesUnrealisable): refused = await _unbind( state, @@ -1786,7 +1838,8 @@ async def _run_materialise_task(state: ResolveState) -> _MaterialiseOutcome: raise _RelaxedPartRefused(refused) await _cover_unbound(state) materialise_task = _materialise_task(state) - place, mat_result = await materialise_task + elected = await materialise_task + place, mat_result = elected.place, elected.outcome if place != state.place: _install_shape( state, @@ -1797,6 +1850,7 @@ async def _run_materialise_task(state: ResolveState) -> _MaterialiseOutcome: index=state.elected_index, ) state.mat_result = mat_result + state.reading = elected.reading finally: # Cancel and await unfinished materialisation so cancellation cannot detach it. if not materialise_task.done(): @@ -1911,7 +1965,7 @@ def _sibling_shapes_of(state: ResolveState) -> dict[int, ShapeDraft]: async def _reelect_zero_observation_sibling( state: ResolveState, mat_result: Materialised -) -> Materialised: +) -> _MaterialiseOutcome: """Status rule 1's stand-ins (`state.reelection_siblings`) when the elected shape has no observations at its places (``has_data=False``, ``available_at_place=False``): its equivalent readings, then its measurement-qualifier siblings. @@ -1935,6 +1989,9 @@ async def _reelect_zero_observation_sibling( place of materialising them (`admit_alternates`). An answering stand-in is installed in place of the elected shape with its bindings and place context (`_install_shape`), replaces the elected result, and warns `elected_shape_no_data` after its place warnings. + Each stand-in's read elects its provider reading first, as the elected shape's read + does, and an answering stand-in is adopted with that read and its election + (`_adopted_read`). Its `elected` starts as the stand-in's draft label and becomes the shape label of the Spec built for the stand-in, on the definite path and among candidates alike (`ResolveState.state_reelected_reading`). A part refused before any such Spec is built @@ -1986,6 +2043,8 @@ async def _reelect_zero_observation_sibling( date_request=state.date_request, deadline=state.deadline, coverage=state.coverage, + selection=state.scope_selection, + operations=state.operations, ) for members in by_scope.values() ] @@ -2046,25 +2105,30 @@ async def _reelect_zero_observation_sibling( return mat_result if _winner is not None: _no_data_label = state.shape_draft.label - _install_shape( - state, - _winner.shape, - _winner.bindings, - _winner.place, - index=_index_by_group[_shape_group_key(_winner.shape)], - ) + index = _index_by_group[_shape_group_key(_winner.shape)] + adopted = _adopted_read(_winner) + _install_shape(state, _winner.shape, _winner.bindings, adopted.place, index=index) state.walked = _winner.walked - state.mat_result = _winner.mat - mat_result = _winner.mat + state.mat_result = adopted.outcome + state.reading = adopted.reading state.reelection_warning = _elected_sibling_warning( no_data=[_no_data_label], elected=[_winner.shape.label] ) state.warnings.append(state.reelection_warning) - else: - state.sibling_candidates = dict(sorted(_with_data.items())) + return adopted.outcome + state.sibling_candidates = dict(sorted(_with_data.items())) return mat_result +def _adopted_read(winner: _Candidate) -> _ElectedRead: + """The read a stand-in answering in place of the elected shape is adopted with: its own + read (`_materialise_alt_shapes`), which elected its provider reading at each place it + tried, as the elected shape's read does (`_elected_read`), so a stand-in that settled on + a composite was read within its population and is labelled by the reading that read + it.""" + return _ElectedRead(place=winner.place, outcome=winner.mat, reading=winner.reading) + + def _carry_readings(readings: Mapping[int, DegradedReads], *, compared: Iterable[int]) -> None: """Carry each stand-in reading's record into the part's (`degraded_reads.carry`): every lost read, and the scopes consulted by the readings at `compared`, those the outcome's @@ -2100,9 +2164,7 @@ async def _phase_materialise(state: ResolveState) -> RegionResult | None: mat_result = await _reelect_zero_observation_sibling(state, mat_result) # Classify terminal outcomes and append the answering result's warnings, after any - # re-election, so they describe the shape in ``state.shape_draft``. Every warning raised - # so far is the request's: from here on they derive from the rows the read holds. - state.request_warnings = tuple(state.warnings) + # re-election, so they describe the shape in ``state.shape_draft``. state.opposite_labels = memoised_labels(state.label_dcids) outcome = interpret_materialised( mat_result, @@ -2210,9 +2272,6 @@ def note_denominator_unverified( spec_id=spec_id, ) state.warnings.append(warning) - # The question's words name the base, and the shape states its own: no row read - # decides it, so it stands on an answer read again within a population. - state.from_request(warning) # The words for the base a schema-term denominator names (`quantity.schema_denominator_kind`). @@ -2267,7 +2326,8 @@ async def _declined_refusal(state: ResolveState) -> RegionResult: flagged = await _elected_flagged_match(state, members=members, open_roles=open_roles) return state.no_data( "qualifier_not_bound", - flagged_match=(flagged,), + flagged_match=(flagged.spec,), + matched=flagged.matched, nearest_real=await _nearest_real_for(state, level=_level_relaxation(state)), unbound_qualifiers=settle_answer(state, state.shape_draft, state.bindings), llm_usage=state.var_usage, @@ -2286,7 +2346,8 @@ async def refuse_unread_answer(state: ResolveState) -> RegionResult: flagged = await _elected_flagged_match(state, members=members, open_roles=open_roles) return state.no_data( "qualifier_not_bound", - flagged_match=(flagged,), + flagged_match=(flagged.spec,), + matched=flagged.matched, unbound_qualifiers=settle_answer(state, state.shape_draft, state.bindings), llm_usage=state.var_usage, ) @@ -2338,11 +2399,12 @@ async def _denominator_refusal( level = _level_relaxation(state) return state.no_data( "denominator_not_available", - flagged_match=(flagged,), + flagged_match=(flagged.spec,), + matched=flagged.matched, nearest_real=None if level is None else await _nearest_real_for(state, level=level), denominator=NoDataDenominator( asked=named.ref if named is not None else None, - found=flagged.resolved.shape.measurement_denominator, + found=flagged.spec.resolved.shape.measurement_denominator, ), llm_usage=state.var_usage, ) @@ -2432,14 +2494,11 @@ async def _nearest_real_for( if not probe.specs: return None near_specs = probe.specs - before = len(state.warnings) stamp_offered_place_warnings( state.warnings, [(spec.spec_id, probe.reads[spec.spec_id].place) for spec in near_specs], departure_facts=state.departure_facts, ) - # Each suggestion's places are read from the request's words. - state.from_request(*state.warnings[before:]) entity_warnings, previews = await _suggestion_previews(state, near_specs, probe.reads) for warning in entity_warnings: if warning not in state.warnings: @@ -2480,6 +2539,9 @@ async def _probe_nearest_real( unsettled=state.unsettled_place, pin_served_provenance=state.pin_served_provenance, level=level, + scope_selection=state.scope_selection, + operations=state.operations, + request_catalogue=state.request_catalogue, ) return _NearestRealProbe( specs=near_specs, warnings=near_warnings, reads=near_reads, presence=near_presence @@ -2493,12 +2555,16 @@ async def _suggestion_previews( ) -> tuple[list[Warning], dict[int, tuple[SpecObservations, Warning | None]]]: """The warnings choosing each suggestion's preview entities raised, in suggestion order, and the observations each previewed suggestion reads, by index in `near_specs`. Only a - buffered request previews: nearest-real suggestions have no SSE observation event.""" + buffered request previews: nearest-real suggestions have no SSE observation event. A + suggestion minted as a composite has none (`SuggestionRead.composite`), since the rows + read belong to the table, not the composite.""" if not state.include_observations or state.stream_observations: return [], {} entity_warnings: list[Warning] = [] jobs: list[tuple[int, dict[str, list[str]]]] = [] for index, near_spec in enumerate(near_specs): + if near_reads[near_spec.spec_id].composite: + continue place = near_reads[near_spec.spec_id].place obs_entities, obs_entities_warn = _observation_entities( state.shape_draft, diff --git a/src/search/engine/region/phase_shared.py b/src/search/engine/region/phase_shared.py index 72a44e716..be73b4c0a 100644 --- a/src/search/engine/region/phase_shared.py +++ b/src/search/engine/region/phase_shared.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import dataclasses from collections.abc import Collection, Sequence from typing import TYPE_CHECKING @@ -18,7 +19,7 @@ from search.engine.bind import SlotBindingDraft from search.engine.departures import measure_not_applied, price_basis_not_applied from search.engine.qualifier_match import match_qualifiers -from search.engine.region.codes import QUERY_QUALIFIER_DROPPED +from search.engine.region.codes import QUERY_QUALIFIER_DROPPED, ROLE_SCOPE_DEFAULTED from search.engine.region.convertibility import served_units from search.engine.region.grounding import ( _build_entity, @@ -36,9 +37,15 @@ untold_direction_names, ) from search.engine.region.qualifier_readings import shape_carries_measure_match +from search.engine.region.reading import ( + ReadingDecision, + ReadScope, + decide_read_scope, + elect_at_read, +) from search.engine.region.requirements import refusal_terms, settle_answer, settle_flagged from search.engine.region.result import RegionResult -from search.engine.region.state import ResolveState +from search.engine.region.state import MintedComposite, ResolveState from search.engine.retrieval.bound_values import ( apply_declaration_version, apply_slot_expansions, @@ -49,6 +56,8 @@ from search.engine.retrieval.retrieve import Materialised from search.engine.shape import identity_shape_for from search.engine.spec_build import ( + AnswerIdentity, + answer_identity, build_shape_model, build_slot, build_spec, @@ -261,7 +270,6 @@ def withdraw_typed_unit(state: ResolveState, exc: UnitConversionRefusal) -> bool if isinstance(exc, NonCurrencyUnitError) else price_basis_not_applied(asked=state.unit_text, shown=None), ) - state.from_request(warning) return True @@ -300,12 +308,28 @@ def _unit_currency(unit: str | None) -> str | None: return classify_unit(unit=unit, denominator=None, currency_units={}).currency +@dataclasses.dataclass(frozen=True) +class FlaggedMatch: + """A flagged match built by `_elected_flagged_match`, holding its Spec and the composite + it is minted as (`MintedComposite`), None where it is minted as none. A refusal passes that + composite to `ResolveState.no_data` as `matched`, so `ResolveState.minted` holds the + part's own answer's composite alone.""" + + spec: Spec + minted: MintedComposite | None + + @property + def matched(self) -> tuple[MintedComposite, ...]: + """The composite the match is minted as, as `ResolveState.no_data` takes it.""" + return () if self.minted is None else (self.minted,) + + async def _elected_flagged_match( state: ResolveState, *, members: Sequence[str] | None = None, open_roles: Collection[RoleAxis] = (), -) -> Spec: +) -> FlaggedMatch: """Build the elected shape's flagged identity without probing availability, for a refusal that judges the shape itself: role incompatibility, or a quantity form or denominator the shape does not serve. The identity names `members`, the StatVars @@ -375,7 +399,29 @@ async def _elected_flagged_match( identity_entities = build_identity_entities(entity_objects_data, role_refs) requested_window = requested_window_from_date_request(state.date_request) slots = apply_slot_expansions(slots, slot_expansions(state.bindings)) + named_groups = request_named_groups(state, state.place.roles) + answer = answer_identity( + shape, + slots, + stat_vars, + entities, + identity_entities=identity_entities, + open_roles=open_roles, + requested_window=requested_window, + facet_bindings=None, + facets=None, + resolved_sources=[], + plan=None, + pin_served_provenance=False, + named_groups=named_groups, + ) + # The election and the scope read coverage and scan families, so they run off the + # event loop. + scoped = await asyncio.to_thread(_flagged_scope, state, answer) + minted = scoped.minted spec = build_spec( + composite=None if minted is None else minted.mint, + answer=answer, shape=shape, slots=slots, stat_vars=stat_vars, @@ -407,11 +453,66 @@ async def _elected_flagged_match( # pairs, as on the definite path below. measure_elected_properties=state.measure_elected_properties, defaulted_properties=state.defaulted_properties, - named_groups=request_named_groups(state, state.place.roles), + named_groups=named_groups, walked=state.walked, query_qualifiers=settle_flagged(state, state.shape_draft, state.bindings), ) - return apply_declaration_version(spec, state.bindings) + spec = apply_declaration_version(spec, state.bindings) + _key_reading_warnings(state, scoped, spec, minted=minted) + return FlaggedMatch(spec=spec, minted=minted) + + +def _flagged_scope(state: ResolveState, answer: AnswerIdentity) -> ReadScope: + """The scope of a flagged match of the elected shape whose identity is `answer` + (`reading.decide_read_scope`), under the reading its read elected (`_flagged_reading`). + A composite that refused its table read elected an undecided reading's choice + (`ReadingDecision.refused`), so the match is the table's Spec and says it reads total + ODA.""" + return decide_read_scope( + answer, + _flagged_reading(state), + selection=state.scope_selection, + operations=state.operations, + coverage=state.coverage, + catalogue=state.request_catalogue, + ) + + +def _key_reading_warnings( + state: ResolveState, scoped: ReadScope, spec: Spec, *, minted: MintedComposite | None +) -> None: + """Add the warnings of `spec`, a flagged match decided as `scoped`, to the part's, each + keyed to its spec_id (`ReadScope.warnings`). A match minted as a composite counts the + part's own `role_scope_defaulted` warnings, those of the read it was elected at, over + the composite's parties, so those warnings speak for it alone and leave the part's + unkeyed list.""" + own = [ + warning + for warning in state.warnings + if warning.code == ROLE_SCOPE_DEFAULTED and warning.spec_id is None + ] + if minted is not None and own: + state.warnings[:] = [warning for warning in state.warnings if warning not in own] + for warning in scoped.warnings(spec, own): + if warning not in state.warnings: + state.warnings.append(warning) + + +def _flagged_reading(state: ResolveState) -> ReadingDecision | None: + """The provider reading a flagged match of the elected shape is minted under. That is the + one its read elected (`ResolveState.reading`), else the one electing at the part's place + (`reading.elect_at_read`) where no read has been made.""" + if state.reading is not None: + return state.reading + return elect_at_read( + state.shape_draft, + state.bindings, + state.place, + date_request=state.date_request, + selection=state.scope_selection, + coverage=state.coverage, + operations=state.operations, + ) async def settle_unjudged_refusal(state: ResolveState, result: RegionResult) -> RegionResult: @@ -449,7 +550,8 @@ async def settle_unjudged_refusal(state: ResolveState, result: RegionResult) -> flagged = await _elected_flagged_match(state, members=members) return state.no_data( "qualifier_not_bound", - flagged_match=(flagged,), + flagged_match=(flagged.spec,), + matched=flagged.matched, nearest_real=result.nearest_real, unbound_qualifiers=settle_answer(state, state.shape_draft, state.bindings), llm_usage=result.llm_usage, @@ -460,10 +562,11 @@ async def settle_unjudged_refusal(state: ResolveState, result: RegionResult) -> async def _role_incompatible_no_data(state: ResolveState) -> RegionResult: """Return the common role-incompatibility outcome without a new probe.""" - flagged_match = await _elected_flagged_match(state) + flagged = await _elected_flagged_match(state) return state.no_data( "role_incompatible", - flagged_match=(flagged_match,), + flagged_match=(flagged.spec,), + matched=flagged.matched, llm_usage=state.var_usage, ) diff --git a/src/search/engine/region/pipeline.py b/src/search/engine/region/pipeline.py index 966f6a2bf..e580945e2 100644 --- a/src/search/engine/region/pipeline.py +++ b/src/search/engine/region/pipeline.py @@ -35,6 +35,7 @@ note_degraded_read, recording_degraded_reads, ) +from search.engine.departures import warning_key from search.engine.envelope import now_ms from search.engine.errors import ( RATE_TABLE_REFUSALS, @@ -48,15 +49,7 @@ from search.engine.qualifier_match import CodedQualifier from search.engine.region.candidate_admission import alternates_unevaluated_warning from search.engine.region.codes import ALTERNATES_UNEVALUATED, UN_SYSTEM_UNAVAILABLE -from search.engine.region.composite_answer import ( - ComponentReads, - population_table_answer, - with_bilateral_readings, - with_composite_observations, - with_decided_scope, - with_population_read, - without_restated_options, -) +from search.engine.region.composite_answer import with_composite_observations from search.engine.region.decisions import PartyReadingWithdrawn from search.engine.region.phase_answer import _phase_outcome from search.engine.region.phase_election import _phase_settle, judge_coverage, log_would_refuse @@ -86,6 +79,7 @@ group_request_names, stated_party_words, ) +from search.engine.region.reading import noting_refusals_once from search.engine.region.related_read import cancel_related_reads, settle_related_reads from search.engine.region.requirements import ( met_measure_phrases, @@ -93,12 +87,8 @@ unbinding_phrases, with_requirement_departures, ) -from search.engine.region.result import ( - RegionResult, - cancel_dropped_tasks, - every_member_phrases, -) -from search.engine.region.scope_decision import decide_guarded, scope_facts +from search.engine.region.result import RegionResult, every_member_phrases +from search.engine.region.scope_decision import ScopeDecision, scope_facts from search.engine.region.source_choice import ( allows_un_system, answer_provenances, @@ -119,7 +109,6 @@ from search.scope.basis import identity_basis_record from search.scope.build import build_part_scope from search.scope.state import ScopeIntake -from search.scope.tables import PartFacts, ScopeFacts from series_coverage.shared import CoverageSnapshot logger = logging.getLogger(__name__) @@ -491,37 +480,41 @@ async def attempt( # (`recording_llm_usage`), and carries the first's record: its lost reads always, and the # scopes it consulted when its answer was withdrawn for want of data, which those scopes # decided. - probe = new_state( - pool="ours", emit=None, deadline=_options.deadline, timing={}, party_words_off=False - ) - named = stated_party_words(probe) - if not named: - result, _lost = await attempt(party_words_off=True, sink=emit) - return result - held: list[Event] = [] - - async def hold(event: Event) -> None: - held.append(event) - - with recording_llm_usage() as spent: - try: - first, carried = await attempt( - party_words_off=False, sink=None if emit is None else hold - ) - except PartyReadingWithdrawn as withdrawn: - carried, no_data = withdrawn.lost, False - else: - if binds_each_party(first, named): - if emit is not None: - for event in held: - await emit(event) - return first - no_data = first.status == "no_data" - result, _lost = await attempt( - party_words_off=True, sink=emit, carried=carried, carried_consulted=no_data - ) - usage = functools.reduce(sum_usage, spent.calls, result.llm_usage) - return dataclasses.replace(result, llm_usage=usage) + # A composite whose table read refuses is noted once in the region however many reads + # elect it, the party reading's and the reading without party words alike: the record + # of the first is always carried into the second. + with noting_refusals_once(): + probe = new_state( + pool="ours", emit=None, deadline=_options.deadline, timing={}, party_words_off=False + ) + named = stated_party_words(probe) + if not named: + result, _lost = await attempt(party_words_off=True, sink=emit) + return result + held: list[Event] = [] + + async def hold(event: Event) -> None: + held.append(event) + + with recording_llm_usage() as spent: + try: + first, carried = await attempt( + party_words_off=False, sink=None if emit is None else hold + ) + except PartyReadingWithdrawn as withdrawn: + carried, no_data = withdrawn.lost, False + else: + if binds_each_party(first, named): + if emit is not None: + for event in held: + await emit(event) + return first + no_data = first.status == "no_data" + result, _lost = await attempt( + party_words_off=True, sink=emit, carried=carried, carried_consulted=no_data + ) + usage = functools.reduce(sum_usage, spent.calls, result.llm_usage) + return dataclasses.replace(result, llm_usage=usage) def binds_each_party(result: RegionResult, named: Sequence[PartyWord]) -> bool: @@ -600,126 +593,54 @@ def _stating_unpublished_wholes(result: RegionResult, state: ResolveState) -> Re return dataclasses.replace(result, unpublished_whole_facts=facts) if facts else result -def _with_scope_choices(result: RegionResult, state: ResolveState) -> RegionResult: +def _with_scope_choices(result: RegionResult) -> RegionResult: """`result` stating how its definite answer was counted (`RegionResult.scope_choices`), - the scope the answer phase or the candidates path decided for it before its preview - launched (`ResolveState.scope_decision`), else one decided here from the answer - (`scope_decision.decide_guarded`), so a decision that raises leaves the answer as one - table. A result - that is not definite states none. The scope is built in memory, and the rules read the - bound release's code index, the request's coverage snapshot and the part's operations, - groups and served datasets (`scope_facts`). Only a dataset_resubmit's posted - `scope_entries` refuse 422. + the scope decision it carries (`RegionResult.scope_decision`). Both producers of a + definite answer, `phase_answer._assemble_definite_answer` and + `candidate_collapse._assemble_definite`, check that decision once before the answer + launches anything (`phase_answer.finished_scope`), answer no_data where none states the + Spec, and carry the one they checked. A definite result carrying none is a producer's + fault and raises `TypeError`. A result that is not definite states none and is returned + as it is. + The rules read the bound release's code index, the request's coverage snapshot and the + part's operations, groups and served datasets (`scope_facts`). Only a dataset_resubmit's + posted `scope_entries` refuse 422. When the binders settle a composite option (the inclusive provider scope, the OECD - headline per year), the scope's base is the composite, and the answer is restated as it - (`composite_answer.with_decided_scope`), carrying the closure that reads its preview per - component (`composite_answer.CompositeRelaunch`) for `_with_composite_preview` to launch - on the event loop. A composite whose table read binds a population restates the answer - `_within_population` read within it. A DAC2a answer with no provider axis that reads - total ODA because the bilateral default could not apply carries the decision's + headline per year), the scope's base is the composite. The answer phase mints the + answer's Spec as it, read within the population its table read binds, with the closure + that reads its preview per component (`composite_answer.CompositeRelaunch`) for + `_with_composite_preview` to launch on the event loop (`state.MintedComposite`), so the + scope is stated as decided. A DAC2a answer with no provider axis that reads total ODA + because the bilateral default could not apply carries the decision's `bilateral_reading_unavailable` warning (`ScopeDecision.warning`). - A result that is not definite states no scope, and the readings it shows in place of a - definite answer, its candidates, the alternate a refusal offers and its flagged matches, - are read as the definite answer reads DAC2a's total ODA - (`composite_answer.with_bilateral_readings`), under the part's scope facts: the answer - phase's decision's where it decided one, else facts of the part's operations alone.""" + Each reading a result that is not definite shows in place of a definite answer (its + candidates, the alternate a refusal offers, its flagged matches and its nearest real + suggestions) elected its provider reading at its own read and was minted as the + composite that reading names where it names one (`reading.decide_read_scope`).""" if result.status != "definite": - decided = state.scope_decision - return with_bilateral_readings( - result, - reads=_component_reads(state), - selection=state.scope_selection, - facts=decided.facts - if decided is not None - else ScopeFacts( - index=get_index(), - coverage=state.coverage, - part=PartFacts(operations=tuple(state.operations)), - ), - operations=state.operations, - request_warnings=state.request_warnings or (), - ) - decision = state.scope_decision - if decision is None or decision.spec_id != result.spec.spec_id: - decision = decide_guarded( - result.spec, - selection=state.scope_selection, - operations=state.operations, - coverage=state.coverage, - ) - return with_decided_scope(result, decision, reads=_component_reads(state)) - - -async def _within_population(result: RegionResult, state: ResolveState) -> RegionResult: - """`result`, the pass's terminal result, read again within the population its composite - binds, or at the population's total in place of a destination it leaves out, where its - definite answer settles on one (`composite_answer.population_table_answer`): the - answer's open roles were discovered over the whole table before its scope settled on - the composite, so the part answers what a resubmit of the composite's table identity - answers, no_data included, and no_data where the composite refuses the read, carrying - the request's words and the warnings raised before the read - (`composite_answer.with_population_read`). It runs before the pass states requirement - departures, so they are stated on that read; stated-word coverage was judged on the - whole table's read, whose StatVars the population read shares. - The scope decision it reads is the answer phase's, else decided here, and is kept for - `_with_scope_choices` as the decision of the read's answer: the scope is decided once, - on the request's own identity, so a read at the population's total, whose Spec names a - destination the request never named, states the scope and composite identity the - request's answer does. The trailing tasks of the whole-table answer are cancelled. - `result` itself for any other result.""" - if result.status != "definite" or state.request_warnings is None: return result - spec = result.spec - decision = state.scope_decision - if decision is None or decision.spec_id != spec.spec_id: - decision = await asyncio.to_thread( - decide_guarded, - spec, - selection=state.scope_selection, - operations=state.operations, - coverage=state.coverage, - ) - state.scope_decision = decision - if not decision.composite or decision.scope is None: - return result - t0 = now_ms() - answered = await asyncio.to_thread( - population_table_answer, - spec, - decision.scope.compiled.identity, - reads=_component_reads(state), - ) - if answered is None: + if result.scope_decision is None: + raise TypeError(f"definite answer {result.spec.spec_id} carries no scope decision") + return _with_decided_scope(result, result.scope_decision) + + +def _with_decided_scope(result: RegionResult, decision: ScopeDecision) -> RegionResult: + """`result`, a definite answer, stating the scope `decision` decided for it + (`RegionResult.scope_choices`), which states its Spec (`scope_decision.states_answer`), + and carrying the decision's warning once (`ScopeDecision.warning`).""" + warning = decision.warning + if warning is not None and warning_key(warning) not in set(map(warning_key, result.warnings)): + result = dataclasses.replace(result, warnings=(*result.warnings, warning)) + if decision.scope is None: return result - grafted = with_population_read(result, answered, request_warnings=state.request_warnings) - cancel_dropped_tasks(result, grafted) - if grafted.status == "definite": - state.scope_decision = dataclasses.replace(decision, spec_id=grafted.spec.spec_id) - return dataclasses.replace( - grafted, timing_by_step={**grafted.timing_by_step, "population_read": now_ms() - t0} - ) - - -def _component_reads(state: ResolveState) -> ComponentReads: - """The request clients a composite answer's component reads run on, with the request's - catalogue as the part found it, before its source scope narrowed it.""" - return ComponentReads( - graph=state.graph, - sdmx=state.sdmx, - coverage=state.coverage, - materialise_graph=state.materialise_graph, - materialise_sdmx=state.materialise_sdmx, - rate_cache=state.rate_cache, - deadline=state.deadline, - catalogue=state.request_catalogue or current_catalogue(), - ) + return dataclasses.replace(result, scope_choices=decision.scope) async def _with_composite_preview(result: RegionResult, state: ResolveState) -> RegionResult: """`result` with the preview of the composite it answers, which `_with_scope_choices` - leaves unlaunched (`with_composite_answer`). It launches as a trailing task where + leaves unlaunched. It launches as a trailing task where the request streams observations, so the `response` event waits on nothing new, and reads inline where the request embeds them. Any other result returns unchanged. @@ -1104,7 +1025,6 @@ async def _run_phases(state: ResolveState) -> RegionResult: if result is None: settled = settled or phase is _phase_settle else: - result = await _within_population(result, state) # Only a vetted election can report qualifier matches: earlier terminals # may hold pre-bind candidates without a forced binding or elected shape. every_member = ( @@ -1140,13 +1060,7 @@ async def _run_phases(state: ResolveState) -> RegionResult: ), state, ) - # The scope build reads coverage and scans families, so it runs off - # the event loop, and the tasks a composite restatement drops are - # cancelled back on it. - stated = _stating_unpublished_wholes(result, state) - answered = await asyncio.to_thread(_with_scope_choices, stated, state) - cancel_dropped_tasks(stated, answered) - answered = without_restated_options(stated, answered) + answered = _with_scope_choices(_stating_unpublished_wholes(result, state)) return await _with_composite_preview(answered, state) if state.capped_shapes and state.scope is not None and not scoped: reads.enter_context(scoped_reads(state.scope.allowed)) diff --git a/src/search/engine/region/place_options.py b/src/search/engine/region/place_options.py index b16ad7435..4ffd6ef29 100644 --- a/src/search/engine/region/place_options.py +++ b/src/search/engine/region/place_options.py @@ -391,9 +391,8 @@ def replayed_place_options( A fresh run launches one task per definite Spec (`place_options_task`), which the response's part carries, and none for a Spec that answers a composite, whose place - options would read its table under the composite's spec_id - (`result.with_composite_answer`). Parts folded into one Spec share its task. - The tasks read `coverage`, the hit's own snapshot, which the bundle's key holds.""" + options would read its table under the composite's spec_id. Parts folded into one Spec + share its task. The tasks read `coverage`, the hit's own snapshot, which the bundle's key holds.""" if not stream or materialise_graph is None or pool is None: return None # Bound once narrowed, so the closure reads them as present. diff --git a/src/search/engine/region/promote_finalize.py b/src/search/engine/region/promote_finalize.py index a6eda536a..5dda0c249 100644 --- a/src/search/engine/region/promote_finalize.py +++ b/src/search/engine/region/promote_finalize.py @@ -22,7 +22,10 @@ _maybe_enrich_slot_options, ) from search.engine.region.phase_timing import timed_step -from search.engine.region.promote_ground import unit_refusal_input_error +from search.engine.region.promote_ground import ( + read_whole_total_warnings, + unit_refusal_input_error, +) from search.engine.region.promote_relaunch import PromoteRelaunch, bind_relaunches from search.engine.region.promote_results import _no_data_region from search.engine.region.promote_state import _PromoteState @@ -30,7 +33,6 @@ from search.engine.region.result import RegionResult, answered_roles from search.engine.region.state import EngineOptions, _elected_stat_vars from search.engine.region.un_system import member_labels -from search.engine.region.whole_totals import whole_total_warnings from search.engine.retrieval.elected_dataset import elected_datasets from search.engine.retrieval.member_facts import ArcsMemberFacts from search.engine.retrieval.read_plan import plan_window_gap, serves_nothing @@ -335,9 +337,7 @@ def _phase_finalize(state: _PromoteState) -> RegionResult: with timed_step(state, "finalize_wholes"): resubmit_warnings = ( *resubmit_warnings, - *whole_total_warnings( - spec, coverage=state.coverage.index, populations=state.inp.populations - ), + *read_whole_total_warnings(state, spec), ) relaunch_t0 = now_ms() diff --git a/src/search/engine/region/promote_ground.py b/src/search/engine/region/promote_ground.py index 2b904b627..1310ce2b4 100644 --- a/src/search/engine/region/promote_ground.py +++ b/src/search/engine/region/promote_ground.py @@ -127,6 +127,7 @@ ResolvedQualifier, Spec, TimeWindow, + Warning, ) @@ -321,11 +322,12 @@ def _install_narrowed_answer( ) -> None: """Record the resubmit's coverage, headline and read plan, and restate the discovered-scope warning to count the members that plan serves, as the scope list and - coverage do.""" + coverage do. A Spec minted as a composite counts its components' members instead + (`_ground_and_build_spec`).""" state.narrowed_coverage = filtered.coverage state.narrowed_headline = filtered.headline state.narrowed_plan = filtered.plan - if state.opposite_labels is not None: + if state.opposite_labels is not None and state.composite is None: restate_role_scope_warnings( state.warnings, mat_result, @@ -460,7 +462,9 @@ def _converted_by(state: _PromoteState) -> IdentityUnitRequest | None: def _ground_and_build_spec( state: _PromoteState, mat_result: Materialised, roles: list[tuple[str, EntityRoleKey]] ) -> Spec: - """Ground the answer, derive scope and axis facts, and build the Spec. + """Ground the answer, derive scope and axis facts, and build the Spec, minted as the + composite `state.composite` names where it names one, whose `role_scope_defaulted` + warning then counts the composite's parties (`CompositeMint.role_scope_warnings`). Option-coverage probes wait until finalisation confirms a definite result.""" t0 = _ground_answer_and_models(state, mat_result, roles) @@ -548,6 +552,7 @@ def _ground_and_build_spec( state.unit_request_basis = _unit_request_basis_from_facets(state.inp.identity.facets) named_groups = resubmit_named_groups(state.inp, graph=state.graph, deadline=state.deadline) spec = state.spec = build_spec( + composite=state.composite, shape=shape_model, slots=slots, stat_vars=stat_vars, @@ -603,6 +608,8 @@ def _ground_and_build_spec( plans=(state.narrowed_plan,) if state.narrowed_plan is not None else (), ) spec = state.spec = apply_declaration_version(spec, state.bindings) + if state.composite is not None: + state.warnings[:] = state.composite.role_scope_warnings(state.warnings, spec) # The resubmit states the `family_total_absent` fact its member read states. The slots # its posted identity leaves open count as named, so every signature of the family it # admits lies inside them and no lone breakdown is stated (`lone_breakdown_absent`). @@ -668,6 +675,20 @@ def _ground_build_gates( return None +def read_whole_total_warnings(state: _PromoteState, spec: Spec) -> list[Warning]: + """`whole_total_available` for `spec` over the identity the resubmit read and the + populations its roles were read within (`ResubmitIdentityPayload.populations`). For a + Spec minted as a composite that identity is the table identity the resubmit read + (`state.inp.identity`), whose members `spec` lists and whose places (a population's total + among them) it binds, as a fresh answer minted as the composite reads + (`state.whole_totals_as_read`). Any other Spec reads its own identity, within the + populations a composite's component is posted with.""" + read = spec.definition.identity if state.composite is None else state.inp.identity + return whole_total_warnings( + spec, identity=read, populations=state.inp.populations, coverage=state.coverage.index + ) + + def _flagged_refusal( state: _PromoteState, spec: Spec, @@ -684,9 +705,7 @@ def _flagged_refusal( reason, warnings=( *state.refusal_warnings(), - *whole_total_warnings( - spec, coverage=state.coverage.index, populations=state.inp.populations - ), + *read_whole_total_warnings(state, spec), ), flagged_match=(spec,), gate=gate, diff --git a/src/search/engine/region/promote_state.py b/src/search/engine/region/promote_state.py index 7e8ce1a9c..a74381017 100644 --- a/src/search/engine/region/promote_state.py +++ b/src/search/engine/region/promote_state.py @@ -73,6 +73,7 @@ from search.engine.region.requirements import Judgement from search.engine.region.state import Requirement from search.engine.retrieval.member_facts import MemberFacts + from search.scope.composite import CompositeMint @dataclasses.dataclass(slots=True, kw_only=True) @@ -95,6 +96,9 @@ class _PromoteState: place_options_pool: PlaceOptionsPool | None = None # Shared coverage snapshot for materialise, unit decisions, and observations; required. coverage: CoverageSnapshot + # The composite the Spec is minted as (`ResubmitIdentityPayload.composite`), None when it + # answers its own identity. + composite: CompositeMint | None = None # --- (b) evolving state ---------------------------------------------------- # Initial display text; the shape phase assigns the verified slug. diff --git a/src/search/engine/region/reading.py b/src/search/engine/region/reading.py new file mode 100644 index 000000000..558179ccb --- /dev/null +++ b/src/search/engine/region/reading.py @@ -0,0 +1,761 @@ +"""The provider reading of a part that binds a published total with a bilateral source, +elected before Search reads the part's table. + +DAC2a's total ODA is read at its bilateral destinations alone where the part names none or +only bilateral ones (`scope.provider.with_bilateral_default`), and that reading reads other +rows of the table than the total does. `elect_reading` makes the part's provider choice +from what the part holds before its read: its shape and bindings, the place the read is +tried at, the scope entries and records its binders hold, the coverage snapshot, and the +years the read reads (`extract.read_window`). The read can then read the composite's table +within its population (`ReadingDecision.table`), and the answer's scope applies the same +choice to the identity minted after the read (`scope_decision.build_part_scope`). + +The window is the read's own. A trailing request settles on a narrower range after the read +(`trailing.settle_trailing`), always inside the read window (`trailing._assert_inside`), and +the bilateral option is judged year by year (`provider._sibling_option`), so a composite +elected over the read window covers every range the answer can settle on.""" + +from __future__ import annotations + +import dataclasses +import logging +import threading +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from typing import TYPE_CHECKING + +from code_index.composites import composite_ref +from code_index.members import SourceNotInFamily, identity_family_query, index_family +from code_index.query import CodeIndex +from code_index.scope_conventions import BILATERAL_SOURCES +from code_index.shared import get_index +from dfo_platform.roles import role_key_axis +from origin_catalogue import bound_catalogue, current_catalogue +from search.engine.degraded_reads import note_degraded_read +from search.engine.errors import EngineInputError +from search.engine.extract import read_window +from search.engine.identity_edit import TableRead, composite_table_identity, open_within +from search.engine.region.codes import ROLE_SCOPE_DEFAULTED +from search.engine.region.grounding import requested_window_from_date_request +from search.engine.region.scope_decision import ( + READ_WINDOW_NOT_COVERED, + ProviderChoice, + ScopeDecision, + answer_scope_facts, + build_part_scope, + choose_provider, + guarded_scope_decision, + note_undecided, + undecided_choice, +) +from search.engine.region.state import MintedComposite +from search.engine.retrieval.bound_values import bound_source +from search.engine.retrieval.retrieve import NoDataDraft +from search.engine.shape import entities_by_role, identity_shape_for +from search.engine.spec_id import identity_spec_id +from search.models import BaseIdentity, Operation, Spec, TimeWindow, Warning +from search.scope.provider import Withheld +from search.scope.state import ScopeState +from search.scope.tables import PartFacts, ScopeFacts +from series_coverage.shared import CoverageSnapshot + +if TYPE_CHECKING: + from materialise.roles import RoleAxis + from origin_catalogue import Catalogue + from search.engine.bind import SlotBindingDraft + from search.engine.extract import DateRequest + from search.engine.region.place_context import PlaceContext + from search.engine.retrieval.standard import FacetBatch + from search.engine.shape import ShapeDraft + from search.engine.spec_build import AnswerIdentity + +logger = logging.getLogger(__name__) + +_ELECTION = "reading election" +"""The subject an election that raises is logged under (`scope_decision.note_undecided`), +and the read an election's fault or refusal is noted as (`note_degraded_read`).""" + +_OPTION_NOT_COVERED = Withheld(why="option_not_covered") +"""The reason of a reading the election withheld because the bilateral destinations do not +cover every year the read reads (`provider.bilateral_default_withheld`).""" + +_BILATERAL_CODES = frozenset((row.code_system, row.code) for row in BILATERAL_SOURCES) + +_REFUSALS: ContextVar[set[str] | None] = ContextVar("search_refused_composites", default=None) +"""The spec_ids of the composites whose table read refused in the bound resolution +(`noting_refusals_once`), each logged and noted once. The set is shared by the threads the +resolution starts, so `_refusals_lock` guards it.""" + +_refusals_lock = threading.Lock() + + +@contextmanager +def noting_refusals_once() -> Iterator[None]: + """Bind a fresh record of refused composites for one region's resolution + (`pipeline.resolve_variable`), so each composite whose table read refuses is logged at + ERROR and noted as a degraded read once there however many of its reads elect it: the + answer's, each alternate's, each stand-in's and each nearest-real suggestion's. A region + binds its own, so each region's record of degraded reads, which decides whether a cache + stores it, holds the note. Outside a binding every refusal is logged and noted.""" + token = _REFUSALS.set(set()) + try: + yield + finally: + _REFUSALS.reset(token) + + +def _first_refusal(composite: str) -> bool: + """Whether `composite`'s refusal is the first the bound resolution meets, recording it. + True outside a binding (`noting_refusals_once`).""" + refused = _REFUSALS.get() + if refused is None: + return True + with _refusals_lock: + if composite in refused: + return False + refused.add(composite) + return True + + +@dataclasses.dataclass(frozen=True) +class ReadingDecision: + """The provider reading `elect_reading` elects for a part before its read. + + `choice` is the provider choice (`scope_decision.ProviderChoice`): the composite the + bilateral default reads or None where the part reads the table, why the default could + not apply, the provider record and the selection its scope reads. `table` is what the + read reads for the composite the part's scope compiles to + (`identity_edit.composite_table_identity`): its table identity, with the populations its + roles bind and the sides it reads. None where the part reads no composite.""" + + choice: ProviderChoice + table: TableRead | None + # The `entities` keys of `table.identity` on a flow axis the part neither names nor + # opens: the population's total a component binds in place of a destination the part + # leaves out (`RolePopulation.total`), which the read names as the composite's table + # read does and the composite's own identity leaves out. + left_out: frozenset[str] = frozenset() + # Whether the composite the part's scope compiles to refuses its table read + # (`identity_edit.composite_table_identity`), so the part reads nothing and answers + # no_data, `facet_not_available`: a fresh search does not refuse its scope, and the + # whole table's rows would answer under the composite. Its `choice` is then an undecided + # reading's (`scope_decision.undecided_choice`), so a flagged match built over the part, + # which reads nothing, is the table's Spec and says it reads total ODA. + refused: bool = False + + +def elect_reading( + shape: ShapeDraft, + bindings: Sequence[SlotBindingDraft], + place: PlaceContext, + *, + date_request: DateRequest | None, + selection: ScopeState, + coverage: CoverageSnapshot, + operations: Sequence[Operation], +) -> ReadingDecision | None: + """The provider reading of the part read for `shape` under `bindings` at `place`, one + settled place the read tries, with the scope entries and records of `selection`, judged + in `coverage` over the years the read reads for `date_request` (`extract.read_window`). + None where no binding counts a published total with a bilateral source + (`BILATERAL_SOURCES`), such as DAC2a's aid type 206. + + The provider rules read an identity, so the election reads the document the part's + pre-read state names (`_pre_read_identity`). Its choice + (`scope_decision.choose_provider`) is built into a scope over that document to name the + composite the scope compiles to, whose table read is elected here + (`ReadingDecision.table`). + + An election that raises anywhere, building the document included, is logged as a scope + decision is (`scope_decision.note_undecided`), and the part reads the table as total + ODA, saying so where the bilateral default reaches it (`_undecided_election`). A + composite that refuses its table read is elected as refused, with an undecided reading's + choice (`ReadingDecision.refused`), and logged at ERROR and noted as a degraded read, so + no cache stores the answer, once per region's resolution (`noting_refusals_once`). + + Raises `EngineInputError` (`request_invalid`) for a reversed window, as the answer's mint + refuses it (`grounding.requested_window_from_date_request`).""" + if not binds_bilateral_total(bindings): + return None + identity: BaseIdentity | None = None + facts: ScopeFacts | None = None + try: + index = get_index() + identity = _pre_read_identity(shape, bindings, place, date_request=date_request) + # The facts of a part serving no dataset yet, which an election that raises past here + # reads the bilateral default's reach in (`_undecided_election`). + facts = ScopeFacts( + index=index, coverage=coverage, part=PartFacts(operations=tuple(operations)) + ) + if (dataset := _elected_dataset(identity, index)) is not None: + facts = dataclasses.replace( + facts, part=dataclasses.replace(facts.part, datasets=(dataset,)) + ) + choice = choose_provider(identity, _snapshot(selection), facts) + decision = build_part_scope(identity, choice, spec_id=_ELECTION, facts=facts) + if not decision.composite: + return ReadingDecision(choice=choice, table=None) + assert decision.scope is not None + composite = decision.scope.compiled.identity + try: + table = composite_table_identity( + composite, pointer="/input/identity", coverage=coverage + ) + except EngineInputError as exc: + composite_id = identity_spec_id(composite) + if _first_refusal(composite_id): + logger.error( + "reading election of %s: the composite refuses its table read (%s): %s", + composite_id, + exc.code, + exc, + ) + note_degraded_read(_ELECTION) + return ReadingDecision( + choice=undecided_choice( + identity, selection=selection, facts=facts, subject=_ELECTION + ), + table=None, + refused=True, + ) + asked = {role_key_axis(key) for key in identity.entities} | set(identity.open_roles or ()) + left_out = frozenset( + key for key in table.identity.entities if role_key_axis(key) not in asked + ) + return ReadingDecision(choice=choice, table=table, left_out=left_out) + except Exception as exc: + if identity is None and isinstance(exc, EngineInputError) and exc.code == "request_invalid": + # A reversed window, refused building the document (`_read_window`). + raise + note_undecided(_ELECTION, exc) + return ReadingDecision( + choice=_undecided_election(identity, selection=selection, facts=facts), table=None + ) + + +def _undecided_election( + identity: BaseIdentity | None, *, selection: ScopeState, facts: ScopeFacts | None +) -> ProviderChoice: + """The provider choice of a part whose election raised (`elect_reading`): the table as + total ODA, withheld with `why` `scope_undecided` where the bilateral default reaches + `identity` (`scope_decision.undecided_choice`). Where the election raised before its + document was built, the reach cannot be read, so it withholds nothing, as a reach check + that raises withholds nothing.""" + if identity is None or facts is None: + return ProviderChoice( + states_scope=False, bilateral=None, withheld=None, selection=selection, provider=None + ) + return undecided_choice(identity, selection=selection, facts=facts, subject=_ELECTION) + + +def elect_at_read( + shape: ShapeDraft, + bindings: Sequence[SlotBindingDraft], + place: PlaceContext, + *, + date_request: DateRequest | None, + selection: ScopeState, + coverage: CoverageSnapshot, + operations: Sequence[Operation], +) -> ReadingDecision | None: + """The provider reading a read of `shape` under `bindings` at `place` elects + (`elect_reading`), as every read of a part elects it: the definite answer's, each + alternate's and each stand-in's. A reversed window elects none, so the answer's mint + refuses it with `request_invalid`, where a read that reaches the mint refuses it.""" + try: + return elect_reading( + shape, + bindings, + place, + date_request=date_request, + selection=selection, + coverage=coverage, + operations=operations, + ) + except EngineInputError as exc: + if exc.code != "request_invalid": + raise + return None + + +def read_within( + shape: ShapeDraft, place: PlaceContext, reading: ReadingDecision | None +) -> tuple[PlaceContext, Within]: + """The context a read of `shape` elected as `reading` at `place` reads in, and the + members each open axis is discovered within (`graph_confirm_resolve`): for a composite, + its table read (`ReadingDecision.table`) with each role bound to a population open within + its members (`identity_edit.open_within`), and the population's total a component binds + in place of a destination the part leaves out named on its axis + (`ReadingDecision.left_out`), as a resubmit of the composite reads it. `place` itself, + discovering no axis within members, for any other reading.""" + if reading is None or reading.table is None: + return place, None + opened, within = open_within(reading.table.identity, reading.table.populations) + if not reading.left_out: + return place, within + return _naming(shape, place, {key: opened.entities[key] for key in reading.left_out}), within + + +type Within = dict[RoleAxis, frozenset[str]] | None +"""The members each open axis of a read is discovered within (`read_within`), None where it +discovers no axis within members.""" + +type WithinRead[T] = Callable[[PlaceContext, FacetBatch | None, Within], T] +"""A read of a part at the context it reads in, with its facet batch and the members each +open axis is discovered within.""" + + +class ElectedReads[T]: + """The read of `shape` under `bindings` at each place a place-set settlement tries + (`place_context.settle_sets`), the definite answer's, each alternate's and each + nearest-real suggestion's. Each read elects its provider reading first, at the place it + tries (`elect_at_read`), and reads within the composite's population that reading names + (`read_within`) through `read`, so a part that settles on a composite is read once, as a + resubmit of the composite reads it. A composite that refuses its table read reads nothing + and answers no_data, `facet_not_available`. + + Each place tried is held by identity with the context it was read in and its election, + so its identity is not reused while the settlement runs, and the place the sets settle on + gives both (`settled`). `bindings` is read at each call, so a caller that edits the list + between rounds reads the edited bindings.""" + + def __init__( + self, + shape: ShapeDraft, + bindings: Sequence[SlotBindingDraft], + *, + date_request: DateRequest | None, + selection: ScopeState, + coverage: CoverageSnapshot, + operations: Sequence[Operation], + read: WithinRead[T], + ) -> None: + self._shape = shape + self._bindings = bindings + self._date_request = date_request + self._selection = selection + self._coverage = coverage + self._operations = operations + self._read = read + self._tried: dict[int, tuple[PlaceContext, PlaceContext, ReadingDecision | None]] = {} + + def __call__(self, place: PlaceContext, facet_batch: FacetBatch | None) -> T | NoDataDraft: + reading = elect_at_read( + self._shape, + self._bindings, + place, + date_request=self._date_request, + selection=self._selection, + coverage=self._coverage, + operations=self._operations, + ) + read_at, within = read_within(self._shape, place, reading) + self._tried[id(place)] = (place, read_at, reading) + if reading is not None and reading.refused: + return NoDataDraft(reason="facet_not_available") + return self._read(read_at, facet_batch, within) + + def settled(self, place: PlaceContext) -> tuple[PlaceContext, ReadingDecision | None]: + """The context the read at `place`, a place this settlement tried, read in, and the + reading it elected.""" + _place, read_at, reading = self._tried[id(place)] + return read_at, reading + + +@dataclasses.dataclass(frozen=True) +class ReadScope: + """The identity `build_spec` mints a reading's Spec from (`answer`), the scope decided + over it (`decision`), and the composite the Spec is minted as where that scope compiles + one (`minted`), as `decide_read_scope` decides them. `decision` is None where the + reading was read within a composite's population and no scope minting that composite + could be decided, so its rows have no label to stand under.""" + + answer: AnswerIdentity + decision: ScopeDecision | None + minted: MintedComposite | None + + def warnings(self, spec: Spec, stated: Sequence[Warning]) -> tuple[Warning, ...]: + """The warnings of the reading `spec`, the Spec minted from this scope, shown beside + a part's own answer, each stamped with `spec`'s spec_id. Where `spec` is minted as a + composite, they hold each `role_scope_defaulted` warning of `stated`, the warnings the + read it was elected at raised, counting the composite's parties + (`CompositeMint.role_scope_warnings`). They also hold the decision's + `bilateral_reading_unavailable` (`ScopeDecision.warning`).""" + scoped: list[Warning] = [] + if self.minted is not None: + defaulted = [warning for warning in stated if warning.code == ROLE_SCOPE_DEFAULTED] + scoped.extend(self.minted.mint.role_scope_warnings(defaulted, spec)) + if self.decision is not None and self.decision.warning is not None: + scoped.append(self.decision.warning) + return tuple(warning.model_copy(update={"spec_id": spec.spec_id}) for warning in scoped) + + +def decide_read_scope( + answer: AnswerIdentity, + reading: ReadingDecision | None, + *, + selection: ScopeState, + operations: Sequence[Operation], + coverage: CoverageSnapshot, + catalogue: Catalogue | None, +) -> ReadScope: + """The scope of the reading `answer` names, whose read elected `reading` + (`elect_at_read`), and the composite its Spec is minted as: the definite answer's + (`phase_answer._decide_answer_scope`), each candidate's + (`candidate_grounding._candidate_answer`), each nearest-real suggestion's + (`grounding._relaxed_binding_suggestion`) and a flagged match's + (`phase_shared._elected_flagged_match`). `selection` holds the part's scope entries and + records, `operations` its operations, and the composite is minted under `catalogue`, + else the one in force. + + The scope applies the provider choice `reading` elected (`scope_decision.build_part_scope`) + and chooses nothing, so the reading that labels the Spec is the one that read its rows. + A reading that elected none applies the choice its own identity makes + (`_unelected_choice`). The scope is built over the identity the request names, without + the population's total the read named in place of a destination the part leaves out + (`request_identity`), and checked against the scope decided over it (`_checked`). A + scope that compiles a composite mints the Spec as it (`MintedComposite`), with the + composite's components listed on its compiled scope. A decision that raises leaves the + reading as a single table that states no scope (`scope_decision.guarded_scope_decision`). + + A read made within a composite's population (`ReadingDecision.table`) holds that + population's rows alone, so it fails closed: where its scope cannot be decided, the mint + raises, or the scope compiles no composite, no table label stands over those rows. It is + logged at ERROR, noted as a degraded read so no cache stores it, and the decision is + None (`_unminted`).""" + # Deferred: resubmit imports the answer phase, which imports this module. + from search.engine.region.resubmit import composite_mint + + identity = request_identity(answer.identity, reading) + minted: list[MintedComposite] = [] + + def decide() -> ScopeDecision: + facts = answer_scope_facts( + entity_groups=answer.entity_groups, + facets=answer.facets or {}, + sources=answer.resolved_sources, + operations=operations, + coverage=coverage, + ) + choice = ( + _unelected_choice(answer.spec_id, identity, selection=selection, facts=facts) + if reading is None + else reading.choice + ) + decision = build_part_scope(identity, choice, spec_id=answer.spec_id, facts=facts) + if reading is not None: + decision = _checked(decision, identity, reading, selection=selection, facts=facts) + built = decision.scope + if not decision.composite or built is None: + return decision + with bound_catalogue(catalogue or current_catalogue()): + mint = composite_mint(built.compiled.identity, coverage=coverage) + compiled = built.compiled.model_copy(update={"components": mint.components()}) + minted.append( + MintedComposite( + mint=mint, + read=answer.identity, + populations=reading.table.populations + if reading is not None and reading.table is not None + else {}, + ) + ) + return dataclasses.replace( + decision, + spec_id=mint.spec_id, + scope=built.model_copy(update={"compiled": compiled}), + ) + + if reading is not None and reading.table is not None: + try: + decision = decide() + except Exception as exc: + return _unminted(answer, exc) + if not decision.composite: + return _unminted(answer, None) + return ReadScope(answer=answer, decision=decision, minted=minted[-1]) + decision = guarded_scope_decision( + answer.spec_id, + decide, + identity=identity, + selection=selection, + operations=operations, + coverage=coverage, + ) + return ReadScope( + answer=answer, decision=decision, minted=minted[-1] if decision.composite else None + ) + + +def _unminted(answer: AnswerIdentity, exc: Exception | None) -> ReadScope: + """The scope of the reading `answer` names, read within a composite's population, whose + scope minting that composite raised `exc` or, with `exc` None, compiled no composite: no + decision (`decide_read_scope`).""" + logger.error( + "reading %s was read within a composite's population, but %s; it has no label and " + "is not cached", + answer.spec_id, + "its scope compiles no composite" if exc is None else "the composite could not be minted", + exc_info=exc, + ) + note_degraded_read("composite mint") + return ReadScope(answer=answer, decision=None, minted=None) + + +def request_identity(identity: BaseIdentity, reading: ReadingDecision | None) -> BaseIdentity: + """`identity`, the identity minted from the read, without the entities the read named in + place of a destination the part leaves out (`ReadingDecision.left_out`): the identity the + request names, whose composite reads that destination at the population's total.""" + if reading is None or not reading.left_out: + return identity + document = identity.model_dump(exclude_none=True) + document["entities"] = { + key: dcids for key, dcids in identity.entities.items() if key not in reading.left_out + } + return BaseIdentity.model_validate(document) + + +def _unelected_choice( + spec_id: str, identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts +) -> ProviderChoice: + """The provider choice of the reading `spec_id` of `identity`, whose read elected no + reading: the one `identity` itself makes under `selection` + (`scope_decision.choose_provider`). A choice that reads the bilateral default would label + rows the read did not read within the composite's population, so it is a fault: it is + logged at ERROR, noted as a degraded read so no cache stores the answer, and the reading + reads its table as total ODA, saying so (`scope_decision.undecided_choice`).""" + choice = choose_provider(identity, selection, facts) + if choice.bilateral is None: + return choice + logger.error( + "reading %s reads its table with no reading elected, but the bilateral default reads " + "it as %s; it reads total ODA and is not cached", + spec_id, + choice.bilateral, + ) + note_degraded_read(_ELECTION) + return undecided_choice(identity, selection=selection, facts=facts, subject=spec_id) + + +def _checked( + decision: ScopeDecision, + identity: BaseIdentity, + reading: ReadingDecision, + *, + selection: ScopeState, + facts: ScopeFacts, +) -> ScopeDecision: + """`decision`, built from the choice `reading` elected before the read, checked against + the scope decided over `identity`, the reading's own identity, itself under `selection` + (`scope_decision.decide_scope`). The check decides nothing the reading states but one + thing: the reading was elected over the years the read reads (`extract.read_window`), and + where the election withheld the composite because it does not cover every year read + (`option_not_covered`) and the range the answer settled on reads it, its + `bilateral_reading_unavailable` warning names `read_window_not_covered`. A part with a + provider axis states that on its bilateral option. Any other difference, or a check that + raises, is a fault in the election: it is logged at ERROR and noted as a degraded read, + so no cache stores the answer, and the reading keeps `decision`, whose label matches the + rows it read. + + A refused reading (`ReadingDecision.refused`) is not checked: the election logged its + refusal and chose an undecided reading on purpose, so its choice differs from the + identity's by design and is no fault of the election.""" + if reading.refused: + return decision + spec_id = decision.spec_id + try: + choice = choose_provider(identity, selection, facts) + # `decision` is `build_part_scope` over this identity, spec_id and facts, so where the + # identity chooses what the election chose, the scope decided over it is `decision` + # by construction and is not built again. + checked = ( + decision + if choice == reading.choice + else build_part_scope(identity, choice, spec_id=spec_id, facts=facts) + ) + except Exception: + logger.exception("reading check of %s raised; the answer is not cached", spec_id) + note_degraded_read("reading check") + return decision + elected, settled = _composite_of(decision), _composite_of(checked) + read_within = reading.table is not None + if elected == settled and decision.composite == read_within: + return decision + if ( + elected is None + and settled is not None + and not read_within + and reading.choice.withheld == _OPTION_NOT_COVERED + ): + logger.warning( + "answer %s reads total ODA: the bilateral reading %s covers the range it settled " + "on, not every year it read", + spec_id, + settled, + ) + unread = dataclasses.replace(reading.choice, withheld=Withheld(why=READ_WINDOW_NOT_COVERED)) + return build_part_scope(identity, unread, spec_id=spec_id, facts=facts) + logger.error( + "answer %s elected %s before its read%s, but its identity decides %s; it keeps the " + "reading it read and is not cached", + spec_id, + elected or "the table", + " within the composite's population" if read_within else "", + settled or "the table", + ) + note_degraded_read(_ELECTION) + return decision + + +def _composite_of(decision: ScopeDecision) -> str | None: + """The composite `decision`'s scope compiles to, None where it compiles none.""" + if not decision.composite or decision.scope is None: + return None + return composite_ref( + decision.scope.compiled.identity.model_dump(mode="json", include={"facets"}) + ) + + +def _naming( + shape: ShapeDraft, place: PlaceContext, named: Mapping[str, Sequence[str]] +) -> PlaceContext: + """`place`, the context of role-bearing `shape`, with the places `named` maps by flow role + key bound on that role's axis after those it binds, its destinations as its where places, + as a posted identity's places bind (`promote_places._posted_place_roles`).""" + roles = place.roles + by_axis: dict[RoleAxis, list[str]] = { + "destination": list(roles.destination_dcids), + "origin": list(roles.origin_dcids), + } + for key, dcids in named.items(): + axis = role_key_axis(key) + assert axis is not None, f"{key!r} names no flow axis" + by_axis[axis].extend(dcid for dcid in dcids if dcid not in by_axis[axis]) + destinations, origins = by_axis["destination"], by_axis["origin"] + bound = dataclasses.replace( + roles, + destination_dcids=destinations, + origin_dcids=origins, + resolved_entities=entities_by_role(shape=shape, by_axis=by_axis), + roles=[(dcid, "destination") for dcid in destinations] + + [(dcid, "origin") for dcid in origins], + where_dcids=destinations, + obs_destination_dcid=destinations[0] if destinations else None, + ) + probe = None if len(destinations) <= 1 else tuple(destinations) + return dataclasses.replace(place, roles=bound, probe_dcids=probe) + + +def _snapshot(selection: ScopeState) -> ScopeState: + """`selection` as it stands, so a record a binder writes after the election leaves the + choice's selection as it was elected.""" + return dataclasses.replace(selection, records=dict(selection.records)) + + +def binds_bilateral_total(bindings: Sequence[SlotBindingDraft]) -> bool: + """Whether some binding of `bindings` binds a bilateral source's total as a value + (`BILATERAL_SOURCES`), the parts `elect_reading` elects a reading for.""" + return any(_binds_bilateral_code(binding) for binding in bindings) + + +def _binds_bilateral_code(binding: SlotBindingDraft) -> bool: + """Whether `binding` binds a bilateral source's total as a value (`BILATERAL_SOURCES`).""" + return ( + binding.kind == "value" + and len(binding.value_dcids) == 1 + and (binding.property_dcid, binding.value_dcids[0]) in _BILATERAL_CODES + ) + + +def _pre_read_identity( + shape: ShapeDraft, + bindings: Sequence[SlotBindingDraft], + place: PlaceContext, + *, + date_request: DateRequest | None, +) -> BaseIdentity: + """The identity document the provider rules read for a part before its read: the + shape's five-tuple, each value, set or unbound binding on the what and how axes as a + slot, the entities `place` binds by role key and the axes it leaves open, the read window + (`_read_window`), and a provenance pin to the dataset the bindings or the shape name. + + It is no answer identity: the mint after the read binds invariant slots and settles the + window, and neither moves the provider choice.""" + slots = [] + for binding in bindings: + if binding.axis not in ("what", "how") or not binding.property_dcid: + continue + if binding.kind == "value" and len(binding.value_dcids) == 1: + member: dict[str, object] = {"state": "value", "ref": binding.value_dcids[0]} + elif binding.kind == "set" and binding.value_dcids: + member = {"state": "set", "refs": sorted(binding.value_dcids)} + elif binding.kind == "unbound": + member = {"state": "unbound"} + else: + continue + slots.append({"property": binding.property_dcid, "binding": member}) + entities: dict[str, list[str]] = {} + for dcid, key in place.roles.roles: + held = entities.setdefault(key, []) + if dcid not in held: + held.append(dcid) + document: dict[str, object] = { + "identity_version": "spec1", + "shape": identity_shape_for( + pop_type_dcid=shape.pop_type_dcid, + meas_prop_dcid=shape.meas_prop_dcid, + stat_type_dcid=shape.stat_type_dcid, + meas_qual_dcid=shape.meas_qual_dcid, + meas_denom_dcid=shape.meas_denom_dcid, + meas_denom_unresolved=shape.meas_denom_unresolved, + ), + "slots": sorted(slots, key=lambda slot: str(slot["property"])), + "requested_window": _read_window(date_request), + } + if entities: + document["entities"] = entities + if place.roles.open_axes: + document["open_roles"] = sorted(place.roles.open_axes) + if dataset := bound_source(bindings) or shape.named_dataset: + document["facets"] = {"provenance": {"state": "value", "ref": dataset}} + return BaseIdentity.model_validate(document) + + +def _read_window(date_request: DateRequest | None) -> dict[str, object]: + """The identity's `requested_window` member for the years the part's read reads + (`extract.read_window`): for a trailing span, every range it can settle on; otherwise + the window the answer's identity requests (`grounding.requested_window_from_date_request`, + which refuses a reversed window), the latest year, or every year.""" + if date_request is not None and date_request.trailing is not None: + window = read_window(date_request) + assert window is not None + return _range(window) + requested = requested_window_from_date_request(date_request) + if requested is None: + return {"kind": "all"} + if requested == "latest": + return {"kind": "latest"} + return _range(requested) + + +def _range(window: TimeWindow) -> dict[str, object]: + """The `range` member of `window`, with the bounds it sets.""" + member: dict[str, object] = {"kind": "range"} + if window.start_year is not None: + member["start_year"] = window.start_year + if window.end_year is not None: + member["end_year"] = window.end_year + return member + + +def _elected_dataset(identity: BaseIdentity, index: CodeIndex) -> str | None: + """The dataset the index elects for `identity`'s family, None where it elects none, as + the answer's provenance facet names it after the read.""" + try: + family = index_family( + index, identity_family_query(index, identity.model_dump(mode="json", exclude_none=True)) + ) + except SourceNotInFamily: + return None + return family.provenance if family is not None else None diff --git a/src/search/engine/region/resubmit.py b/src/search/engine/region/resubmit.py index 1d21dee80..02e23c4fc 100644 --- a/src/search/engine/region/resubmit.py +++ b/src/search/engine/region/resubmit.py @@ -9,8 +9,9 @@ import dataclasses import logging +from code_index.shared import get_index from materialise.member_conversion import request_conversion_reader -from origin_catalogue import recording_un_blocked_places +from origin_catalogue import current_catalogue, recording_un_blocked_places from search.engine.clients.graph import EngineGraphClient from search.engine.clients.sdmx import EngineSdmxClient from search.engine.degraded_reads import recording_degraded_reads @@ -21,7 +22,10 @@ from search.engine.region.result import RegionResult from search.engine.region.state import EngineOptions from search.engine.region.un_system import with_un_unavailable_warning -from search.models import ResubmitIdentityPayload +from search.engine.spec_id import identity_spec_id +from search.models import BaseIdentity, ResubmitIdentityPayload +from search.scope.composite import CompositeMint, served_components +from search.scope.tables import ScopeFacts from series_coverage.shared import CoverageSnapshot logger = logging.getLogger(__name__) @@ -50,6 +54,23 @@ def _with_nested_identity_pointer(exc: EngineInputError) -> EngineInputError: ) +def composite_mint(identity: BaseIdentity, *, coverage: CoverageSnapshot) -> CompositeMint: + """What a Spec answering composite `identity` is minted from (`CompositeMint`): its + components with the years each serves (`served_components`), read with facts of the + release's code index and `coverage` alone, and the catalogue bound when it is called, + which its provenance facet and download read. A caller builds it before any read narrows + the catalogue to a source scope (`resubmit_request.pinned_provenance_scope`).""" + facts = ScopeFacts(index=get_index(), coverage=coverage) + document = identity.model_dump(mode="json", exclude_none=True) + return CompositeMint( + identity=identity, + spec_id=identity_spec_id(identity), + served=tuple(served_components(document, facts=facts)), + facts=facts, + catalogue=current_catalogue(), + ) + + def resolve_spec_resubmit( *, inp: ResubmitIdentityPayload, @@ -81,10 +102,17 @@ def resolve_spec_resubmit( `coverage` and `options` carry request-scoped enrichment inputs. `options=None` uses the un-enriched baseline. The result is `degraded` when the promote swallowed a read (`degraded_reads`). + + Where `inp.composite` names the composite `inp.identity` is the table identity of, the + Spec is minted as that composite (`composite_mint`), under its spec_id, its warnings + keyed to it and its defaulted scope counted over its components + (`CompositeMint.role_scope_warnings`). """ check_described_arm(inp.identity.shape, pointer="/input/identity/shape") check_not_composite(inp.identity, pointer="/input/identity/facets/provenance") _options = options if options is not None else EngineOptions() + # Built before the identity's provenance pin narrows the catalogue. + mint = None if inp.composite is None else composite_mint(inp.composite, coverage=coverage) opened, _ = open_within(inp.identity, inp.populations) if opened is not inp.identity: inp = inp.model_copy(update={"identity": opened}) @@ -119,6 +147,7 @@ def resolve_spec_resubmit( population_memo=_options.population_memo, place_options_pool=_options.place_options_pool, coverage=coverage, + composite=mint, ) except EngineInputError as exc: raise _with_nested_identity_pointer(exc) from exc diff --git a/src/search/engine/region/resubmit_promote.py b/src/search/engine/region/resubmit_promote.py index 3a8b9fc17..8bf733295 100644 --- a/src/search/engine/region/resubmit_promote.py +++ b/src/search/engine/region/resubmit_promote.py @@ -45,6 +45,7 @@ from search.engine.options.population_memo import OptionsPopulationMemo from search.engine.region.place_options import PlaceOptionsPool from search.models import BaseIdentity + from search.scope.composite import CompositeMint def _resolve_standard_promote( @@ -64,8 +65,10 @@ def _resolve_standard_promote( population_memo: OptionsPopulationMemo | None = None, place_options_pool: PlaceOptionsPool | None = None, coverage: CoverageSnapshot, + composite: CompositeMint | None = None, ) -> RegionResult: - """Resolve a resubmit from the identity alone. + """Resolve a resubmit from the identity alone, its Spec minted as `composite` where one + is given (`spec_build.build_spec`). The identity's members are elected as the Datasets mint elects them (`promote_reread._elected_members`): the release's code index for a named measure family or @@ -100,6 +103,7 @@ def _resolve_standard_promote( population_memo=population_memo, place_options_pool=place_options_pool, coverage=coverage, + composite=composite, role_class_walks=( RoleClassWalks( materialise_graph, diff --git a/src/search/engine/region/result.py b/src/search/engine/region/result.py index 39fdf1cbc..8fe38aae4 100644 --- a/src/search/engine/region/result.py +++ b/src/search/engine/region/result.py @@ -8,8 +8,6 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, cast -from code_index.composites import DECLARED_COMPOSITES, composite_ref, destination_population -from code_index.shared import get_index from dfo_platform import Cause from materialise.roles import axis_for_role, axis_noun from search.engine.clients.graph import Facet @@ -23,8 +21,6 @@ from search.engine.region.codes import ( ALTERNATES_UNEVALUATED, QUERY_QUALIFIER_DROPPED, - ROLE_SCOPE_DEFAULTED, - WHOLE_TOTAL_AVAILABLE, _ObsProbeResult, _OptionProbeResult, ) @@ -44,7 +40,6 @@ from search.heading.departure_fact import DepartureFact from search.models import ( Assumption, - BaseIdentity, EntityRoleKey, GraphRef, NearestRealSpec, @@ -72,6 +67,7 @@ from search.engine.region.observations.dispatch import PreviewRead from search.engine.region.option_coverage import RoleServedFilter from search.engine.region.promote_relaunch import PromoteRelaunch + from search.engine.region.scope_decision import ScopeDecision from search.engine.retrieval.coverage import CoverageForm from search.engine.retrieval.read_plan import ReadPlan @@ -305,6 +301,12 @@ class RegionResult: # whose request posted scope entries or whose binders settled a scope axis. Its definite # part attaches it to its own Spec; a part folded into a same-shape merge does not. scope_choices: PartScope | None = None + # The scope a fresh pass's definite answer states (`phase_answer.finished_scope`), which + # each producer of one, `phase_answer._assemble_definite_answer` and + # `candidate_collapse._assemble_definite`, carries from the check it makes before the + # answer launches anything, and which `pipeline._with_scope_choices` states as + # `scope_choices`. None on every other result and door. + scope_decision: ScopeDecision | None = None # The assumptions each reading of a dataset resubmit candidates entry echoed back # (`ResubmitCandidate.assumptions`), keyed by the rebuilt reading's spec_id. Its # candidates part reads each reading with its echo (`render.assumptions.assumptions_for`); @@ -569,137 +571,6 @@ def answered_roles(shape: ShapeDraft) -> tuple[str, ...]: return shape.declared_roles or ("observationAbout",) -def with_composite_answer( - region: RegionResult, - spec: Spec, - *, - observations_relaunch: Callable[[str], asyncio.Task[_ObsProbeResult]], -) -> RegionResult: - """`region` answering `spec`, the composite its definite answer was restated as - (`scope.composite.assemble_composite`). Its trailing slot options, place options and - observation preview read the one table the region resolved, under that answer's spec_id, - so the composite carries none of their tasks and no slot or place options. Its preview - is read per component by `observations_relaunch` (`composite_answer.CompositeRelaunch`), - which the caller launches on the event loop where the request streams observations - (`enrichment._launch_trailing_tasks`). The table answer's own warnings, unpublished-whole - facts and assumption echoes are keyed to the composite (`with_restated_keys`), and its - `role_scope_defaulted` warnings count the composite's open axes (`recounted_scope`). - - The restatement touches no task, so it runs on any thread. A task `region` held belongs - to the request's event loop, and the caller cancels it there - (`cancel_dropped_tasks`).""" - recounted = dataclasses.replace( - region, warnings=recounted_scope(region.warnings, region.spec, spec, own=True) - ) - return dataclasses.replace( - with_restated_keys(recounted, {region.spec.spec_id: spec}), - specs=(spec, *region.specs[1:]), - slot_options_task=None, - slot_options_relaunch=None, - place_options_task=None, - place_options_relaunch=None, - observations_task=None, - observations_relaunch=observations_relaunch, - ) - - -def recounted_scope( - warnings: Sequence[Warning], table: Spec, composite: Spec, *, own: bool -) -> tuple[Warning, ...]: - """`warnings`, with each `role_scope_defaulted` warning about `table` on an axis - `composite` counts (its `params` axis, `grounding.role_scope_defaulted_warning`) stating - the composite's count in place of `table`'s, built from the composite's own facts by the - builder a fresh answer's warning is built by (`grounding.stated_role_scope_message`): - the composite counts the parties of every component, the table answer those of its one - table (`scope.composite.assemble_composite`). A warning is about `table` when keyed to - it, or keyed to none where `table` is the region's `own` reading. Every other warning - stands.""" - # grounding imports region.facets, which imports this module. - from search.engine.region.grounding import stated_role_scope_message - - restated = { - fact.role: message - for fact in composite.resolved.role_axis_facts or () - if (message := stated_role_scope_message(composite, fact.role)) is not None - } - if not restated: - return tuple(warnings) - - def recounted(warning: Warning) -> Warning: - about = warning.spec_id == table.spec_id or (own and warning.spec_id is None) - axis = (warning.params or {}).get("axis") - if not about or warning.code != ROLE_SCOPE_DEFAULTED or not isinstance(axis, str): - return warning - message = restated.get(axis) - return warning if message is None else warning.model_copy(update={"message": message}) - - return tuple(map(recounted, warnings)) - - -def with_restated_keys(region: RegionResult, renamed: Mapping[str, Spec]) -> RegionResult: - """`region` with what it keys by the spec_id of each reading `renamed` restates keyed to - the restated Spec's: its warnings, without those the restated Spec cannot stand beside - (`_stands_beside`), its unpublished-whole facts and its assumption echoes. A warning keyed - to another reading, or to none, stands as it is.""" - - def warning(held: Warning) -> Warning | None: - spec = renamed.get(held.spec_id) if held.spec_id is not None else None - if spec is None: - return held - if not _stands_beside(held, spec.definition.identity): - return None - return held.model_copy(update={"spec_id": spec.spec_id}) - - def key(spec_id: str) -> str: - spec = renamed.get(spec_id) - return spec_id if spec is None else spec.spec_id - - return dataclasses.replace( - region, - warnings=tuple(kept for held in region.warnings if (kept := warning(held)) is not None), - unpublished_whole_facts={ - key(spec_id): facts for spec_id, facts in region.unpublished_whole_facts.items() - }, - reading_assumption_echoes={ - key(spec_id): echo for spec_id, echo in region.reading_assumption_echoes.items() - }, - ) - - -def _stands_beside(warning: Warning, identity: BaseIdentity) -> bool: - """Whether `warning`, read over a table answer restated as composite `identity`, holds - for the composite: every warning but a `whole_total_available` naming a destination - total that a component of the composite reading one side of its destinations leaves out - (`code_index.composites.destination_population`, `RolePopulation.named`). "All - Recipients, Total" counts the core contributions a bilateral reading leaves out.""" - if warning.code != WHOLE_TOTAL_AVAILABLE: - return True - total = (warning.params or {}).get("total") - ref = composite_ref(identity.model_dump(mode="json", include={"facets"})) - composite = DECLARED_COMPOSITES.get(ref) if ref is not None else None - if composite is None or not isinstance(total, str): - return True - index = get_index() - party = index.parties.get(total) - return not any( - party is not None - and party.publishes("destination", (recipe.dataset,)) - and total not in destination_population(index, recipe.dataset, composite=ref).named - for recipe in composite.components - if recipe.destinations is not None - ) - - -def cancel_dropped_tasks(before: RegionResult, after: RegionResult) -> None: - """Cancel each trailing task `before` holds that `after`, its restatement, does not - (`with_composite_answer`). `Task.cancel` is not thread-safe, so the caller runs this on - the event loop the tasks belong to.""" - held = (after.slot_options_task, after.place_options_task, after.observations_task) - for task in (before.slot_options_task, before.place_options_task, before.observations_task): - if task is not None and all(task is not kept for kept in held): - task.cancel() - - def with_runnable_operations(region: RegionResult, operations: Sequence[Operation]) -> RegionResult: """Return `region` carrying the `operations` its data can run, and a warning for each one it cannot. diff --git a/src/search/engine/region/scope_decision.py b/src/search/engine/region/scope_decision.py index 1b686607c..6a160cb7c 100644 --- a/src/search/engine/region/scope_decision.py +++ b/src/search/engine/region/scope_decision.py @@ -4,14 +4,15 @@ The answer phase decides it once the answer's identity is known (`spec_build.answer_identity`), and the candidates path from its survivor's Spec, so an answer the binders settle as a composite reads no preview of the one table it resolved. -`pipeline._with_scope_choices` states the decision on the finished answer, and decides it -there for an answer neither path decided. Every decision over a finished answer's Spec is -guarded (`decide_guarded`).""" +The finished answer is checked against the decision once (`phase_answer.finished_scope`), +and the definite result carries the decision that check returns +(`RegionResult.scope_decision`), which `pipeline._with_scope_choices` states. A finished +answer no decision states is decided again from its Spec, which chooses nothing and fails +closed where it disagrees with the Spec (`decide_after_read`).""" from __future__ import annotations import dataclasses -import functools import logging from collections.abc import Callable, Iterable, Mapping, Sequence @@ -26,14 +27,18 @@ Operation, PartScope, ProvenanceRef, + ProviderBinding, ResolvedFacet, Spec, Warning, ) +from search.scope import build as scope_build from search.scope.basis import basis_dataset, unbound_basis_record -from search.scope.build import build_part_scope +from search.scope.edits import apply_binding from search.scope.provider import ( CONTRIBUTIONS, + Withheld, + WithheldWhy, bilateral_default_reaches, bilateral_default_reading, bilateral_default_withheld, @@ -41,7 +46,7 @@ with_summed_provider, without_unoffered_multilateral, ) -from search.scope.state import ScopeState +from search.scope.state import AxisRecord, ScopeState from search.scope.tables import PartFacts, ScopeFacts from series_coverage.shared import CoverageSnapshot @@ -52,7 +57,7 @@ class ScopeDecision: """The scope the answer `spec_id` states, built over `facts`; None when it states none. `composite` when its compiled identity is a declared composite (`composite_ref`), which - the answer is then restated as (`scope.composite.assemble_composite`). `warning` is the + the answer's Spec is then minted as (`spec_build.build_spec(composite=)`). `warning` is the answer's `bilateral_reading_unavailable` warning where it reads total ODA because the bilateral default could not apply (`bilateral_withheld_warning`).""" @@ -63,6 +68,29 @@ class ScopeDecision: warning: Warning | None = None +@dataclasses.dataclass(frozen=True) +class ProviderChoice: + """What a fresh reading chooses of its provider scope before its scope is built + (`choose_provider`), which `build_part_scope` applies to the identity it is given. + + `states_scope` is false where the answer states no scope at all (`states_no_scope`, with + the bilateral default not applied). `bilateral` is the composite the bilateral default + reads (`provider.bilateral_default_reading`), None where the reading keeps the table. + `withheld` is why the default reaches the reading and cannot apply + (`provider.bilateral_default_withheld`), None otherwise. `selection` is the part's + selection with the entries the default served removed and the summed and multilateral + words read (`provider.with_summed_provider`, `provider.without_unoffered_multilateral`), + and `provider` the provider record settled over the served identity + (`provider.settle_provider`), None where the axis does not apply or no scope is stated. + The scope build writes into a copy, so `selection` keeps the entries it holds.""" + + states_scope: bool + bilateral: str | None + withheld: Withheld | None + selection: ScopeState + provider: AxisRecord | None + + def answer_scope_facts( *, entity_groups: Mapping[str, EntityGroup], @@ -130,7 +158,7 @@ def guarded_scope_decision( table it resolved. `decide` builds the facts it reads, so an error there is guarded too. Where the bilateral default reaches `identity`, that answer reads DAC2a's total ODA with no scope to say so, so the fallback carries the `bilateral_reading_unavailable` warning - with `why` `scope_undecided` (`_undecided_warning`). + with `why` `scope_undecided` (`undecided_choice`). A composite its components cannot serve (`CompositeNotServable`) or a read the decision lost (`degraded_reads.LOST_READ_ERRORS`) is logged, and the answer can be stored. Any @@ -143,74 +171,154 @@ def guarded_scope_decision( are those of an empty part, which nothing reads of a decision that is not a composite.""" try: return decide() - except (CompositeNotServable, *LOST_READ_ERRORS): + except Exception as exc: + note_undecided(spec_id, exc) + facts = answer_scope_facts( + entity_groups={}, facets={}, sources=(), operations=operations, coverage=coverage + ) + choice = undecided_choice(identity, selection=selection, facts=facts, subject=spec_id) + return build_part_scope(identity, choice, spec_id=spec_id, facts=facts) + + +def note_undecided(subject: str, exc: Exception) -> None: + """Log `exc`, raised deciding the scope or provider choice of `subject`, whose answer + then reads as one table that states no scope. A composite its components cannot serve + (`CompositeNotServable`) or a read the decision lost (`degraded_reads.LOST_READ_ERRORS`) + is a warning, and the answer can be stored. Any other error is a fault in the scope + code, logged with its traceback and noted as a degraded read (`note_degraded_read`), so + the answer is served but no cache stores it.""" + if isinstance(exc, (CompositeNotServable, *LOST_READ_ERRORS)): logger.warning( "scope decision for %s failed; the answer reads as one table that states no scope", - spec_id, - exc_info=True, - ) - except Exception: - logger.exception( - "scope decision for %s raised; the answer reads as one table that states no scope " - "and is not cached", - spec_id, + subject, + exc_info=exc, ) - note_degraded_read("scope decision") - facts = answer_scope_facts( - entity_groups={}, facets={}, sources=(), operations=operations, coverage=coverage + return + logger.error( + "scope decision for %s raised; the answer reads as one table that states no scope " + "and is not cached", + subject, + exc_info=exc, ) - return ScopeDecision( - spec_id=spec_id, - facts=facts, - scope=None, - composite=False, - warning=_undecided_warning(spec_id, identity, selection=selection, facts=facts), + note_degraded_read("scope decision") + + +def undecided_choice( + identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts, subject: str +) -> ProviderChoice: + """The provider choice of a reading of `identity`, named `subject` in logs, whose + choice or scope could not be decided: it states no scope and keeps the table. Where the + bilateral default reaches `identity` (`provider.bilateral_default_reaches`) and no + stated or carried provider entry asks for total ODA, the reading is DAC2a's total ODA + with no scope to say so, so it is withheld with `why` `scope_undecided`. A reach check + that raises too is logged and withholds nothing.""" + return ProviderChoice( + states_scope=False, + bilateral=None, + withheld=Withheld(why="scope_undecided") + if _undecided_reach(identity, selection, facts, subject) + else None, + selection=selection, + provider=None, ) -def _undecided_warning( - spec_id: str, identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts -) -> Warning | None: - """The `bilateral_reading_unavailable` warning, `why` `scope_undecided`, for the answer - `spec_id` of `identity` whose scope decision raised, where the bilateral default reaches - `identity` (`provider.bilateral_default_reaches`) and no stated or carried provider entry - asks for total ODA. None otherwise, and where the reach check raises too, which is - logged.""" +def _undecided_reach( + identity: BaseIdentity, selection: ScopeState, facts: ScopeFacts, subject: str +) -> bool: + """Whether an undecided reading of `identity` says it reads total ODA + (`undecided_choice`).""" stated = selection.entries_on("provider") if stated and stated[0].option_id == CONTRIBUTIONS: - return None + return False try: - reaches = bilateral_default_reaches(identity, facts) + return bilateral_default_reaches(identity, facts) except Exception: - logger.exception("bilateral default reach of %s could not be read", spec_id) - return None - return _withheld_warning(spec_id, "scope_undecided") if reaches else None + logger.exception("bilateral default reach of %s could not be read", subject) + return False + + +def states_answer(decision: ScopeDecision, spec: Spec, *, minted: bool, read_within: bool) -> bool: + """Whether `decision` is the scope the finished answer `spec` states: decided for its + spec_id, compiling the composite `spec` is where `spec` is minted as one (`minted`), and + compiling none where it is not. A Spec whose rows were read within a composite's + population (`read_within`) is stated only as that composite, so where it is not minted + as one, no decision states it.""" + if decision.spec_id != spec.spec_id: + return False + if not minted: + return not read_within and not decision.composite + return ( + decision.composite + and decision.scope is not None + and decision.scope.compiled.spec_id == spec.spec_id + ) -def decide_guarded( +def decide_after_read( spec: Spec, *, + minted: bool, + read_within: bool, selection: ScopeState, operations: Sequence[Operation], coverage: CoverageSnapshot, -) -> ScopeDecision: - """The scope the finished answer `spec` states (`decide_spec_scope`), guarded - (`guarded_scope_decision`), so a decision that raises leaves `spec` as the one table it - resolved.""" - return guarded_scope_decision( - spec.spec_id, - functools.partial( - decide_spec_scope, - spec, - selection=selection, - operations=operations, - coverage=coverage, - ), - identity=spec.definition.identity, - selection=selection, - operations=operations, - coverage=coverage, +) -> ScopeDecision | None: + """The scope the finished answer `spec` states where no decision made before its read + states it (`states_answer`). The reading was decided once, before the read, so this + decision chooses nothing. It is decided from `spec` (`decide_spec_scope`) and kept only + where it states `spec` as minted, the composite `spec` is where `minted`, no composite + otherwise. + + A Spec whose rows were read within a composite's population (`read_within`) and that is + not minted as that composite is a fault before anything is decided: no decision can label + those rows, the table's would state the whole table and a composite's would not state the + Spec. A decision that raises, or that states any other scope, is a fault too. A fault is + logged at ERROR and noted as a degraded read, so no cache stores the answer, and the + answer fails closed. A Spec minted as a composite, or read within a composite's + population, has no table label to stand under, so the decision is None and the caller + answers no_data `facet_not_available`. Any other answer reads as the one table it + resolved and states no scope, saying it reads total ODA where the bilateral default + reaches it (`undecided_choice`).""" + if read_within and not minted: + logger.error( + "the finished answer %s was read within a composite's population but is not " + "minted as the composite; it fails closed and is not cached", + spec.spec_id, + ) + note_degraded_read("scope decision") + return None + try: + decision = decide_spec_scope( + spec, selection=selection, operations=operations, coverage=coverage + ) + except Exception as exc: + logger.error( + "scope decision for the finished answer %s raised; it fails closed and is not cached", + spec.spec_id, + exc_info=exc, + ) + else: + if states_answer(decision, spec, minted=minted, read_within=read_within): + return decision + logger.error( + "the finished answer %s is %s, but its scope decided after the read %s; it fails " + "closed and is not cached", + spec.spec_id, + "minted as a composite" if minted else "its table", + "compiles the composite " + decision.scope.compiled.spec_id + if decision.composite and decision.scope is not None + else "compiles no composite", + ) + note_degraded_read("scope decision") + if minted or read_within: + return None + facts = answer_scope_facts( + entity_groups={}, facets={}, sources=(), operations=operations, coverage=coverage ) + identity = spec.definition.identity + choice = undecided_choice(identity, selection=selection, facts=facts, subject=spec.spec_id) + return build_part_scope(identity, choice, spec_id=spec.spec_id, facts=facts) def states_no_scope(identity: BaseIdentity, selection: ScopeState) -> bool: @@ -220,37 +328,81 @@ def states_no_scope(identity: BaseIdentity, selection: ScopeState) -> bool: return not selection.records and basis_dataset(identity, get_index()) is None -def decide_scope( - spec_id: str, identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts -) -> ScopeDecision: - """The scope the definite answer `spec_id` of `identity` states, built from the axis - records the pass's binders hold (`selection`) and the provider scope settled over the - identity (`provider.settle_provider`), after words summing the bilateral and - multilateral lines are read as the answer's dataset sums them - (`provider.with_summed_provider`) and words naming multilateral ODA are dropped where - that dataset offers no such option (`provider.without_unoffered_multilateral`). Only - the entries a binder applied compile (`AxisRecord.entry`). The other entries that attach - to the part (`ScopeState.attached`) are reported in `PartScope.dropped_entries`. An - answer the recorded axes do not reach, as a region read at its published total, states - none. A part whose binders settled no axis, such as one naming no place, states the - scope of its identity where the measure-basis axis applies to it, with the stated or - carried basis that identity holds applied (`basis.unbound_basis_record`). A fresh - search does not refuse its scope (`build_part_scope` with `refuse` false). +def choose_provider( + identity: BaseIdentity, selection: ScopeState, facts: ScopeFacts +) -> ProviderChoice: + """The provider choice of a fresh reading of `identity` under the axis records and + entries the pass's binders hold (`selection`), read in `facts` (`ProviderChoice`). DAC2a's total ODA is read at its bilateral destinations alone where the part names none - or only bilateral ones (`provider.with_bilateral_default`), before any axis is read, so - the scope's base is that composite, with or without a provider axis. A part with no axis - then states a scope with no choice, whose compiled identity is the composite. A provider - entry asking for total ODA withholds that default (`provider.bilateral_default_reading`). - A part with no provider axis whose default could not apply carries the warning saying - so (`bilateral_withheld_warning`).""" - warning = bilateral_withheld_warning(spec_id, identity, selection=selection, facts=facts) + or only bilateral ones (`provider.with_bilateral_default`), before any axis is read. A + provider entry asking for total ODA withholds that default + (`provider.bilateral_default_reading`). The provider record is settled over the + identity so served, once words summing the bilateral and multilateral lines are read as + its dataset sums them and words naming multilateral ODA are dropped where that dataset + offers no such option. An answer whose default does not apply and whose binders settled + no axis, on an identity the measure-basis axis does not apply to, states no scope, and + nothing more is settled.""" + withheld = bilateral_default_withheld(identity, selection, facts) served, selection = bilateral_default_reading(identity, selection, facts) if served is identity and states_no_scope(identity, selection): + return ProviderChoice( + states_scope=False, + bilateral=None, + withheld=withheld, + selection=selection, + provider=None, + ) + bilateral = ( + None + if served is identity + else composite_ref(served.model_dump(mode="json", include={"facets"})) + ) + # Words summing the bilateral and multilateral lines ask for the dataset's own sum. + selection = with_summed_provider(served, selection, facts) + # "Multilateral ODA" names DAC1's row only on a dataset that publishes it. + selection = without_unoffered_multilateral(served, selection, facts) + return ProviderChoice( + states_scope=True, + bilateral=bilateral, + withheld=withheld, + selection=selection, + provider=settle_provider(served, selection, facts), + ) + + +def build_part_scope( + identity: BaseIdentity, choice: ProviderChoice, *, spec_id: str, facts: ScopeFacts +) -> ScopeDecision: + """The scope the definite answer `spec_id` of `identity` states under the provider + `choice` made for it (`choose_provider`), built over `facts`. It chooses nothing. The + composite `choice.bilateral` names is bound to `identity` as the bilateral default binds + it, and `choice.provider` is written as the provider record. + + The scope is built from the axis records the pass's binders hold + (`choice.selection`) and the provider record. Only the entries a binder applied compile + (`AxisRecord.entry`). The other entries that attach to the part (`ScopeState.attached`) + are reported in `PartScope.dropped_entries`. An answer the recorded axes do not reach, as + a region read at its published total, states none. A part whose binders settled no axis, + such as one naming no place, states the scope of its identity where the measure-basis + axis applies to it, with the stated or carried basis that identity holds applied + (`basis.unbound_basis_record`). A fresh search does not refuse its scope + (`search.scope.build.build_part_scope` with `refuse` false). + + Where the bilateral default applies, the scope's base is its composite, with or without + a provider axis, and a part with no axis states a scope with no choice, whose compiled + identity is the composite. A choice whose default could not apply carries the warning + saying so (`bilateral_withheld_warning`).""" + warning = None if choice.withheld is None else _withheld_warning(spec_id, choice.withheld) + if not choice.states_scope: return ScopeDecision( spec_id=spec_id, facts=facts, scope=None, composite=False, warning=warning ) - identity = served + if choice.bilateral is not None: + identity = apply_binding( + identity, ProviderBinding(scope="bilateral", provenance=choice.bilateral), [] + ) + selection = choice.selection records = dict(selection.records) if not records: # No binder settled an axis, as on a part naming no place. The basis axis applies @@ -258,15 +410,10 @@ def decide_scope( stated = selection.entries_on("measure_basis") if (basis := unbound_basis_record(identity, get_index(), stated)) is not None: records[basis.axis] = basis - # Words summing the bilateral and multilateral lines ask for the dataset's own sum. - selection = with_summed_provider(identity, selection, facts) - # "Multilateral ODA" names DAC1's row only on a dataset that publishes it. - selection = without_unoffered_multilateral(identity, selection, facts) settled = ScopeState(entries=selection.entries, records=records) - provider = settle_provider(identity, selection, facts) - if provider is not None: - settled.write(provider) - built = build_part_scope( + if choice.provider is not None: + settled.write(choice.provider) + built = scope_build.build_part_scope( base=identity, entries=[], records=settled.records, @@ -288,16 +435,40 @@ def decide_scope( ) -_WITHHELD_REASONS: Mapping[str, str] = { +def decide_scope( + spec_id: str, identity: BaseIdentity, *, selection: ScopeState, facts: ScopeFacts +) -> ScopeDecision: + """The scope the definite answer `spec_id` of `identity` states: the provider choice + made over `identity` itself (`choose_provider`), applied to it (`build_part_scope`). + A fresh reading whose choice is made before its read applies it to the identity minted + after it (`reading.elect_reading`).""" + return build_part_scope( + identity, choose_provider(identity, selection, facts), spec_id=spec_id, facts=facts + ) + + +READ_WINDOW_NOT_COVERED: WithheldWhy = "read_window_not_covered" +"""The `why` of an answer whose reading was elected over the years its read reads +(`reading.elect_reading`), where the bilateral destinations cover the range the answer +settled on but not every year read, so it reads total ODA.""" + +_WITHHELD_REASONS: Mapping[WithheldWhy, str] = { "destination_unclassified": ( "the data holds a destination no side declaration places, so the bilateral " "destinations cannot be told apart" ), "option_not_covered": "the bilateral destinations do not cover every year of the answer", + READ_WINDOW_NOT_COVERED: ( + "the bilateral destinations do not cover every year the request reads, though they " + "cover the years shown" + ), "scope_undecided": "the answer's scope could not be decided", + "slot_not_served": ( + "the bilateral destinations cannot be read with what the part binds ({slot})" + ), } -"""The message's reason for each `why` with words of its own; any other `why` names the -slot the bilateral composite cannot serve.""" +"""The message's reason for each `why` (`provider.WithheldWhy`), `{slot}` the property of the +slot a `slot_not_served` reason names.""" def bilateral_withheld_warning( @@ -309,26 +480,25 @@ def bilateral_withheld_warning( DAC2a figures are total ODA, core contributions to multilateral agencies included. Its params name why: the reason `provider.bilateral_default_withheld` gives. None otherwise.""" - why = bilateral_default_withheld(identity, selection, facts) - return None if why is None else _withheld_warning(spec_id, why) + withheld = bilateral_default_withheld(identity, selection, facts) + return None if withheld is None else _withheld_warning(spec_id, withheld) -def _withheld_warning(spec_id: str, why: str) -> Warning: - """The `bilateral_reading_unavailable` warning for the reading `spec_id`, naming `why`.""" +def _withheld_warning(spec_id: str, withheld: Withheld) -> Warning: + """The `bilateral_reading_unavailable` warning for the reading `spec_id`, its params + naming why the bilateral default is `withheld` (`Withheld.params`).""" return Warning( code=BILATERAL_READING_UNAVAILABLE, severity="info", message=( "These figures are total ODA, which includes donors' core contributions to " - f"multilateral agencies: {_withheld_reason(why)}." + f"multilateral agencies: {_withheld_reason(withheld)}." ), spec_id=spec_id, - params={"why": why}, + params=withheld.params, ) -def _withheld_reason(why: str) -> str: - """The message's words for `why` (`_WITHHELD_REASONS`), else the slot it names.""" - return _WITHHELD_REASONS.get( - why, f"the bilateral destinations cannot be read with what the part binds ({why})" - ) +def _withheld_reason(withheld: Withheld) -> str: + """The message's words for `withheld` (`_WITHHELD_REASONS`), naming its slot.""" + return _WITHHELD_REASONS[withheld.why].format(slot=withheld.slot) diff --git a/src/search/engine/region/state.py b/src/search/engine/region/state.py index aab47d1c7..faed6e581 100644 --- a/src/search/engine/region/state.py +++ b/src/search/engine/region/state.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Literal, cast from code_index.members import FamilyQuery +from code_index.scope_conventions import RolePopulation from code_index.shared import get_index, get_measure_index from code_index.tokens import word_tokens from code_index.types import BroadReading, summable_axes @@ -163,24 +164,75 @@ # Shares the same runtime cycle, so this import is type-checking only. from search.engine.region.place_options import PlaceOptionsPool + from search.engine.region.reading import ReadingDecision from search.engine.region.related_read import RelatedRead # `requirements` imports this module. from search.engine.region.requirements import Judgement - from search.engine.region.scope_decision import ScopeDecision - -__all__ = ["EngineOptions", "ResolveState", "UnLeg", "raised_facts"] + from search.scope.composite import CompositeMint + from series_coverage.index import CoverageIndex + +__all__ = [ + "EngineOptions", + "MintedComposite", + "ResolveState", + "UnLeg", + "raised_facts", + "whole_totals_as_read", +] # A served measure partition and its sibling partitions (`bound_values.observed_measure_family`). type MeasureFamily = tuple[MeasurePartition | None, tuple[MeasurePartition, ...]] +@dataclass(frozen=True, slots=True, kw_only=True) +class MintedComposite: + """A definite answer minted as the composite `mint` names (`spec_build.build_spec`): + `read` is the identity its rows were read for, the composite's table identity with each + population role open within its members, and `populations` the roles bound to a + population (`identity_edit.TableRead.populations`), as a resubmit of the composite reads + them (`resubmit.resolve_spec_resubmit`).""" + + mint: CompositeMint + read: BaseIdentity + populations: Mapping[str, RolePopulation] + + +def whole_totals_as_read( + spec: Spec, minted: MintedComposite | None, *, coverage: CoverageIndex +) -> list[Warning]: + """`whole_total_available` for `spec` (`whole_totals.whole_total_warnings`), read over the + identity its rows were read for: where `spec` is the Spec minted as `minted`, the + composite's table identity, read within its populations (`MintedComposite.read`, + `MintedComposite.populations`), else `spec`'s own.""" + if minted is None: + return whole_total_warnings( + spec, identity=spec.definition.identity, populations=None, coverage=coverage + ) + return whole_total_warnings( + spec, identity=minted.read, populations=minted.populations, coverage=coverage + ) + + def raised_facts(departed: Iterable[Departed]) -> dict[WarningKey, DepartureFact]: """Each departure fact in `departed` under the key of the warning that discloses it.""" return {warning_key(warning): fact for warning, fact in departed} +@dataclass(frozen=True, slots=True) +class AnswerMark: + """What a part had raised and judged when a producer started on its definite answer + (`ResolveState.mark_answer`): its warnings, departure facts, re-election warning and + judgement. A refused answer restores them (`ResolveState.restore`), so it contributes + no warning, departure fact or judgement.""" + + warnings: tuple[Warning, ...] + departure_facts: Mapping[WarningKey, DepartureFact] + reelection_warning: Warning | None + judgement: Judgement | None + + @dataclass(frozen=True, slots=True, kw_only=True) class EngineOptions: """Optional enrichments and clients for resolve and resubmit calls. @@ -887,20 +939,18 @@ class ResolveState: # the record of each axis they settle, from which the pass's definite answer states its # scope choices (`pipeline._with_scope_choices`). scope_selection: ScopeState = field(default_factory=ScopeState) - # The scope the definite answer states, decided before its preview launches, which - # `pipeline._with_scope_choices` reuses for that answer. A single-table answer sets it in - # `phase_answer._gather_definite_coverage` (`phase_answer._decide_answer_scope`) and - # decides it again in `phase_answer._finished_scope` when the finished answer has another - # `spec_id`; a candidates answer sets it in `candidate_collapse._assemble_definite`. None - # until then. - scope_decision: ScopeDecision | None = None - # The warnings of the request's words, its places, its shapes and its bindings: every - # warning the pass raised before it interpreted the elected shape's read - # (`phase_place_materialise._phase_materialise`), and each one raised later from those - # alone (`from_request`). A definite answer read again within a composite's population - # keeps these and takes the read's own in place of the rest - # (`composite_answer.with_population_read`). None until the read is interpreted. - request_warnings: tuple[Warning, ...] | None = None + # The provider reading the elected shape's read elected before it read + # (`reading.elect_reading`), set beside `mat_result` by + # `phase_place_materialise._run_materialise_task`: the choice the definite answer's + # scope is built from (`phase_answer._decide_answer_scope`) and the composite's table read + # the read read within. None where the part binds no total with a bilateral source, and + # where a stand-in's read answers in place of the elected shape's. + reading: ReadingDecision | None = None + # The composite the definite answer's Spec is minted as (`spec_build.build_spec`), with + # the identity its rows were read for and the populations its roles were read within, + # set once the answer's check returns a decision (`phase_answer.finished_scope`). + # A flagged match minted at its own build carries its own (`phase_shared.FlaggedMatch`). + minted: MintedComposite | None = None # Shared warnings list; mutations here affect every stage holding the list. warnings: list[Warning] # The departure facts raised beside warnings, each under its warning's key @@ -1537,6 +1587,27 @@ def all_qualifier_matches(self) -> tuple[QualifierMatch, ...]: """Every vocabulary's qualifier matches, in vocabulary order (`merge_lists`).""" return merge_lists(self.query_qualifier_matches.values()) + def mark_answer(self) -> AnswerMark: + """What the part has raised and judged so far, taken where a producer's definite + answer becomes specific: everything raised or judged after it is that answer's own.""" + return AnswerMark( + warnings=tuple(self.warnings), + departure_facts=dict(self.departure_facts), + reelection_warning=self.reelection_warning, + judgement=self.judgement, + ) + + def restore(self, mark: AnswerMark) -> None: + """Make the part's warnings, departure facts, re-election warning and judgement + those of `mark`, the list and the mapping in place, since stages hold them: a + warning raised since is dropped, one reworded since reads as it did, and the + answer's judgement is undone.""" + self.warnings[:] = mark.warnings + self.departure_facts.clear() + self.departure_facts.update(mark.departure_facts) + self.reelection_warning = mark.reelection_warning + self.judgement = mark.judgement + def depart(self, warning: Warning, fact: DepartureFact) -> None: """Append `warning` and record `fact`, the departure it discloses, under its key.""" self.warnings.append(warning) @@ -1672,13 +1743,19 @@ def family_total_absent_departures(self, specs: Iterable[Spec]) -> list[Departed specs, self.family_total_absent, variable_text=self.variable, label_of=self.label_dcids ) - def from_request(self, *warnings: Warning) -> None: - """Record `warnings`, raised once the read is interpreted from the request's words, - places or election and not from the rows read, among the request's warnings - (`request_warnings`). The caller raises them in `warnings` as before. Before the read - is interpreted every warning is the request's, so nothing is recorded.""" - if self.request_warnings is not None: - self.request_warnings = (*self.request_warnings, *warnings) + def minted_as( + self, spec: Spec, matched: Sequence[MintedComposite] = () + ) -> MintedComposite | None: + """The composite `spec` is minted as, among the part's answer's (`minted`) and + `matched`, None where it is minted as none.""" + return next( + ( + minted + for minted in (self.minted, *matched) + if minted is not None and spec.spec_id == minted.mint.spec_id + ), + None, + ) def settled_dataset(self, candidate: _Candidate | None = None) -> str | None: """The dataset the part's run has settled on for `candidate`, else for the elected @@ -1734,6 +1811,7 @@ def no_data( denominator: NoDataDenominator | None = None, role_assignments: tuple[Mapping[str, RoleAxis], ...] = (), candidate: _Candidate | None = None, + matched: Sequence[MintedComposite] = (), ) -> RegionResult: """Build a no-data result, preserving `cause`. A `no_observations` answer over an elected result holding data outside the years read carries its `year_unavailable` @@ -1751,8 +1829,10 @@ def no_data( `unanswerable` when one did and the part still could not be answered from it. The part holds `flagged_match` and `alternate_match`, so those Specs raise - `family_total_absent` and `whole_total_available` (`whole_totals.whole_total_warnings`). - `elected_stat_vars` names members without a Spec and raises none. + `family_total_absent` and `whole_total_available` (`whole_totals.whole_total_warnings`), + each read over the identity its rows were read for (`whole_totals_as_read`): `matched` holds + the composites a match minted at its own build is minted as, which the part's answer + (`minted`) is not. `elected_stat_vars` names members without a Spec and raises none. """ if (refused := self.source_code_no_data(reason, llm_usage=llm_usage)) is not None: return refused @@ -1775,7 +1855,9 @@ def no_data( *( warning for match in matches - for warning in whole_total_warnings(match, coverage=self.coverage.index) + for warning in whole_totals_as_read( + match, self.minted_as(match, matched), coverage=self.coverage.index + ) ), ), departure_facts={**self.departure_facts, **raised_facts(family)}, diff --git a/src/search/engine/region/whole_totals.py b/src/search/engine/region/whole_totals.py index bed0439f8..9d121ecc1 100644 --- a/src/search/engine/region/whole_totals.py +++ b/src/search/engine/region/whole_totals.py @@ -292,15 +292,18 @@ def population_totals(axes: Collection[RoleAxis]) -> tuple[str, ...]: def whole_total_warnings( spec: Spec, *, + identity: BaseIdentity, + populations: Mapping[str, RolePopulation] | None, coverage: CoverageIndex, - populations: Mapping[str, RolePopulation] | None = None, ) -> list[Warning]: """`whole_total_available` for each flow side `spec` lists member by member, opened by `open_roles` or defaulted (`ResolvedSpec.role_axis_facts`), where the dataset elected - for its identity (`bound_values.identity_dataset`) publishes the total of that side's - whole population over the identity's requested years (`whole_reading`) and `coverage` - holds a series of the Spec's own variables at that total beside the places its other - side binds (`coverage_series.series_held`). Those variables are among the ones binding + for `identity`, the identity `spec`'s rows were read for (`bound_values.identity_dataset`), + publishes the total of that side's whole population over the identity's requested years + (`whole_reading`) and `coverage` holds a series of the Spec's own variables at that total + beside the places its other side binds (`coverage_series.series_held`). A Spec minted as + a composite states the composite's identity, and its rows were read for the composite's + table identity (`state.MintedComposite.read`), whose dataset and places these read. Those variables are among the ones binding repair asks of the total (`place_roles.holds_whole_total`), so the warning names no total binding would not serve. The warning names the total in `params`, so a caller that sums the members sees the published figure first, and is stamped with `spec`'s @@ -315,7 +318,6 @@ def whole_total_warnings( An answer that reads the other side whole states none: the source publishes no figure between two whole totals ("Official Donors, Total" to "All Recipients, Total"), so no published total stands beside the members' sum.""" - identity = spec.definition.identity dataset = identity_dataset(identity) groups = spec.resolved.entity_groups or {} if dataset is None or any(group.whole_of for group in groups.values()): diff --git a/src/search/engine/response_cache_keys.py b/src/search/engine/response_cache_keys.py index 3dceb57fb..94e4008d0 100644 --- a/src/search/engine/response_cache_keys.py +++ b/src/search/engine/response_cache_keys.py @@ -432,7 +432,8 @@ def part_memo_key( population enters as its side and the dataset it is drawn from, which with the build commit and the graph generation fix its members, so a role read open within a population and one naming the same parties key apart. A payload binding none adds nothing to the - key. + key. The composite a payload answers as enters whole, so a table answer stored for its + table identity never serves the composite. `engine_flags` are the four enrichment flags of the call's `EngineOptions`. The build commit covers the code index, and the graph generation, coverage stamp, conversion catalogue revision, origin catalogue build with its UN System setting, and UTC day cover what the graph, @@ -455,6 +456,8 @@ def part_memo_key( key: [population.side, population.source.dataset] for key, population in sorted(payload.populations.items()) } + if payload.composite is not None: + document["composite"] = payload.composite.model_dump(mode="json") return _digest("m", document) diff --git a/src/search/engine/retrieval/retrieve.py b/src/search/engine/retrieval/retrieve.py index e989ae2c1..8578bfd85 100644 --- a/src/search/engine/retrieval/retrieve.py +++ b/src/search/engine/retrieval/retrieve.py @@ -324,7 +324,7 @@ def materialise( open_axes: frozenset[RoleAxis] = frozenset(), coverage: CoverageSnapshot, facet_batch: FacetBatch | None = None, - within: Mapping[RoleAxis, frozenset[str]] | None = None, + within: Mapping[RoleAxis, frozenset[str]] | None, ) -> Materialised | NoDataDraft | MaterialisedCandidates | UnsupportedDraft: """Resolve a shape through the standard or role-bearing path. diff --git a/src/search/engine/retrieval/role_resolve.py b/src/search/engine/retrieval/role_resolve.py index bfcc46ff0..627e60f6d 100644 --- a/src/search/engine/retrieval/role_resolve.py +++ b/src/search/engine/retrieval/role_resolve.py @@ -975,7 +975,7 @@ def graph_confirm_resolve( discover_scope: bool = False, open_axes: frozenset[RoleAxis] = frozenset(), coverage_snapshot: CoverageSnapshot, - within: Mapping[RoleAxis, frozenset[str]] | None = None, + within: Mapping[RoleAxis, frozenset[str]] | None, ) -> Materialised | NoDataDraft | UnsupportedDraft: """Filter a role-bearing shape's members by bound values, then confirm observations. diff --git a/src/search/engine/same_shape.py b/src/search/engine/same_shape.py index 8e1e5cd55..df1ef062f 100644 --- a/src/search/engine/same_shape.py +++ b/src/search/engine/same_shape.py @@ -53,10 +53,12 @@ from search.engine.retrieval.merged_plan import merged_read_plan from search.engine.spec_build import ( RoleAxisFactInput, + answer_identity, build_spec, entity_from_resolved, spec_named_groups, ) +from search.engine.spec_id import identity_spec_id from search.engine.unit.conversion import with_unit_conversion from search.models import ( BindingSet, @@ -82,6 +84,7 @@ from search.engine.retrieval.member_facts import MemberFacts from search.engine.retrieval.read_plan import ReadPlan from search.models import ( + BaseIdentity, Coverage, Entity, FacetOptions, @@ -97,6 +100,7 @@ SpecObservations, Warning, ) + from search.scope.composite import CompositeMint from series_coverage.index import CoverageIndex ScopeReader = Callable[["RegionResult", "Sequence[RegionResult]"], "PartScope | None"] @@ -651,8 +655,8 @@ def collapse_same_shape( facet options over the merged spec (`_merge_facet_options`), preserve shared identity facets, and set ``earliest_index`` to the group's minimum. A merged region states the scope ``scope_for`` builds over the merged identity, none without it. A group - of composite answers merges into the composite over the merged identity - (`composite_answer.as_merged_composite`), and refuses without ``preview_reads``, the + of composite answers merges into the composite over the merged identity, minted from the + members' facets (`_merged_composite`), and refuses without ``preview_reads``, the request's clients. Where its members preview, a merged region previews the read of its merged identity through ``preview_reads`` (`_with_merged_preview`). The merged regions whose members embed their previews read theirs together in the calling thread @@ -738,7 +742,7 @@ def _check_merge_gates(group: list[RegionResult], base_spec: Spec) -> None: base_facets = base_spec.definition.identity.facets if any(r.spec.definition.identity.facets != base_facets for r in group[1:]): raise _RefuseMerge - # A composite answer merges only with composites, so the merge restates the composite. + # A composite answer merges only with composites, so the merge mints the composite. if len({_binds_composite(r.spec) for r in group}) > 1: raise _RefuseMerge @@ -1099,8 +1103,8 @@ def _merged_counts( `_merged_unserved`) and no download, where - `reads`, the request's clients, is absent; - - the merge is a composite, which states its components' download - (`composite_answer.as_merged_composite`); + - the merge is a composite, which states its components' download and years + (`_merged_composite`); - a member's plan was narrowed to the units a unit request elected over that member's own group (`RegionResult.units_elected`), which a resolve of the merged identity elects over the merged group; @@ -1313,6 +1317,7 @@ def _finalize_merged_region( served_filter: RoleServedFilter | None, served_members: tuple[MemberFacts, ...], served_plans: tuple[ReadPlan, ...], + composite: CompositeMint | None = None, ) -> RegionResult: """Assemble the merged region and preserve group-wide trace data. @@ -1322,7 +1327,8 @@ def _finalize_merged_region( id so they remain attached to their result, and add the facet-options read's warning. The merged region is one answer, so a warning several members raise appears once. Its preview is a read of the merged identity, of `preview_read` where the merge rebuilt its - read plan (`_with_merged_preview`). + read plan (`_with_merged_preview`). `composite` is the composite `merged_spec` was + minted as (`_merged_composite`), None where the merge minted none. """ base_resolved = group[0].spec.resolved merged_spec = merged_spec.model_copy( @@ -1366,7 +1372,9 @@ def _finalize_merged_region( served_members=served_members, served_plans=served_plans, ) - return _with_merged_preview(merged, group, reads=preview_reads, preview_read=preview_read) + return _with_merged_preview( + merged, group, reads=preview_reads, preview_read=preview_read, composite=composite + ) def _with_merged_preview( @@ -1375,20 +1383,24 @@ def _with_merged_preview( *, reads: ComponentReads | None, preview_read: PreviewRead | None, + composite: CompositeMint | None = None, ) -> RegionResult: - """`merged`, the region `group` merged into, restated as the merged identity answers and - with the relaunch that previews it when the members preview. The merged identity's - composite binding (`_binds_composite`, which the merge gate holds every member to) - chooses between them. A group of composite answers is restated as the composite over - the merged identity (`composite_answer.as_merged_composite`), whose `CompositeRelaunch` - reads the components it serves. Any other merge keeps its Spec. With `preview_read`, what - a read of the merged read plan the merge rebuilt previews (`_merged_preview_read`), it - reads that, as one answer reads its preview, and states it as its `preview_read` - (`composite_answer.PlanPreview`), so its preview and its download come from one plan. - Without it, it previews by one resubmit of its identity + """`merged`, the region `group` merged into, with the relaunch that previews it when the + members preview. The merged identity's composite binding (`_binds_composite`, which the + merge gate holds every member to) chooses the relaunch. A group of composite answers + merges into the composite over the merged identity, minted as `composite` + (`_merged_composite`), whose `CompositeRelaunch` reads the components it serves to the + later of the deadline of `reads` and the latest any member's reads ran to + (`RegionResult.reads_deadline`), and whose `role_scope_defaulted` warnings count the + composite's parties (`CompositeMint.role_scope_warnings`). Its trailing slot and place + options read the one table the merge resolved, so it carries none. With `preview_read`, + what a read of the merged read plan the merge rebuilt previews + (`_merged_preview_read`), any other merge reads that, as one answer reads its preview, + and states it as its `preview_read` (`composite_answer.PlanPreview`), so its preview and + its download come from one plan. Without it, it previews by one resubmit of its identity (`composite_answer.IdentityPreview`). Each reads through `reads`, the request's clients. - The members' own preview warnings leave the merged answer, which states its own read's. Where the members do not preview, the merged region - reads no preview. + The members' own preview warnings leave the merged answer, which states its own read's. + Where the members do not preview, the merged region reads no preview. Where the members stream their previews, `enrichment._reconcile_after_collapse` launches the relaunch and cancels the members' tasks. Where they embed them, @@ -1400,11 +1412,11 @@ def _with_merged_preview( streamed = any(r.observations_relaunch is not None for r in group) embedded = any(r.spec.resolved.observations is not None for r in group) previews = streamed or embedded - composite = _binds_composite(group[0].spec) + binds = _binds_composite(group[0].spec) unread = dataclasses.replace( merged, observations_task=None, observations_relaunch=None, preview_read=None ) - if not previews and not composite: + if not previews and not binds: return unread if reads is None: raise ValueError( @@ -1413,18 +1425,17 @@ def _with_merged_preview( # composite_answer reads through resubmit, which imports conjoin, which imports this module. from search.engine.region.composite_answer import ( PREVIEW_WARNING_CODES, + CompositeRelaunch, IdentityPreview, PlanPreview, - as_merged_composite, ) # The codes only a preview read raises, without `un_system_unavailable`, which a member's # resolve can raise too. member_codes = (PREVIEW_WARNING_CODES - {UN_SYSTEM_UNAVAILABLE}) | {DEFLATOR_PROJECTED} - unread = dataclasses.replace( - unread, warnings=tuple(w for w in merged.warnings if w.code not in member_codes) - ) - if not composite and preview_read is not None: + warnings = tuple(w for w in merged.warnings if w.code not in member_codes) + unread = dataclasses.replace(unread, warnings=warnings) + if not binds and preview_read is not None: return dataclasses.replace( unread, preview_read=preview_read, @@ -1434,12 +1445,29 @@ def _with_merged_preview( variable_labels=member_labels((merged.spec,)), ), ) - if not composite: + if not binds: return dataclasses.replace( unread, observations_relaunch=IdentityPreview(spec=merged.spec, reads=reads) ) - restated = as_merged_composite(unread, group, reads=reads) - return restated if previews else dataclasses.replace(restated, observations_relaunch=None) + deadlines = [ + deadline + for deadline in (reads.deadline, *(r.reads_deadline for r in group)) + if deadline is not None + ] + if deadlines: + reads = dataclasses.replace(reads, deadline=max(deadlines)) + if composite is not None: + warnings = composite.role_scope_warnings(warnings, merged.spec) + relaunch = CompositeRelaunch( + spec=merged.spec, reads=reads, served=None if composite is None else composite.served + ) + return dataclasses.replace( + unread, + warnings=warnings, + slot_options_relaunch=None, + place_options_relaunch=None, + observations_relaunch=relaunch if previews else None, + ) _COMBINE_TAIL_S = 0.25 @@ -1478,7 +1506,7 @@ def _read_merged_previews(pending: Sequence[RegionResult]) -> list[RegionResult] relaunch = region.observations_relaunch assert isinstance(relaunch, CompositeRelaunch | IdentityPreview | PlanPreview) relaunches.append(relaunch) - # A composite's reads run to its latest member's (`as_merged_composite`), which may pass + # A composite's reads run to its latest member's (`_with_merged_preview`), which may pass # the request clients' own. deadlines = [r.reads.deadline for r in relaunches if r.reads.deadline is not None] bound = bounded_deadline(max(deadlines) - _COMBINE_TAIL_S if deadlines else None) @@ -1596,6 +1624,37 @@ def _merged_data_years(group: list[RegionResult]) -> tuple[int, int] | None: return min(first for first, _last in spans), max(last for _first, last in spans) +def _merged_composite( + identity: BaseIdentity, member: Spec, *, reads: ComponentReads | None +) -> CompositeMint: + """The composite a merge of composite answers mints its Spec as + (`spec_build.build_spec(composite=)`): `identity`, the merged table identity, binding + `member`'s facets, the composite provenance among them, which the merge gate holds every + member to, with the components it serves read through `reads`, the request's clients + (`composite_answer.composite_served`). The merge rebuilds its Spec from its members' + slots, which bind no composite, so the identity it mints is the table identity over + every member's entities. Each member was read within the composite's population, so + the mint reads none. + + Raises `ValueError` where `reads` is None: `_collapse_group` refuses a composite group + without them.""" + if reads is None: + raise ValueError("a merge of composite answers needs the request's preview reads") + # composite_answer reads through resubmit, which imports conjoin, which imports this module. + from search.engine.region.composite_answer import composite_served + from search.scope.composite import CompositeMint + from search.scope.tables import ScopeFacts + + composite = identity.model_copy(update={"facets": member.definition.identity.facets}) + return CompositeMint( + identity=composite, + spec_id=identity_spec_id(composite), + served=tuple(composite_served(composite, reads=reads)), + facts=ScopeFacts(index=get_index(), coverage=reads.coverage), + catalogue=reads.catalogue, + ) + + def _merge_group( group: list[RegionResult], set_ref_for: Callable[[list[str]], GraphRef | None] | None, @@ -1639,34 +1698,72 @@ def _merge_group( served_members = tuple(member for r in group for member in r.served_members) served_plans = tuple(plan for r in group for plan in r.served_plans) + entities = _union_entities(group) + # Identity slots omit where; its entity buckets define the merged identity. + # Use each member's hashed buckets, not its displayed entity projection. + identity_entities = _union_identity_entities(group) + # The merge gate holds every member to the base's open roles. + open_roles = base_spec.definition.identity.open_roles or () + # All members share this request window; use the base's hashed value. + requested_window = base_spec.resolved.trace.requested_window + # Shared identity facets pass through. build_spec derives provenance from merged slots. + facet_bindings = base_spec.definition.identity.facets + facets = _merge_resolved_facets(group, bound_facet_axes) + resolved_sources = _union_sources(group) + plan = counted.plan if counted is not None else None + # A member's group survives on a role the merge leaves unchanged. + named_groups = [named for region in group for named in spec_named_groups(region.spec)] + answer = answer_identity( + merged_shape, + merged_slots, + all_stat_vars, + entities, + identity_entities=identity_entities, + open_roles=open_roles, + requested_window=requested_window, + facet_bindings=facet_bindings, + facets=facets, + resolved_sources=resolved_sources, + plan=plan, + pin_served_provenance=False, + named_groups=named_groups, + ) + composite = ( + _merged_composite(answer.identity, base_spec, reads=preview_reads) + if _binds_composite(base_spec) + else None + ) + headline = counted.headline if counted is not None else None + if composite is not None and headline is None: + # The composite's headline names the request's window, which the gate holds every + # member to. + headline = base_spec.resolved.headline + merged_spec = build_spec( + composite=composite, + answer=answer, shape=merged_shape, slots=merged_slots, stat_vars=all_stat_vars, stat_vars_member_count=stat_vars_member_count, stat_vars_member_count_exact=stat_vars_member_count_exact, walked=_merged_walked(group), - entities=_union_entities(group), - # Identity slots omit where; its entity buckets define the merged identity. - # Use each member's hashed buckets, not its displayed entity projection. - identity_entities=_union_identity_entities(group), - # The merge gate holds every member to the base's open roles. - open_roles=base_spec.definition.identity.open_roles or (), - # All members share this request window; use the base's hashed value. - requested_window=base_spec.resolved.trace.requested_window, + entities=entities, + identity_entities=identity_entities, + open_roles=open_roles, + requested_window=requested_window, coverage=counted.coverage if counted is not None else _merge_coverage(group, base_spec), - headline=counted.headline if counted is not None else None, - plan=counted.plan if counted is not None else None, + headline=headline, + plan=plan, coverage_index=counted.coverage_index if counted is not None else None, - resolved_sources=_union_sources(group), + resolved_sources=resolved_sources, slot_options=merged_slot_options, entity_options=entity_options, scope=base_spec.resolved.scope, inapplicable_slots=inapplicable_slots, unresolved_slots=unresolved_slots, - # Shared identity facets pass through. build_spec derives provenance from merged slots. - facet_bindings=base_spec.definition.identity.facets, - facets=_merge_resolved_facets(group, bound_facet_axes), + facet_bindings=facet_bindings, + facets=facets, facet_options=[], equivalent_readings=_union_equivalent_readings(group), text_recognized_properties=text_recognized, @@ -1680,25 +1777,23 @@ def _merge_group( measure_siblings=measure_siblings, open_axis_facts=open_axis_facts, role_axis_facts=role_axis_facts, - # A member's group survives on a role the merge leaves unchanged. - named_groups=[named for region in group for named in spec_named_groups(region.spec)], + named_groups=named_groups, # The merged identity's alternatives read every member's entities. measure_served_rows=measure_served_counter(served_filter), ) merged_spec = with_unit_conversion(merged_spec, members=served_members, plans=served_plans) if counted is None: # With no merged plan, `build_spec` states none of these, so they come from the - # members. A rebuilt plan states them as its download counts them. - merged_spec = merged_spec.model_copy( - update={ - "resolved": merged_spec.resolved.stating_unserved( - _merged_unserved(group, merged_spec) - ) - .stating_unserved_members(_merged_unserved_members(group)) - .stating_data_years(_merged_data_years(group)) - .stating_continued(any(region.spec.resolved.continued for region in group)) - } + # members. A rebuilt plan states them as its download counts them. A composite + # states the years its components serve. + resolved = ( + merged_spec.resolved.stating_unserved(_merged_unserved(group, merged_spec)) + .stating_unserved_members(_merged_unserved_members(group)) + .stating_continued(any(region.spec.resolved.continued for region in group)) ) + if composite is None: + resolved = resolved.stating_data_years(_merged_data_years(group)) + merged_spec = merged_spec.model_copy(update={"resolved": resolved}) merged_spec, merged_probe, facet_warning = _merge_facet_options( group, merged_spec, facet_options_for @@ -1706,6 +1801,7 @@ def _merge_group( return _finalize_merged_region( group, merged_spec, + composite=composite, preview_reads=preview_reads, preview_read=None if counted is None else _merged_preview_read(group, counted.plan), facet_probe=merged_probe, diff --git a/src/search/engine/spec_build.py b/src/search/engine/spec_build.py index 6d02759d2..8d86d9166 100644 --- a/src/search/engine/spec_build.py +++ b/src/search/engine/spec_build.py @@ -136,6 +136,9 @@ Spec, SpecAlternativeMeasure, ) + + # For its type alone: importing it at runtime closes the cycle download.py notes. + from search.scope.composite import CompositeMint from series_coverage.index import CoverageIndex @@ -1389,10 +1392,19 @@ def build_spec( measure_served_rows: Callable[[BaseIdentity], ServedRows | None] | None = None, pin_served_provenance: bool = False, answer: AnswerIdentity | None = None, + composite: CompositeMint | None, ) -> Spec: """Assemble a Spec with a deterministic spec_id. Args: + composite: The composite these arguments' identity is the table identity of + (`code_index.composites.table_identity`), when the Spec answers as the + composite. The Spec is then minted under the composite's identity and spec_id, + and each field the composite states is the mint's (`CompositeMint`): coverage + in place of ``coverage``, headline, open role axes, StatVar counts, when slot, + download, measure label, published tables and sources, provenance facet and + data years, with no observation preview. Every other field is the table + identity's. None mints the table identity itself. answer: The `answer_identity` of these same arguments, when the caller decided something over it before building. The Spec is then minted from it as given, and a caller passes it only when `shape`, `slots`, `stat_vars`, `entities`, @@ -1494,7 +1506,11 @@ def build_spec( entity_groups = answer.entity_groups index = get_index() served_datasets = answer.served_datasets - definition = SpecDefinition(spec_id=spec_id, identity=identity_doc) + definition = ( + SpecDefinition(spec_id=spec_id, identity=identity_doc) + if composite is None + else SpecDefinition(spec_id=composite.spec_id, identity=composite.identity) + ) slots = slots_in_identity_order(slots, identity_doc) stated = stated_members(identity_doc, walked) # A stated family is the one a mint of the definition digests and counts. The admitted @@ -1535,7 +1551,7 @@ def build_spec( ) resolved_entity_refs = [e.ref for e in entities] - resolved_sources = answer.resolved_sources + resolved_sources = answer.resolved_sources if composite is None else composite.sources() slot_filters = _build_slot_filters( slots, @@ -1558,7 +1574,9 @@ def build_spec( provenance = (facets or {}).get("provenance") download = ( - download_facts( + composite.download() + if composite is not None + else download_facts( plan, coverage_index, requested_window=requested_window, @@ -1572,25 +1590,40 @@ def build_spec( ) unserved = unserved_entities(plan) left_out = set(unserved_members(plan)) + resolved_stat_vars = _resolved_stat_vars( + stat_vars, + member_count=stat_vars_member_count, + member_count_exact=stat_vars_member_count_exact, + stated=stated, + identity=identity_doc, + provenance=provenance.ref if provenance is not None else None, + ) + resolved_slots = apply_slot_summability(slots, identity_doc) + stated_facets = facets or {} + published_tables = _published_tables(provenance, index) + years = data_years(plan) + if composite is not None: + coverage = composite.coverage() + headline = composite.headline(headline) + scope, resolved_role_axis_facts = composite.open_axes(scope, resolved_role_axis_facts) + resolved_stat_vars = composite.stat_vars(resolved_stat_vars) + resolved_slots = composite.slots(resolved_slots) + stated_facets = composite.facets(stated_facets) + published_tables = composite.published_tables() + observations = None + years = composite.data_years() spec = Spec( - spec_id=spec_id, + spec_id=definition.spec_id, definition=definition, resolved=ResolvedSpec( shape=wire_shape, - slots=apply_slot_summability(slots, identity_doc), + slots=resolved_slots, inapplicable_slots=inapplicable_slots or [], unresolved_slots=unresolved_slots or [], slot_options=slot_options or [], entity_options=entity_options or [], facet_options=facet_options or [], - stat_vars=_resolved_stat_vars( - stat_vars, - member_count=stat_vars_member_count, - member_count_exact=stat_vars_member_count_exact, - stated=stated, - identity=identity_doc, - provenance=provenance.ref if provenance is not None else None, - ), + stat_vars=resolved_stat_vars, entities=_resolved_entities(entities, parties=index.parties, datasets=served_datasets), entity_groups=entity_groups or None, coverage=coverage, @@ -1598,7 +1631,7 @@ def build_spec( download=download, coverage_stamp=coverage_stamp, scope=scope, - facets=facets or {}, + facets=stated_facets, trace=trace, available_at_place=available_at_place, confidence=confidence, @@ -1607,13 +1640,17 @@ def build_spec( sentence_plan=sentence_plan, equivalent_readings=equivalent_readings, alternative_measures=alternative_measures, - measure_label=measure_label.text, + measure_label=( + measure_label.text + if composite is None + else composite.measure_label(measure_label.text) + ), measure_label_axes=measure_label.axes, measure_axis_values=elected_axis_values, concept_label=sector_concept(identity_doc, index) or ", ".join(concept_qualifiers) or None, - published_tables=_published_tables(provenance, index), + published_tables=published_tables, open_axis_facts=resolved_open_axis_facts, role_axis_facts=resolved_role_axis_facts, ) @@ -1632,7 +1669,7 @@ def build_spec( frozenset(alone for group in named_groups for alone in group.named_alone), ) .stating_group_totals(role_group_totals(named_groups, hashed_entities)) - .stating_data_years(data_years(plan)) + .stating_data_years(years) .stating_continued(serves_continuation(plan)) .stating_members(stat_vars) .stating_measure_label_named_values(measure_label.named_values) diff --git a/src/search/eval/checks/materialisation.py b/src/search/eval/checks/materialisation.py index 24a059d25..643130c93 100644 --- a/src/search/eval/checks/materialisation.py +++ b/src/search/eval/checks/materialisation.py @@ -59,10 +59,10 @@ from code_index.members import SourceNotInFamily, identity_family_query, index_family from code_index.shared import get_index from dc_graph import Facet -from dfo_platform.roles import role_key_axis from graph_release.context import ReleaseNotBound from materialise.facets import Cadence, at_cadence, cadences_by_member, continuation from materialise.window import RequestedWindow, latest_period +from search.engine.identity_edit import population_members from search.engine.region.codes import ( CURRENCY_UNMAPPED_OMITTED, OBSERVATIONS_PROBE_FAILED, @@ -513,9 +513,8 @@ def _counted_reads( ) members = {row.dcid for row in family.rows} within = { - _COVERAGE_ROLE_FOR_AXIS[axis]: frozenset(population.members) - for key, population in component.populations.items() - if (axis := role_key_axis(key)) is not None + _COVERAGE_ROLE_FOR_AXIS[axis]: members + for axis, members in population_members(component.populations).items() } reads.append( _CountedRead( diff --git a/src/search/models/envelope.py b/src/search/models/envelope.py index f8a950db4..3eb115440 100644 --- a/src/search/models/envelope.py +++ b/src/search/models/envelope.py @@ -74,10 +74,12 @@ class Warning(StrictModel): "unavailable. whole_total_available: {total, total_label, whole_of}, the dcid " "and label of the total the source publishes for the side the answer lists " "member by member, and the classes of party it covers, comma-joined (official, " - "private, residual). bilateral_reading_unavailable: {why}, why a DAC2a total " - "ODA answer could not read its bilateral destinations alone: " - "destination_unclassified, option_not_covered, scope_undecided, or the slot " - "the bilateral composite cannot serve. " + "private, residual). bilateral_reading_unavailable: {why, slot}, why a DAC2a " + "total ODA answer could not read its bilateral destinations alone, one of the " + "reasons dfo-spec-kit's registries/warning-reasons.json lists for the code " + "(destination_unclassified, option_not_covered, read_window_not_covered, " + "scope_undecided, slot_not_served), and for slot_not_served alone the property " + "of the slot the bilateral composite cannot serve. " "currency_unmapped_omitted: {count, entities, entity_labels}, " "how many entities the conversion left out, the dcids of the first ten, sorted, " "and each one's name, in entities order, '' where the conversion tables hold " diff --git a/src/search/models/request.py b/src/search/models/request.py index b25077ba4..225c0b1fe 100644 --- a/src/search/models/request.py +++ b/src/search/models/request.py @@ -624,6 +624,12 @@ class ResubmitIdentityPayload(StrictModel): (`region.resubmit.resolve_spec_resubmit`). Only the engine sets it, from a composite's table or component identity, so it stays out of every dump; the part memo keys it by the dataset whose population it is (`response_cache_keys.part_memo_key`). + + `composite` is the composite identity whose table identity `identity` is + (`code_index.composites.table_identity`), when the part answers as that composite. The + resolver mints its Spec as the composite (`spec_build.build_spec` with a + `scope.composite.CompositeMint`). Only the engine sets it, so it stays out of every + dump, and the part memo keys it (`response_cache_keys.part_memo_key`). """ identity: BaseIdentity = Field( @@ -642,6 +648,14 @@ class ResubmitIdentityPayload(StrictModel): "the population's members; set by the engine alone" ), ) + composite: BaseIdentity | None = Field( + default=None, + exclude=True, + description=( + "the composite identity whose table identity `identity` is, which the answer " + "is minted as; set by the engine alone" + ), + ) # --------------------------------------------------------------------------- diff --git a/src/search/models/scope.py b/src/search/models/scope.py index 3f20afe96..91676c8f0 100644 --- a/src/search/models/scope.py +++ b/src/search/models/scope.py @@ -185,7 +185,8 @@ class OptionAvailability(StrictModel): detail: dict[str, str] = Field( default_factory=dict, description=( - "the code's parameters: `why` for provider_not_offered, `dataset` for " + "the code's parameters: `why` for provider_not_offered, with `slot`, the " + "property at fault, where why is slot_not_served, `dataset` for " "basis_not_published and contributions_counted, `destination` for " "multilateral_destination, the multilateral agency the part names as recipient, " "whose flows are core contributions, `entry` and `cause` for conflicts_with" diff --git a/src/search/observations.py b/src/search/observations.py index 1f5f28a45..62969476a 100644 --- a/src/search/observations.py +++ b/src/search/observations.py @@ -30,8 +30,10 @@ from code_index.composites import DECLARED_COMPOSITES from code_index.query import CodeIndex +from code_index.shared import get_index from code_index.types import CompositeDeclaration from description.blocks import documentation_link +from materialise.bundle_sources import composite_bundle_label from materialise.enrichment import read_party_kinds from materialise.provenance_doc import ProvenanceNameReader, provenance_names from materialise.roles import axis_for_role @@ -159,10 +161,10 @@ def provenance_facet_ref(dcid: str, labels: Mapping[str, str]) -> SeriesFacetRef """A series' provenance-axis facet ref for `dcid`, with the catalogue row's members and the route of its document. A provenance the catalogue does not list takes its name from `labels`, if any. A declared composite (`code_index.composites`), which neither the graph - nor the catalogue holds, takes its declared label and no document route.""" + nor the catalogue holds, takes its bundle name (`composite_facet_ref`) and no document route.""" composite = DECLARED_COMPOSITES.get(dcid) if composite is not None: - return composite_facet_ref(composite) + return composite_facet_ref(composite, index=get_index()) row = current_catalogue().provenance(dcid) documentation = documentation_link(dcid) if row is None: @@ -173,23 +175,25 @@ def provenance_facet_ref(dcid: str, labels: Mapping[str, str]) -> SeriesFacetRef ) -def composite_facet_ref(composite: CompositeDeclaration) -> SeriesFacetRef: +def composite_facet_ref(composite: CompositeDeclaration, *, index: CodeIndex) -> SeriesFacetRef: """The provenance-axis facet ref of a declared composite, which neither the graph nor the - catalogue holds: its declared label, and the source and Data Commons its component tables' + catalogue holds: its bundle name in `index` (`composite_bundle_label`), which reads apart + from a component table of the same label, and the source and Data Commons its component tables' catalogue rows give when every row names the same one. A component table the catalogue does not list, or rows that name different sources or Data Commons, leave that field out. It carries no document route, since the Graph API documents each component table and no composite.""" + label = composite_bundle_label(index, composite) rows = [current_catalogue().provenance(recipe.dataset) for recipe in composite.components] listed = [row for row in rows if row is not None] if len(listed) < len(rows): - return SeriesFacetRef(ref=composite.provenance, label=composite.label) + return SeriesFacetRef(ref=composite.provenance, label=label) sources = {(row.source.ref, row.source.label) for row in listed} origins = {(row.data_commons.id, row.data_commons.label) for row in listed} source, origin = _source_and_origin(listed[0]) return SeriesFacetRef( ref=composite.provenance, - label=composite.label, + label=label, source=source if len(sources) == 1 else None, data_commons=origin if len(origins) == 1 else None, ) diff --git a/src/search/render/heading.py b/src/search/render/heading.py index bc2fada5a..b1811b04e 100644 --- a/src/search/render/heading.py +++ b/src/search/render/heading.py @@ -260,6 +260,16 @@ def _normalize_entities(entities: list[Entity] | list[ResolvedEntity]) -> list[_ return result +def _asks_latest(spec: Spec) -> bool: + """Whether `spec` answers a request for the latest year: its identity requests it, or, + for a composite, whose identity holds the latest year it elected + (`provider.elect_latest_window`), the request did (`ResolutionTrace.requested_window`).""" + return ( + spec.definition.identity.requested_window.kind == "latest" + or spec.resolved.trace.requested_window == "latest" + ) + + def heading_for( spec: Spec, *, @@ -275,7 +285,8 @@ def heading_for( sibling-distinguishing constraints visible. The slots the identity's five-tuple pins (`identity_pinned_properties`) state no how clause. The byline states the measurement method the identity binds (`_bound_method_text`), and a unit converted to - constant prices whose prices deflated it (`UnitConversion.deflation_basis`). + constant prices whose prices deflated it (`UnitConversion.deflation_basis`). It states + the latest-year label where the request asked for the latest year (`_asks_latest`). """ resolved = spec.resolved plan = heading_from_parts( @@ -283,7 +294,7 @@ def heading_for( resolved.entities, resolved.shape, resolved.coverage.window, - latest=spec.definition.identity.requested_window.kind == "latest", + latest=_asks_latest(spec), facts=resolved.sentence_facts, served=resolved.stat_vars.data_date_range, resolved_sources=resolved.trace.resolved_sources, diff --git a/src/search/scope/composite.py b/src/search/scope/composite.py index 1fc9dc401..ef79ca7da 100644 --- a/src/search/scope/composite.py +++ b/src/search/scope/composite.py @@ -1,5 +1,5 @@ -"""Composite answers in Search: the years a composite's components cover, and an answer -restated as the composite it serves. +"""Composite answers in Search: the years a composite's components cover, and the Spec minted +as the composite it serves. A composite identity binds a declared composite provenance (`code_index.composites`): the provider-inclusive total of a donor's bilateral flows and its imputed multilateral share @@ -10,13 +10,14 @@ component holds under the part's places (`component_years`), so the window a scope option offers is the window the mint serves. -`assemble_composite` restates a Search answer resolved for the composite's table identity -(`code_index.composites.table_identity`) as the composite: its identity and spec_id, its -coverage over the composite's years, its provenance facet naming the composite, and one -`Component` per component with the years it serves, and the facts of the file a mint of it -writes (`composite_download`). The table identity's observation preview reads one table, so the -restated answer carries none. The engine reads the composite's preview per component and -composes it here (`compose_observations`, `search.engine.region.composite_answer`). +`CompositeMint` mints a Search answer's Spec as the composite (`spec_build.build_spec(composite=)`) +over the table identity's arguments (`code_index.composites.table_identity`): its identity and +spec_id, its coverage over the composite's years, its headline, its open role axes, its StatVar +counts, its provenance facet naming the composite, its download facts (`composite_download`), and +one `Component` per component with the years it serves. The table identity's observation preview +reads one table, so the minted Spec carries none. The engine reads the composite's preview per +component and composes it (`compose_observations`, `search.engine.region.composite_answer`). One +method derives each field. """ from __future__ import annotations @@ -24,7 +25,9 @@ import json import logging from collections.abc import Iterable, Iterator, Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass +from functools import cached_property from typing import cast from code_index.composites import ( @@ -41,6 +44,7 @@ from dfo_platform.roles import role_key_axis from materialise.composite import spans, stacked_years, table_years from materialise.enrichment import party_count, party_kind_of +from origin_catalogue import Catalogue, bound_catalogue from search.engine.retrieval.bound_values import served_members from search.models import ( BaseIdentity, @@ -50,6 +54,7 @@ DateRange, DownloadFacts, DownloadLicence, + EntityRoleKey, Headline, HeadlineYear, ObservationSeries, @@ -58,14 +63,16 @@ ProvenanceRef, ResolvedEntity, ResolvedFacet, - ResolvedSpec, + ResolvedScope, ResolvedStatVars, RoleAxis, + RoleAxisFact, + Slot, SlotValue, Spec, - SpecDefinition, SpecObservations, TimeWindow, + Warning, omitted_observations, ) from search.observations import composite_facet_ref, provenance_refs @@ -406,161 +413,241 @@ def _dispatched(read: SpecObservations) -> int: return read.omitted.dispatched or 0 -def assemble_composite( - spec: Spec, - identity: BaseIdentity, - spec_id: str, - *, - served: Sequence[ServedComponent], - facts: ScopeFacts, -) -> tuple[Spec, list[Component]]: - """`spec`, the answer resolved for composite `identity`'s table identity - (`code_index.composites.table_identity`), restated as the composite under `spec_id`, and - the composite's components (`CompiledScope.components`). - - `served` holds `identity`'s components with the years each serves (`served_components`), - which the composite's preview filters by too. The restated answer's definition is - `identity`; its coverage counts every component's series in the years each serves, over - the window those years span; its - headline counts the latest of those years over every component serving it - (`_headline`); its StatVar counts are the components' summed, and it states whether a - groupby sum adds the components (`_stat_vars`); its - provenance facet names the composite, with the composite's label, the source and Data - Commons its component tables share (`search.observations.composite_facet_ref`) and the - account they give; its sources are its component tables, whose labels its - `published_tables` joins, and its measure label reads the declaration's `measure_noun` - (`_measure_label`), so its sentence cites both tables and names the composite's measure; - its open role axes (`ResolvedSpec.scope`, `role_axis_facts`) list and count the parties - of every component's series in the years it serves (`_open_axes`), as the mint's frame - holds them, where the table identity's answer read the first component's table alone; - its concept label stays the table identity's answer's, which names no basis; its download - facts are the composite file's (`composite_download`); and it carries no observation - preview, since the table identity's reads one table. The engine reads the composite's - preview per component (`search.engine.region.composite_answer`). - Each component names its role, the table it reads, the years it serves, how many - StatVars it serves and its own identity. Raises - `code_index.composites.CompositeNotServable` as `component_identities` does.""" - document = identity.model_dump(mode="json", exclude_none=True) - index = facts.index - declaration = _declaration(index, document) - years = sorted({year for item in served for year in item.served}) - observations = sum(item.years.counts.get(year, 0) for item in served for year in item.served) - window = TimeWindow(start_year=years[0], end_year=years[-1]) if years else None - sources = _component_sources(index, declaration) - provenance = composite_facet_ref(declaration) - facets = { - **spec.resolved.facets, - "provenance": ResolvedFacet( - basis="bound", - ref=provenance.ref, - label=provenance.label, - source=provenance.source, - data_commons=provenance.data_commons, - account=provenance_account(index, declaration.provenance), - ), - } - resolved = spec.resolved.model_copy( - update={ - "coverage": CoverageExact( - has_data=observations > 0, observation_count=observations, window=window - ), - "facets": facets, - "observations": None, - "headline": _headline(served, spec.resolved.headline, facts=facts), - **_open_axes(spec.resolved, served, declaration, facts=facts), - "stat_vars": _stat_vars(served, spec.resolved.stat_vars, declaration, facts=facts), - "slots": [ - slot.model_copy(update={"binding": _window_binding(window)}) - if slot.key.axis == "when" and window is not None - else slot - for slot in spec.resolved.slots - ], - "download": composite_download(served, identity, facts=facts), - "measure_label": _measure_label(declaration, spec.resolved.measure_label), - "published_tables": join_labels([source.label for source in sources]), - "trace": spec.resolved.trace.model_copy(update={"resolved_sources": sources}), - } - ).stating_data_years((years[0], years[-1]) if years else None) - restated = spec.model_copy( - update={ - "spec_id": spec_id, - "definition": SpecDefinition(spec_id=spec_id, identity=identity), - "resolved": resolved, - } - ) - return restated, _components(served) +@dataclass(frozen=True, eq=False) +class CompositeMint: + """What `spec_build.build_spec` mints a Spec as the composite `identity` from: its + identity and `spec_id`, its components with the years each serves + (`served_components`), the part's `facts`, and the request's `catalogue`, which its + provenance facet and download licence read (`ComponentReads.catalogue`), else the bound + one. + + Each method returns the value of one field the composite states in place of the table + identity's answer (`code_index.composites.table_identity`), from that answer's own value + where the field derives from it. Every other field, the slots and entities among them, is + the table identity's: the composite reads the same StatVars and places.""" + + identity: BaseIdentity + spec_id: str + served: tuple[ServedComponent, ...] + facts: ScopeFacts + catalogue: Catalogue | None = None + + @cached_property + def declaration(self) -> CompositeDeclaration: + """The composite `identity`'s provenance facet binds.""" + return _declaration( + self.facts.index, self.identity.model_dump(mode="json", exclude_none=True) + ) + @cached_property + def years(self) -> list[int]: + """The years some component serves, in order.""" + return sorted({year for item in self.served for year in item.served}) + + @cached_property + def window(self) -> TimeWindow | None: + """The window the served years span, None where no component serves a year.""" + years = self.years + return TimeWindow(start_year=years[0], end_year=years[-1]) if years else None + + def coverage(self) -> CoverageExact: + """Every component's series counted in the years it serves, over `window`.""" + observations = sum( + item.years.counts.get(year, 0) for item in self.served for year in item.served + ) + return CoverageExact( + has_data=observations > 0, observation_count=observations, window=self.window + ) -def _open_axes( - stated: ResolvedSpec, - served: Sequence[ServedComponent], - declaration: CompositeDeclaration, - *, - facts: ScopeFacts, -) -> dict[str, object]: - """`stated`'s open role axes, the table identity's answer's, restated over every - component of the composite `declaration`: its scope lists, and each role axis fact - counts (`enrichment.party_count`, in the components' datasets), the parties on the - axis's coverage role of the series every component writes in the years it serves, each - read under its own places (`_served_parties`). A stacked composite's imputed line can - hold recipients its bilateral line lacks, which the table identity's answer, reading - the bilateral table alone, leaves out. A scope member the answer listed keeps its label - and type, and one only another component holds is labelled from the code index. Empty - where the answer leaves no axis open, or where the snapshot cannot list the parties, - which is logged at WARNING with the components it cannot list, since the answer then - counts the table identity's parties alone.""" - if stated.scope is None and not stated.role_axis_facts: - return {} - coverage = facts.coverage.index - unlisted = [item for item in served if item.served and not _lists_parties(item, coverage)] - if unlisted: - logger.warning( - "composite %s keeps the open axes of its table read: the snapshot cannot list " - "the parties of %s", - declaration.provenance, - ", ".join(item.component.recipe.dataset for item in unlisted), + def headline(self, stated: Headline | None) -> Headline | None: + """The composite's headline (`_headline`), naming `stated`'s window.""" + return _headline(self.served, stated, facts=self.facts) + + def parties(self, axis: RoleAxis) -> list[str] | None: + """The parties on `axis` of the series every component writes in the years it + serves, each read under its own places (`_served_parties`), without the totals + `party_count` leaves out (`_parties`). These are the members of the scope `open_axes` + states, and `member_count` counts the same parties. None where the snapshot cannot list some + serving component's parties (`_lists_parties`).""" + held = self._held_parties + if held is None: + return None + return _parties( + held[_ROLES[axis]], axis=axis, index=self.facts.index, datasets=self._datasets ) - return {} - held = _served_parties(served, coverage) - index = facts.index - datasets = [recipe.dataset for recipe in declaration.components] - restated: dict[str, object] = {} - if stated.role_axis_facts: - restated["role_axis_facts"] = [ - fact.model_copy( - update={ - "member_count": party_count( - held[_ROLES[fact.role]], - axis=fact.role, - parties=index.parties, - datasets=datasets, - ) - } + + def member_count(self, axis: RoleAxis) -> int | None: + """How many parties `parties(axis)` lists, as `enrichment.party_count` counts them + in the components' datasets. It is the count of each `RoleAxisFact` `open_axes` states + and of the `role_scope_defaulted` warning (`role_scope_warnings`). None where they + cannot be listed.""" + held = self._held_parties + if held is None: + return None + return party_count( + held[_ROLES[axis]], axis=axis, parties=self.facts.index.parties, datasets=self._datasets + ) + + @cached_property + def _datasets(self) -> list[str]: + return [recipe.dataset for recipe in self.declaration.components] + + @cached_property + def _held_parties(self) -> dict[Role, frozenset[str]] | None: + coverage = self.facts.coverage.index + unlisted = [ + item for item in self.served if item.served and not _lists_parties(item, coverage) + ] + if unlisted: + logger.warning( + "composite %s keeps the open axes of its table read: the snapshot cannot list " + "the parties of %s", + self.declaration.provenance, + ", ".join(item.component.recipe.dataset for item in unlisted), ) + return None + return _served_parties(self.served, coverage) + + def open_axes( + self, scope: ResolvedScope | None, facts: list[RoleAxisFact] | None + ) -> tuple[ResolvedScope | None, list[RoleAxisFact] | None]: + """`scope` and `facts`, the table identity's answer's open role axes, over every + component: the scope lists, and each counted fact counts, `parties` of its axis. A + stacked composite's imputed line can hold recipients its bilateral line lacks, which + the table identity's answer, reading the bilateral table alone, leaves out. A + member the answer listed keeps its label and type, and one only another component + holds is labelled from the code index. Both stand where the answer leaves no axis + open, and where the snapshot cannot list the parties, which `parties` logs, since + the answer then counts the table identity's parties alone.""" + if (scope is None and not facts) or self._held_parties is None: + return scope, facts + counted = facts and [ + fact.model_copy(update={"member_count": self.member_count(fact.role)}) if fact.member_count is not None else fact - for fact in stated.role_axis_facts + for fact in facts ] - scope = stated.scope - if scope is not None and scope.axis != "observationAbout": - role = _ROLES[scope.axis] + if scope is None or scope.axis == "observationAbout": + return scope, counted listed = {member.ref: member for member in scope.members} - restated["scope"] = scope.model_copy( - update={ - "members": [ - listed.get(dcid) - or ResolvedEntity( - ref=dcid, - label=row.label if (row := index.parties.get(dcid)) else None, - role=scope.axis, - ) - for dcid in _parties( - held[role], axis=scope.axis, index=index, datasets=datasets - ) - ] - } + parties = self.facts.index.parties + members = [ + listed.get(dcid) + or ResolvedEntity( + ref=dcid, label=row.label if (row := parties.get(dcid)) else None, role=scope.axis + ) + for dcid in self.parties(scope.axis) or () + ] + return scope.model_copy(update={"members": members}), counted + + def stat_vars(self, stated: ResolvedStatVars) -> ResolvedStatVars: + """`stated`, the table identity's StatVar counts, over every component (`_stat_vars`).""" + return _stat_vars(self.served, stated, self.declaration, facts=self.facts) + + def slots(self, stated: Sequence[Slot]) -> list[Slot]: + """`stated` with the when slot bound to `window` (`_window_binding`), where some + component serves a year.""" + window = self.window + return [ + slot.model_copy(update={"binding": _window_binding(window)}) + if slot.key.axis == "when" and window is not None + else slot + for slot in stated + ] + + def download(self) -> DownloadFacts | None: + """The facts of the composite file a Datasets mint writes (`composite_download`).""" + with self._reading(): + return composite_download(self.served, self.identity, facts=self.facts) + + def measure_label(self, stated: str | None) -> str | None: + """The measure label the composite's sentence names (`_measure_label`).""" + return _measure_label(self.declaration, stated) + + def sources(self) -> list[ProvenanceRef]: + """The component tables, each under its declared label (`_component_sources`).""" + return _component_sources(self.facts.index, self.declaration) + + def published_tables(self) -> str | None: + """The component tables' labels, joined.""" + return join_labels([source.label for source in self.sources()]) + + def provenance_facet(self) -> ResolvedFacet: + """The provenance facet naming the composite, with its label, the source and Data + Commons its component tables share (`search.observations.composite_facet_ref`), and + the account they give.""" + with self._reading(): + ref = composite_facet_ref(self.declaration, index=self.facts.index) + return ResolvedFacet( + basis="bound", + ref=ref.ref, + label=ref.label, + source=ref.source, + data_commons=ref.data_commons, + account=provenance_account(self.facts.index, self.declaration.provenance), ) - return restated + + def facets(self, stated: Mapping[str, ResolvedFacet]) -> dict[str, ResolvedFacet]: + """`stated` with the provenance facet naming the composite.""" + return {**stated, "provenance": self.provenance_facet()} + + def data_years(self) -> tuple[int, int] | None: + """The first and last year some component serves (`ResolvedSpec.data_years`).""" + years = self.years + return (years[0], years[-1]) if years else None + + def components(self) -> list[Component]: + """The composite's components (`CompiledScope.components`).""" + return _components(self.served) + + def role_scope_warnings(self, warnings: Sequence[Warning], spec: Spec) -> tuple[Warning, ...]: + """`warnings`, with each `role_scope_defaulted` warning about `spec`, the Spec minted + from this composite, worded to count the `parties` of its `params["axis"]`, as a + fresh answer's warning states its count (`grounding.role_scope_defaulted_message`). + A warning on an axis whose parties cannot be listed states no count. A warning is + about `spec` where it is keyed to `spec` or to no Spec. One keyed to another reading, + such as a flagged match or a suggestion beside the answer, counts that reading's + parties and keeps its words. An axis on which `spec` has no counted role axis fact + keeps its warning, as does every other warning.""" + # Deferred: grounding imports this module while it loads. + from search.engine.region.codes import ROLE_SCOPE_DEFAULTED + from search.engine.region.grounding import role_scope_defaulted_message + + counted = { + fact.role + for fact in spec.resolved.role_axis_facts or () + if fact.member_count is not None + } + labels = {entity.ref: entity.label or entity.ref for entity in spec.resolved.entities} + + def stated(warning: Warning) -> Warning: + axis = (warning.params or {}).get("axis") + if ( + warning.code != ROLE_SCOPE_DEFAULTED + or warning.spec_id not in (None, spec.spec_id) + or axis not in counted + ): + return warning + role = cast("RoleAxis", axis) + opposite: EntityRoleKey = "origin" if role == "destination" else "destination" + message = role_scope_defaulted_message( + role, + member_count=self.member_count(role), + bound_dcids=list(dict.fromkeys(self.identity.entities.get(opposite, ()))), + label_of=lambda dcids: [labels.get(dcid, dcid) for dcid in dcids], + ) + return warning.model_copy(update={"message": message}) + + return tuple(map(stated, warnings)) + + @contextmanager + def _reading(self) -> Iterator[None]: + """Binds `catalogue`, where one is given, for the body of the `with` block.""" + if self.catalogue is None: + yield + return + with bound_catalogue(self.catalogue): + yield def _parties( @@ -677,7 +764,7 @@ def _stat_vars( *, facts: ScopeFacts, ) -> ResolvedStatVars: - """`stated`, the table identity's StatVar counts, restated over every component of the + """`stated`, the table identity's StatVar counts, counted over every component of the composite `declaration`. The member count is the StatVars holding a series under their component's own places (`_bound`) in the years it serves, the served count sums the StatVars a mint of each component serves (None where the index does not settle some diff --git a/src/search/scope/provider.py b/src/search/scope/provider.py index 1fbd5eedb..d628b4633 100644 --- a/src/search/scope/provider.py +++ b/src/search/scope/provider.py @@ -51,7 +51,7 @@ from collections.abc import Collection, Mapping, Sequence from dataclasses import dataclass from datetime import date -from typing import cast, get_args +from typing import Literal, cast, get_args from code_index.composites import ( CompositeNotServable, @@ -163,11 +163,36 @@ _DONOR_ORGANISATIONS = frozenset(DONOR_ORGANISATIONS) _NET = "net" _CONSTANT = "RealValue" -_SLOT_WHY: Mapping[str, str] = { - "crsCooperationModality": "modality", - "crsTypeOfFinance": "finance_type", -} -"""The `why` of `provider_not_offered` for each slot the imputed table's rows do not declare.""" +_SLOT_NOT_SERVED = "slot_not_served" +"""The `why` of `provider_not_offered` where a provider composite cannot serve a slot the part +binds, whose property the detail's `slot` names (`_composite_fault`).""" + +WithheldWhy = Literal[ + "destination_unclassified", + "option_not_covered", + "read_window_not_covered", + "scope_undecided", + "slot_not_served", +] +"""Why a fresh reading the bilateral default reaches on a part with no provider axis reads total +ODA: the reasons dfo-spec-kit's `registries/warning-reasons.json` lists for +`bilateral_reading_unavailable`, and no others.""" +WITHHELD_WHYS: frozenset[str] = frozenset(get_args(WithheldWhy)) + + +@dataclass(frozen=True, slots=True) +class Withheld: + """Why the bilateral default reaches a reading and cannot apply (`WithheldWhy`), and for + `slot_not_served` the property of the slot the bilateral composite cannot serve.""" + + why: WithheldWhy + slot: str | None = None + + @property + def params(self) -> dict[str, str]: + """The `bilateral_reading_unavailable` warning's `params`: `why`, and `slot` where the + reason names one.""" + return {"why": self.why} if self.slot is None else {"why": self.why, "slot": self.slot} @dataclass(frozen=True, slots=True) @@ -489,8 +514,9 @@ def _bilateral_option(identity: BaseIdentity, rule: DestinationRule, facts: Scop destination, whose flows are core contributions the composite leaves out, and not offered (`provider_not_offered`) where the composite's population is unavailable (why `destination_unclassified`, `_population_unavailable`) or the composite refuses what - `identity` binds (`code_index.composites.CompositeNotServable`; why the slot at fault, as - `_inclusive_option` names it). Otherwise it serves the years its one component serves + `identity` binds (`code_index.composites.CompositeNotServable`, `_composite_fault`: why + `slot_not_served` naming the slot at fault, or `option_not_covered` where the window reaches + none of the composite's years). Otherwise it serves the years its one component serves (`_composite_years`), judged as a sibling total is (`_sibling_option`, `_latest_sibling_option`, each `option_not_covered`): available when they cover every year the part's answer holds, partial over their latest run when they cover fewer, and @@ -523,14 +549,9 @@ def _read_bilateral_option( try: years = _composite_years(identity, rule.read(identity), source.composite, facts) except CompositeNotServable as exc: - why = ( - "destination_unclassified" - if isinstance(exc, PopulationUnavailable) - else _SLOT_WHY.get(prop := _slot_at(identity, exc.pointer), prop) - ) return _option( "bilateral", - _unavailable("provider_not_offered", {"why": why}), + _composite_fault(identity, exc, uncovered="option_not_covered"), binding=binding, label=source.label, ) @@ -701,14 +722,14 @@ def _default_rule(identity: BaseIdentity, facts: ScopeFacts) -> DestinationRule def bilateral_default_withheld( identity: BaseIdentity, selection: ScopeState, facts: ScopeFacts -) -> str | None: +) -> Withheld | None: """Why a fresh reading of `identity` that the bilateral default reaches (`with_bilateral_default`) reads total ODA, where no provider axis states it (`provider_rule`) and no stated or carried provider entry asks for total ODA: the - unavailable bilateral option's `why` (`_bilateral_option`), as `destination_unclassified` - for an unavailable population or the slot at fault where the composite refuses what - `identity` binds, else its reason code, `option_not_covered` where it does not cover - every year the answer holds. A part with a provider axis states the reason on its + reason its unavailable bilateral option states (`_bilateral_option`, `_withheld`), + `destination_unclassified` for an unavailable population, `slot_not_served` naming the + slot where the composite cannot serve one `identity` binds, and `option_not_covered` where + it does not cover every year the answer holds. A part with a provider axis states the reason on its bilateral option instead. None where the default applies or does not reach `identity`.""" defaulted = _default_bilateral_option(identity, facts) if defaulted is None: @@ -721,15 +742,34 @@ def bilateral_default_withheld( return None if provider_rule(identity, None, facts) is not None: return None - return availability.detail.get("why") or availability.code + return _withheld(availability) + + +def _withheld(availability: OptionAvailability) -> Withheld: + """The reason an unavailable or partial bilateral option states (`_read_bilateral_option`): + `destination_unclassified` or `slot_not_served`, with its slot, from a + `provider_not_offered` detail, and `option_not_covered` from that code. Raises + `ValueError` for any other: the option is unavailable with `multilateral_destination` + only on a part naming a multilateral destination, which the default does not reach.""" + detail = availability.detail + match availability.code, detail.get("why"): + case "provider_not_offered", "destination_unclassified": + return Withheld(why="destination_unclassified") + case "provider_not_offered", "slot_not_served": + return Withheld(why="slot_not_served", slot=detail["slot"]) + case "option_not_covered", None: + return Withheld(why="option_not_covered") + raise ValueError(f"the bilateral option states no withheld reason: {availability!r}") def bilateral_default_reading( identity: BaseIdentity, selection: ScopeState, facts: ScopeFacts ) -> tuple[BaseIdentity, ScopeState]: """The identity a fresh reading of `identity` is served as, and the selection its scope - reads: one rule for a fresh search's definite answer (`scope_decision.decide_scope`) and - every other reading it shows (`composite_answer.with_bilateral_readings`). + reads: the one rule the reading elected before a part's read (`reading.elect_reading`) + and the reading elected at the read of each candidate, refusal alternate, flagged match + and nearest-real suggestion (`reading.elect_at_read`) both apply, through + `scope_decision.choose_provider`. DAC2a's total ODA reads its bilateral destinations (`with_bilateral_default`) unless the part's first provider entry, explicit before carried (`ScopeState.entries_on`), asks for @@ -1097,7 +1137,8 @@ def _inclusive_option( """The inclusive option binding `source`'s composite over `bilateral`, the part's identity without it. It is not offered for what `_not_offered` and `_providers_not_offered` refuse, or where the imputed line cannot serve a slot - `bilateral` binds, even when the part holds it. For a part asking for the latest year it + `bilateral` binds (why `slot_not_served`, `_composite_fault`), even when the part holds + it. For a part asking for the latest year it is available over the year `inclusive_reach` elects, which its window states (`elect_latest_window`). Otherwise it is available when both lines cover every year the bilateral answer holds; partial over the latest run of years they cover, with that run as @@ -1108,8 +1149,7 @@ def _inclusive_option( try: why = _not_offered(bilateral, source) or _providers_not_offered(bilateral, source, facts) except CompositeNotServable as exc: - prop = _slot_at(bilateral, exc.pointer) - why = _SLOT_WHY.get(prop, prop) + return _option("inclusive", _composite_fault(bilateral, exc, uncovered="imputed_coverage")) if why is not None: return _option("inclusive", _unavailable("provider_not_offered", {"why": why})) measure = held_measure(bilateral) @@ -1248,12 +1288,35 @@ def _providers_not_offered( return None if carried and not _pools_redistributed(providers) else "pooled_providers" -def _slot_at(identity: BaseIdentity, pointer: str) -> str: - """The slot property at `pointer` (`/slots/`) of `identity`, or the pointer itself.""" +def _composite_fault( + identity: BaseIdentity, exc: CompositeNotServable, *, uncovered: ScopeReasonCode +) -> OptionAvailability: + """The availability of a provider option whose composite refuses what `identity` binds + (`exc`): not offered (`provider_not_offered`) with why `destination_unclassified` for an + unavailable population (`PopulationUnavailable`) and why `slot_not_served`, its `slot` + the property at fault, for a slot the composite's components do not declare; unavailable + with `uncovered`, the option's code for a composite serving no year, where the window + reaches none of the composite's component periods. Re-raises `exc` at any other member + (`/shape`, `/entities/`, a provenance set or a composite the index does not serve), + which no part reaches here: the options bind the composite as a value over a described + identity whose named destinations are on the composite's side.""" + if isinstance(exc, PopulationUnavailable): + return _unavailable("provider_not_offered", {"why": "destination_unclassified"}) + if exc.pointer == "/requested_window": + return _unavailable(uncovered) + slot = _slot_at(identity, exc.pointer) + if slot is None: + raise exc + return _unavailable("provider_not_offered", {"why": _SLOT_NOT_SERVED, "slot": slot}) + + +def _slot_at(identity: BaseIdentity, pointer: str) -> str | None: + """The slot property at `pointer` (`/slots/`) of `identity`, None at any other + member.""" head, _, position = pointer.rpartition("/") if head == "/slots" and position.isdigit() and int(position) < len(identity.slots): return identity.slots[int(position)].property - return pointer + return None def provider_convention( diff --git a/tests/code_index/test_destination_population.py b/tests/code_index/test_destination_population.py index 715d9f593..2a7735b75 100644 --- a/tests/code_index/test_destination_population.py +++ b/tests/code_index/test_destination_population.py @@ -6,6 +6,7 @@ from __future__ import annotations +import gc import gzip import json import weakref @@ -317,3 +318,16 @@ def recording(obj, *args, **kwargs): assert population_check(index, coverages[0]) is first assert sum(obj is index for obj in finalized) == on_index assert [obj for obj in finalized if obj is not index] == coverages + + +def test_dropping_a_code_index_frees_its_check_memo_while_a_coverage_index_lives(): + """The finalizer on a coverage index must not hold the code index's memo of checks.""" + index = _parse() + coverage = _coverage([_KENYA]) + population_check(index, coverage) + memo = weakref.ref(composites_module._checks(index)) + + del index + gc.collect() + + assert memo() is None diff --git a/tests/engine/_assemble_fixtures.py b/tests/engine/_assemble_fixtures.py index 7eea51353..49f84ff49 100644 --- a/tests/engine/_assemble_fixtures.py +++ b/tests/engine/_assemble_fixtures.py @@ -152,6 +152,7 @@ def _make_spec( ] return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape_model, @@ -223,6 +224,7 @@ def _make_promoting_spec(purpose_dcid: str | None, place_dcid: str): ] return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape_model, diff --git a/tests/engine/_builders.py b/tests/engine/_builders.py index 2c5d35dec..46ccc5c13 100644 --- a/tests/engine/_builders.py +++ b/tests/engine/_builders.py @@ -2,12 +2,17 @@ from __future__ import annotations +import asyncio from collections.abc import Mapping, Sequence from typing import Any +import pytest + from code_index.shared import get_index from search.engine.bind import SlotBindingDraft +from search.engine.region import related_read as related_read_module from search.engine.region.candidates import _Candidate, _materialise_alt_shapes, elect_alternate +from search.engine.region.grounding import requested_window_from_date_request from search.engine.region.place_context import PlaceContext from search.engine.region.place_roles import ( PlaceAssignment, @@ -16,6 +21,7 @@ settled_roles, ) from search.engine.region.place_set import PlaceNode, PlaceSet +from search.engine.region.related_read import RelatedRead from search.engine.retrieval.bound_values import ( MeasurePartition, RoleClassWalks, @@ -24,6 +30,7 @@ from search.engine.retrieval.retrieve import UnsupportedDraft from search.engine.shape import ShapeDraft, entities_by_role from search.engine.spec_build import build_spec +from search.heading.departure_fact import DepartureFact from search.models import ( Axis, BindingSet, @@ -37,7 +44,9 @@ SlotKey, SlotValue, StatVar, + Warning, ) +from search.scope.state import ScopeState def gref(dcid: str, label: str | None = None) -> GraphRef: @@ -165,6 +174,7 @@ def conjoin_spec( """ slot = constraint_value_slot("constraintProp", constraint_val) return build_spec( + composite=None, open_roles=(), shape=shape, slots=[slot], @@ -330,5 +340,43 @@ def materialise_alternates( places=[place for _, place in kept], graph=graph, **kwargs, + selection=ScopeState(), + operations=(), ) return read.candidates, read.blocking + + +def slow_related_reads(monkeypatch: pytest.MonkeyPatch) -> list[RelatedRead]: + """Every related-measures read the pass starts from here on, each a task that settles + after 0.2 s, so a read nothing cancels ends uncancelled. The answers here qualify for + none of their own.""" + started: list[RelatedRead] = [] + + def slow(state: Any, *, stat_vars: Any, date_request: Any, **_kwargs: Any) -> RelatedRead: + read = RelatedRead( + stat_var=stat_vars[0], + window=requested_window_from_date_request(date_request), + task=asyncio.create_task(asyncio.sleep(0.2, result=None)), + ) + state.related_reads.append(read) + started.append(read) + return read + + monkeypatch.setattr(related_read_module, "start_related_read", slow) + return started + + +def raise_answer_warnings(state: Any) -> None: + """Raise on `state` what a definite answer raises as it is assembled: a + `currency_unmapped_omitted` warning, an option coverage warning, each with a departure + fact, and a rewording of each `role_scope_defaulted` warning already raised, as a Spec + minted as a composite counts its defaulted scope (`CompositeMint.role_scope_warnings`).""" + for code in ("currency_unmapped_omitted", "option_coverage_timed_out"): + warning = Warning(code=code, severity="warn", message=f"{code} raised by the answer.") + state.depart(warning, DepartureFact(code=code, binding_ref=None, shown=code)) + state.warnings[:] = [ + warning.model_copy(update={"message": f"{warning.message} Reworded by the answer."}) + if warning.code == "role_scope_defaulted" + else warning + for warning in state.warnings + ] diff --git a/tests/engine/test_arc_tables_serving.py b/tests/engine/test_arc_tables_serving.py index 4fa5733ff..b07ef3fc1 100644 --- a/tests/engine/test_arc_tables_serving.py +++ b/tests/engine/test_arc_tables_serving.py @@ -111,6 +111,7 @@ ) from search.engine.shape import ShapeDraft from search.models import GraphRef +from search.scope.state import ScopeState from statvar_members import MultiValuedConstraint from tests.engine._binders import elected_reply from tests.engine._builders import subject_place_roles @@ -726,7 +727,7 @@ def _flagged_member_constraint_properties(shape: ShapeDraft) -> list[list[str] | coverage_judged=True, ) with held_family_memo(): - spec = asyncio.run(_elected_flagged_match(state, members=[_IMF_GG])) + spec = asyncio.run(_elected_flagged_match(state, members=[_IMF_GG])).spec return [member.constraint_properties for member in spec.resolved.members] @@ -1514,6 +1515,8 @@ def read_default_presence(**_kwargs: object) -> ConsultedPresence: sdmx=FakeSdmxClient(observations={}, availability={}), date_request=None, coverage=empty_coverage_snapshot(), + selection=ScopeState(), + operations=(), ) state = SimpleNamespace( capped_shapes=[shape], @@ -1572,6 +1575,8 @@ def read_default_presence(**_kwargs: object) -> ConsultedPresence: sdmx=FakeSdmxClient(observations={}, availability={}), date_request=None, coverage=empty_coverage_snapshot(), + selection=ScopeState(), + operations=(), ) assert len(reads) == 1 @@ -1606,6 +1611,8 @@ def read_default_presence(*, stat_vars, known, **_kwargs: object) -> ConsultedPr sdmx=FakeSdmxClient(observations={}, availability={}), date_request=None, coverage=empty_coverage_snapshot(), + selection=ScopeState(), + operations=(), ) # The alternate's reading consulted its top-up's countries fallback into its own record, diff --git a/tests/engine/test_assemble_spec_params.py b/tests/engine/test_assemble_spec_params.py index a4d599a6a..218b26b77 100644 --- a/tests/engine/test_assemble_spec_params.py +++ b/tests/engine/test_assemble_spec_params.py @@ -38,6 +38,7 @@ def test_build_spec_facets_serialised_body_has_the_four_resolved_facet_shapes(): ), } result = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=spec.resolved.shape, @@ -76,6 +77,7 @@ def test_build_spec_facets_defaults_to_empty_when_not_supplied(): place_dcid="country/ETH", ) result = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=spec.resolved.shape, @@ -105,6 +107,7 @@ def test_build_spec_rejects_request_basis_on_a_non_unit_axis(): } with pytest.raises(ValidationError, match="request_basis is only meaningful"): build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=spec.resolved.shape, diff --git a/tests/engine/test_bound_property_assertion.py b/tests/engine/test_bound_property_assertion.py index 6d5f3537e..853f7c07a 100644 --- a/tests/engine/test_bound_property_assertion.py +++ b/tests/engine/test_bound_property_assertion.py @@ -90,6 +90,7 @@ def _resolve(members: dict[str, dict], bindings: list[SlotBindingDraft]): sdmx=FakeSdmxClient(observations={}), coverage=_coverage(list(members)).snapshot(), discover_scope=True, + within=None, ) diff --git a/tests/engine/test_candidate_collapse_rule_one.py b/tests/engine/test_candidate_collapse_rule_one.py index c30393356..376a80623 100644 --- a/tests/engine/test_candidate_collapse_rule_one.py +++ b/tests/engine/test_candidate_collapse_rule_one.py @@ -18,10 +18,12 @@ from search.engine.region.candidates import _Candidate from search.engine.region.convertibility import UnitCoverage from search.engine.region.place_context import PlaceContext +from search.engine.region.scope_decision import ScopeDecision from search.engine.region.state import ResolveState from search.engine.retrieval.retrieve import Materialised, NoDataDraft from search.engine.shape import ShapeDraft from search.models import CandidateBasis, CoverageBare, Spec +from search.scope.tables import ScopeFacts from tests.engine._assemble_fixtures import _make_spec, _standard_shape_draft from tests.engine.test_candidate_grounding_thread import _state @@ -49,6 +51,7 @@ def _entry( walked=None, date_request=None, basis=basis, + reading=None, ), is_elected=False, plan=None, @@ -67,6 +70,10 @@ def _entry( unit_refusals={}, unit_coverage=UnitCoverage(), unit_gate_reason=None, + decision=ScopeDecision( + spec_id=spec.spec_id, facts=cast(ScopeFacts, None), scope=None, composite=False + ), + minted=None, ) @@ -81,7 +88,7 @@ def _settle( """The specs `assemble_candidates` hands definite assembly for `grounded`.""" monkeypatch.setattr(candidate_collapse, "ground_candidates", lambda _state, _c: grounded) - async def _definite(_state, specs, _grounded, _rate_cache): + async def _definite(_state, specs, _grounded, _rate_cache, **_kwargs): return specs monkeypatch.setattr(candidate_collapse, "_assemble_definite", _definite) diff --git a/tests/engine/test_composite_answer.py b/tests/engine/test_composite_answer.py index 2a0ba151f..e529a2ac9 100644 --- a/tests/engine/test_composite_answer.py +++ b/tests/engine/test_composite_answer.py @@ -6,7 +6,6 @@ import asyncio import dataclasses -import threading from collections.abc import Sequence from types import SimpleNamespace from typing import Any, cast @@ -14,11 +13,9 @@ import pytest import dc_graph -from code_index.shared import get_index from origin_catalogue import current_catalogue, scoped_reads from search.engine import response_cache_flow, same_shape from search.engine.concurrency import scoped_concurrency -from search.engine.enrichment import _launch_trailing_tasks from search.engine.entry import dataset_resubmit from search.engine.errors import ( EngineInfraError, @@ -29,15 +26,19 @@ from search.engine.events import ( DoneEvent, Event, - FacetOptionsEvent, ObservationsEvent, ResponseEvent, ) -from search.engine.region import composite_answer, pipeline, place_options, result +from search.engine.region import ( + composite_answer, + phase_answer, + pipeline, + place_options, + resubmit, +) from search.engine.region.codes import ( MEMBERS_ABSENT_FROM_GRAPH, _ObsProbeResult, - _OptionProbeResult, ) from search.engine.region.composite_answer import ( ComponentReads, @@ -45,7 +46,6 @@ composite_served, read_composite_preview, ) -from search.engine.region.promote_relaunch import bind_relaunches, without_relaunches from search.engine.region.result import RegionResult from search.engine.region.state import EngineOptions, ResolveState from search.engine.response_cache import ResponseCache @@ -55,7 +55,6 @@ CoverageExact, DatasetResubmitInput, DefinitePart, - Entry, IdentityBindingSet, IdentityBindingValue, ObservationPoint, @@ -76,14 +75,11 @@ ) from search.scope import composite from search.scope.composite import ServedComponent, compose_observations -from search.scope.state import ScopeState -from search.scope.tables import ScopeFacts from series_coverage.shared import CoverageSnapshot from tests.engine._fake_coverage import FakeCoverageService, patch_coverage from tests.engine._fake_sdmx import FakeSdmxClient from tests.engine._harness import offline_events, offline_resolve from tests.scope import test_provider as provider -from tests.scope._axes import carried, explicit from tests.scope.test_basis import _graph, _PickingLLM _WINDOW = range(2015, 2021) @@ -539,27 +535,32 @@ async def run() -> _ObsProbeResult: def test_the_answer_s_composite_facts_read_the_catalogue_the_preview_reads(monkeypatch): """The part's source scope narrows the catalogue its binders read, here to a catalogue in which the bilateral table holds no 2020 series, and the composite they settle is - restated under the request's catalogue, as its preview reads: the component years the - answer states are the years the preview filters by.""" + minted under the request's catalogue (`resubmit.composite_mint`), as its preview reads: + the component years the answer states are the years the preview filters by.""" first, _ = _components(monkeypatch) kept: list[tuple[RegionResult, ResolveState]] = [] - real = pipeline._with_scope_choices + preview = pipeline._with_composite_preview - def capture(result: RegionResult, state: ResolveState) -> RegionResult: + async def capture(result: RegionResult, state: ResolveState) -> RegionResult: kept.append((result, state)) - return real(result, state) + return await preview(result, state) - monkeypatch.setattr(pipeline, "_with_scope_choices", capture) - previewed = _dates(_observed(_stream(_fresh())).observations) - (result, state), *_ = kept + monkeypatch.setattr(pipeline, "_with_composite_preview", capture) scopes: list[frozenset[str] | None] = [] - assemble = composite_answer.assemble_composite + mint = resubmit.composite_mint - def record(*args: Any, **kwargs: Any) -> Any: + def recording(identity: BaseIdentity, *, coverage: Any) -> Any: scopes.append(current_catalogue().scope) - return assemble(*args, **kwargs) + return mint(identity, coverage=coverage) - monkeypatch.setattr(composite_answer, "assemble_composite", record) + monkeypatch.setattr(resubmit, "composite_mint", recording) + decide = phase_answer._decide_answer_scope + + def narrowed_decide(*args: Any, **kwargs: Any) -> Any: + with scoped_reads(frozenset({provider._CRS, provider._IMPUTED})): + return decide(*args, **kwargs) + + monkeypatch.setattr(phase_answer, "_decide_answer_scope", narrowed_decide) held = composite._component_years def narrowed(identity: Any, *, facts: Any) -> composite.YearCounts: @@ -573,61 +574,48 @@ def narrowed(identity: Any, *, facts: Any) -> composite.YearCounts: monkeypatch.setattr(composite, "_component_years", narrowed) - with scoped_reads(frozenset({provider._CRS, provider._IMPUTED})): - restated = real(result, state) + previewed = _dates(_observed(_stream(_fresh())).observations) + (stated, state), *_ = kept - assert restated.scope_choices is not None - assert restated.scope_choices.compiled.spec_id == first.spec_id + assert stated.scope_choices is not None + assert stated.scope_choices.compiled.spec_id == first.spec_id assert state.request_catalogue is not None assert scopes == [state.request_catalogue.scope] - stated = { + components = { _STAT_VARS[_provenance(component.identity)]: sorted( str(year) for start, end in component.periods or () for year in range(start, end + 1) ) - for component in restated.scope_choices.compiled.components or () + for component in stated.scope_choices.compiled.components or () } - assert stated == previewed - relaunch = restated.observations_relaunch - assert isinstance(relaunch, CompositeRelaunch) - assert relaunch.served is not None - assert { - _STAT_VARS[_provenance(BaseIdentity.model_validate(item.component.identity))]: sorted( - str(year) for year in item.served - ) - for item in relaunch.served - } == stated - - -class _RecordingTask: - """Stands in for a trailing task, recording the thread each `cancel` runs on.""" - - def __init__(self) -> None: - self.cancelled_on: list[int] = [] - - def cancel(self) -> bool: - self.cancelled_on.append(threading.get_ident()) - return True + assert components == previewed -def test_a_fresh_composite_answer_cancels_the_tasks_it_drops_on_the_event_loop(monkeypatch): - """The restatement as the composite runs in a worker thread (`_with_scope_choices`) and - drops the trailing tasks the table answer held, which `Task.cancel` may only cancel on - the event loop they belong to, so the loop cancels them once the thread returns.""" - _components(monkeypatch) - held = _RecordingTask() - loop_threads: list[int] = [] - stating = pipeline._stating_unpublished_wholes +def test_a_fresh_composite_answer_launches_none_of_its_table_s_trailing_tasks(monkeypatch): + """A fresh answer minted as the composite reaches `_with_scope_choices` holding no slot + options, place options or preview task of the table it read, nor a closure relaunching + them, so none streams under the composite's spec_id. Its preview reads each component + (`composite_answer.CompositeRelaunch`).""" + first, _ = _components(monkeypatch) + arrived: list[RegionResult] = [] + real = pipeline._with_scope_choices - def holding(result: RegionResult, state: ResolveState) -> RegionResult: - loop_threads.append(threading.get_ident()) - return dataclasses.replace(stating(result, state), slot_options_task=cast(Any, held)) + def capture(result: RegionResult) -> RegionResult: + arrived.append(result) + return real(result) - monkeypatch.setattr(pipeline, "_stating_unpublished_wholes", holding) + monkeypatch.setattr(pipeline, "_with_scope_choices", capture) _observed(_stream(_fresh())) - assert loop_threads - assert held.cancelled_on == loop_threads[:1] + (region, *_) = arrived + assert region.spec.spec_id == first.spec_id + assert (region.slot_options_task, region.place_options_task, region.observations_task) == ( + None, + None, + None, + ) + assert (region.slot_options_relaunch, region.place_options_relaunch) == (None, None) + assert isinstance(region.observations_relaunch, CompositeRelaunch) def test_a_component_serving_no_year_is_not_read_and_warns_nothing(monkeypatch): @@ -1006,257 +994,12 @@ async def run() -> _ObsProbeResult: assert _dates(observations) == _BOTH -def _dac2a_and_crs_snapshot(monkeypatch: pytest.MonkeyPatch) -> CoverageSnapshot: - """A snapshot of the CRS rows `provider._coverage_rows` patches in and DAC2a's rows from - the United States to Kenya in every year.""" - rows, extra = provider._coverage_rows(provider._ALL_YEARS) - dac2a_rows = [ - (dcid, "dac2a", provider._USA, provider._KEN, list(provider._ALL_YEARS)) - for dcid in provider.basis._DAC2A_MEMBERS - ] - snapshot = FakeCoverageService( - patch_coverage( - monkeypatch, - [*rows, *dac2a_rows], - role_names={ - **extra["role_names"], - **dict.fromkeys(provider.basis._DAC2A_MEMBERS, provider._FLOW), - }, - facet_metadata={ - **extra["facet_metadata"], - "dac2a": provider.FacetMetadata(provenance_id=provider._DAC2A, unit="USD"), - }, - ) - ).snapshot() - return snapshot - - -def test_a_candidate_set_restating_a_dac2a_reading_keeps_its_other_readings_options( - monkeypatch: pytest.MonkeyPatch, -): - """A candidate set of a DAC2a reading and a CRS reading restates the DAC2a one as its - bilateral destinations. Its trailing slot options keep the CRS reading's events and - withhold the DAC2a reading's, which read the table under a spec_id the set does not - show, as a composite answer carries none.""" - snapshot = _dac2a_and_crs_snapshot(monkeypatch) - crs = provider._ask( - "health aid from the United States to Kenya, 2015 to 2020", time="2015-2020" - ) - dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) - assert provider._source(crs) is None and provider._source(dac2a) is None - - def events(spec_id: str) -> list[FacetOptionsEvent]: - return [FacetOptionsEvent(spec_id=spec_id, axis="unit", options=[], truncated=False)] - - async def run() -> tuple[RegionResult, list[str]]: - async with scoped_concurrency(): - - async def probe() -> _OptionProbeResult: - return (*events(dac2a.spec_id), *events(crs.spec_id)), () - - before = RegionResult( - variable_text="aid", - status="candidates", - specs=(dac2a, crs), - no_data_reason=None, - warnings=(), - timing_by_step={}, - slot_options_task=asyncio.ensure_future(probe()), - ) - after = _readings(before, reads=_reads(snapshot), selection=ScopeState()) - after = composite_answer.without_restated_options(before, after) - assert after.slot_options_task is not None - served, _ = await after.slot_options_task - return after, [event.spec_id for event in served] - - after, served = asyncio.run(run()) - - restated, kept = after.specs - assert provider._source(restated) == provider._DAC2A_BILATERAL - assert kept.spec_id == crs.spec_id - assert served == [crs.spec_id] - - -@pytest.mark.parametrize( - ("entries", "source"), - [ - pytest.param((), provider._DAC2A_BILATERAL, id="no-entry"), - pytest.param((explicit("provider:contributions"),), None, id="stated-total"), - pytest.param((carried("provider:contributions"),), None, id="carried-total"), - pytest.param((explicit("provider:bilateral"),), provider._DAC2A_BILATERAL, id="stated"), - ], -) -def test_a_reading_beside_the_answer_honours_the_part_s_provider_entry( - monkeypatch: pytest.MonkeyPatch, entries: tuple[Entry, ...], source: str | None -): - """A candidate, alternate, flagged or nearest reading of DAC2a's total ODA is read as the - definite answer is: a stated or carried `provider:contributions` keeps total ODA, and - otherwise the reading is restated as the bilateral destinations.""" - snapshot = _dac2a_and_crs_snapshot(monkeypatch) - dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) - before = RegionResult( - variable_text="aid", - status="candidates", - specs=(dac2a,), - no_data_reason=None, - warnings=(), - timing_by_step={}, - alternate_match=dac2a, - flagged_match=(dac2a,), - nearest_real=(dac2a,), - ) - - after = _readings(before, reads=_reads(snapshot), selection=ScopeState(entries=entries)) - - assert after.alternate_match is not None - assert after.flagged_match is not None and after.nearest_real is not None - readings = (*after.specs, after.alternate_match, *after.flagged_match, *after.nearest_real) - assert [provider._source(spec) for spec in readings] == [source] * 4 - - -def test_a_set_left_with_a_crs_reading_answers_the_composite_its_scope_compiles( - monkeypatch: pytest.MonkeyPatch, -): - """A candidate set of a DAC2a reading whose population holds no row and a CRS reading of - total health aid is left with the CRS reading, whose scope decision compiles the inclusive - composite though no DAC2a default serves it as one. It answers that composite as a fresh - definite answer of the reading does: the composite's spec_id, the compiled scope and its - components, and the closure that reads its preview, never the table under a composite's - scope.""" - snapshot = _dac2a_and_crs_snapshot(monkeypatch) - dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) - _components(monkeypatch) - kept: list[tuple[RegionResult, ResolveState]] = [] - real = pipeline._with_scope_choices - - def capture(result: RegionResult, state: ResolveState) -> RegionResult: - kept.append((result, state)) - return real(result, state) - - monkeypatch.setattr(pipeline, "_with_scope_choices", capture) - _stream(_fresh()) - (table, state), *_ = kept - fresh = real(table, state) - assert fresh.scope_choices is not None and fresh.spec.spec_id != table.spec.spec_id - read = composite_answer.population_table_answer - - def holding_no_dac2a_row(spec: Spec, identity: BaseIdentity, *, reads: Any) -> Any: - if spec.spec_id == dac2a.spec_id: - return composite_answer._unread_population("no_observations") - return read(spec, identity, reads=reads) - - monkeypatch.setattr(composite_answer, "population_table_answer", holding_no_dac2a_row) - before = dataclasses.replace(table, status="candidates", specs=(dac2a, table.spec)) - - after = _readings(before, reads=_reads(snapshot), selection=state.scope_selection) - - assert (after.status, after.spec.spec_id) == ("definite", fresh.spec.spec_id) - assert after.scope_choices is not None - assert after.scope_choices.compiled == fresh.scope_choices.compiled - assert after.scope_choices.compiled.components - assert isinstance(after.observations_relaunch, CompositeRelaunch) - - -def test_a_restated_reading_relaunches_no_table_probe_on_a_replay( - monkeypatch: pytest.MonkeyPatch, -): - """A set whose own reading is restated as DAC2a's bilateral destinations carries no - closure that reads the table again, nor the facts a part memo hit binds them from: a - replay that launches the trailing tasks it lacks (`promote_relaunch.bind_relaunches`, - `enrichment._launch_trailing_tasks`) launches none under the composite's spec_id, as a - composite answer launches none.""" - snapshot = _dac2a_and_crs_snapshot(monkeypatch) - dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) - launched: list[str] = [] - - def relaunch(*args: Any) -> Any: - launched.append(repr(args)) - raise AssertionError("a restated reading relaunched a table probe") - - before = RegionResult( - variable_text="aid", - status="candidates", - specs=(dac2a,), - no_data_reason=None, - warnings=(), - timing_by_step={}, - slot_options_relaunch=relaunch, - place_options_relaunch=relaunch, - observations_relaunch=relaunch, - promote_relaunch=cast(Any, object()), - ) - - after = _readings(before, reads=_reads(snapshot), selection=ScopeState()) - replayed = bind_relaunches( - without_relaunches(after), - graph=cast(Any, None), - sdmx=cast(Any, None), - coverage=snapshot, - options=cast( - Any, - SimpleNamespace( - stream_slot_options=True, - stream_observations=True, - materialise_graph=object(), - place_options_pool=object(), - ), - ), - ) - launched_region = _launch_trailing_tasks( - replayed, launch_slot_options=True, launch_observations=True - ) - - assert provider._source(after.spec) == provider._DAC2A_BILATERAL - assert ( - launched_region.slot_options_task, - launched_region.place_options_task, - launched_region.observations_task, - ) == (None, None, None) - assert launched == [] - - -def test_a_population_read_keeps_the_request_s_warnings_and_replaces_the_read_s( - monkeypatch: pytest.MonkeyPatch, -): - """A fresh answer read again within its composite's population keeps the warnings raised - before its read and takes every other from that read: a whole-table count, and a - currency omission naming an agency the population leaves out, do not survive, whether - or not the population read raises their code.""" - _dac2a_and_crs_snapshot(monkeypatch) - table = provider._resubmitted(provider._dac2a_identity("Net"), None) - - def warning(code: str, message: str) -> Warning: - return Warning(code=code, severity="warn", message=message) - - words = warning("query_qualifier_dropped", "words") - fresh = RegionResult( - variable_text="aid", - status="definite", - specs=(table,), - no_data_reason=None, - warnings=( - words, - warning("role_scope_defaulted", "both"), - warning("currency_unmapped_omitted", "European Commission"), - ), - timing_by_step={}, - ) - answered = dataclasses.replace(fresh, warnings=(warning("role_scope_defaulted", "one"),)) - - grafted = composite_answer.with_population_read(fresh, answered, request_warnings=(words,)) - - assert [(w.code, w.message) for w in grafted.warnings] == [ - ("query_qualifier_dropped", "words"), - ("role_scope_defaulted", "one"), - ] - - def test_a_cache_hit_launches_no_place_options_for_a_composite_answer( monkeypatch: pytest.MonkeyPatch, ): """A response cache hit launches the place options a fresh run launches: one task per definite table answer, none for a composite answer, whose place options a fresh run - drops (`result.with_composite_answer`).""" + launches none of.""" composite_part = provider._posted_dac2a(monkeypatch, provider._DAC2A_BILATERAL, provider._KEN) table_part = provider._resubmitted_part( provider._dac2a_identity("Gross", destination=provider._DEVELOPING), None @@ -1287,101 +1030,6 @@ def recorded(spec: Spec, **_kwargs: Any) -> Any: assert launched == [table_part.spec.spec_id] -def test_a_restated_reading_reads_the_request_catalogue_whatever_scope_the_part_read_under( - monkeypatch: pytest.MonkeyPatch, -): - """A candidate restated as DAC2a's bilateral destinations is settled and restated through - the request's catalogue, as the definite answer's composite is, so a source scope the - part's own reads ran under narrows none of its figures.""" - snapshot = _dac2a_and_crs_snapshot(monkeypatch) - dac2a = provider._resubmitted(provider._dac2a_identity("Net"), None) - reads = _reads(snapshot) - scopes: dict[str, list[frozenset[str] | None]] = {"default": [], "assemble": []} - default, assemble = composite_answer.bilateral_default_reading, composite.assemble_composite - - def recorded_default(*args: Any, **kwargs: Any) -> Any: - scopes["default"].append(current_catalogue().scope) - return default(*args, **kwargs) - - def recorded_assemble(*args: Any, **kwargs: Any) -> Any: - scopes["assemble"].append(current_catalogue().scope) - return assemble(*args, **kwargs) - - monkeypatch.setattr(composite_answer, "bilateral_default_reading", recorded_default) - monkeypatch.setattr(composite_answer, "assemble_composite", recorded_assemble) - before = RegionResult( - variable_text="aid", - status="candidates", - specs=(dac2a,), - no_data_reason=None, - warnings=(), - timing_by_step={}, - ) - - with scoped_reads(frozenset({provider._DAC2A})): - (restated,) = _readings(before, reads=reads, selection=ScopeState()).specs - - assert provider._source(restated) == provider._DAC2A_BILATERAL - assert scopes == {"default": [reads.catalogue.scope], "assemble": [reads.catalogue.scope]} - - -def test_a_restated_answer_rekeys_only_its_own_keyed_facts(monkeypatch: pytest.MonkeyPatch): - """Restating a DAC2a answer as its bilateral destinations moves its own warnings, - unpublished-whole facts and assumption echoes to the composite's spec_id, and drops its - own all-recipients total, which the composite cannot stand beside. Another reading's - warnings and a warning keyed to no reading stand as they are.""" - snapshot = _dac2a_and_crs_snapshot(monkeypatch) - crs = provider._ask( - "health aid from the United States to Kenya, 2015 to 2020", time="2015-2020" - ) - table = provider._resubmitted(provider._dac2a_identity("Net"), None) - restated = _readings( - RegionResult( - variable_text="aid", - status="candidates", - specs=(table,), - no_data_reason=None, - warnings=(), - timing_by_step={}, - ), - reads=_reads(snapshot), - selection=ScopeState(), - ).spec - assert provider._source(restated) == provider._DAC2A_BILATERAL - - def warning(code: str, spec_id: str | None) -> Warning: - params = {"total": provider._ALL_RECIPIENTS} if code == "whole_total_available" else None - return Warning(code=code, severity="info", message=code, spec_id=spec_id, params=params) - - region = RegionResult( - variable_text="aid", - status="definite", - specs=(table, crs), - no_data_reason=None, - warnings=( - warning("role_scope_defaulted", table.spec_id), - warning("whole_total_available", table.spec_id), - warning("whole_total_available", crs.spec_id), - warning("whole_total_available", None), - ), - timing_by_step={}, - unpublished_whole_facts={table.spec_id: (), crs.spec_id: ()}, - reading_assumption_echoes={table.spec_id: (), crs.spec_id: ()}, - ) - - answered = result.with_composite_answer( - region, restated, observations_relaunch=cast(Any, lambda _spec_id: None) - ) - - assert [(w.code, w.spec_id) for w in answered.warnings] == [ - ("role_scope_defaulted", restated.spec_id), - ("whole_total_available", crs.spec_id), - ("whole_total_available", None), - ] - assert set(answered.unpublished_whole_facts) == {restated.spec_id, crs.spec_id} - assert set(answered.reading_assumption_echoes) == {restated.spec_id, crs.spec_id} - - def test_a_merge_of_composite_parts_with_no_axis_states_the_composite_scope( monkeypatch: pytest.MonkeyPatch, ): @@ -1400,16 +1048,3 @@ def test_a_merge_of_composite_parts_with_no_axis_states_the_composite_scope( group = [cast(RegionResult, SimpleNamespace(scope_choices=single))] * 2 assert pipeline.merged_scope_choices(merged, group, coverage=facts.coverage) == single - - -def _readings( - region: RegionResult, *, reads: ComponentReads, selection: ScopeState -) -> RegionResult: - """`region` with its readings restated (`with_bilateral_readings`) under scope facts of - no operation, as a part with no answer phase decision reads them.""" - return composite_answer.with_bilateral_readings( - region, - reads=reads, - selection=selection, - facts=ScopeFacts(index=get_index(), coverage=reads.coverage), - ) diff --git a/tests/engine/test_composite_decided_before_preview.py b/tests/engine/test_composite_decided_before_preview.py index d66dc7fe3..9485695a7 100644 --- a/tests/engine/test_composite_decided_before_preview.py +++ b/tests/engine/test_composite_decided_before_preview.py @@ -1,7 +1,7 @@ """A fresh answer's scope is decided once, before its preview launches -(`phase_answer._decide_answer_scope`). An answer the binders settle as a composite then reads -no preview of the one table it resolved, streamed or buffered; any other answer reads its -preview as before.""" +(`phase_answer._decide_answer_scope`, `reading.decide_read_scope`). An answer the binders +settle as a composite then reads no preview of the one table it resolved, streamed or +buffered. Any other answer reads its preview of that table.""" from __future__ import annotations @@ -18,9 +18,16 @@ from search.engine.concurrency import scoped_concurrency from search.engine.errors import GraphInfraError from search.engine.events import DoneEvent, ResponseEvent -from search.engine.region import phase_answer, pipeline, scope_decision +from search.engine.region import ( + candidate_collapse, + phase_answer, + reading, + scope_decision, +) from search.engine.region.candidates import resolve_candidates +from search.engine.region.composite_answer import CompositeRelaunch from search.engine.region.observations.dispatch import PreviewRead, PreviewReader +from search.engine.region.reading import ReadScope from search.engine.region.result import RegionResult from search.engine.region.scope_decision import ScopeDecision from search.engine.region.state import ResolveState @@ -47,6 +54,13 @@ _observed, _stream, ) +from tests.engine.test_dac2a_candidate_readings import ( + india_clients, + india_request, + patch_india_coverage, + refuse_the_recipient_reading_s_mint, +) +from tests.engine.test_dac2a_destination_populations import _BILATERAL as _DAC2A_BILATERAL from tests.engine.test_observations_rate_cache_dispatch import _resolve_candidates_in_pass # The provider rule settles this total as CRS's bilateral flows, a single-table answer. @@ -81,16 +95,23 @@ def _buffered(question: str) -> Spec: def _decisions(monkeypatch: pytest.MonkeyPatch) -> list[ScopeDecision]: - """Every scope decided from here on, by the answer phase or by `_with_scope_choices`.""" + """Every scope decided from here on, by the answer phase (`reading.decide_read_scope`) + or by `_with_scope_choices` (`scope_decision.decide_scope`).""" decided: list[ScopeDecision] = [] - real = phase_answer.decide_scope + read_scope, decide = phase_answer.decide_read_scope, scope_decision.decide_scope + + def record_read(*args: Any, **kwargs: Any) -> ReadScope: + scope = read_scope(*args, **kwargs) + assert scope.decision is not None + decided.append(scope.decision) + return scope def record(*args: Any, **kwargs: Any) -> ScopeDecision: - decision = real(*args, **kwargs) + decision = decide(*args, **kwargs) decided.append(decision) return decision - monkeypatch.setattr(phase_answer, "decide_scope", record) + monkeypatch.setattr(phase_answer, "decide_read_scope", record_read) monkeypatch.setattr(scope_decision, "decide_scope", record) return decided @@ -202,7 +223,7 @@ def test_the_preview_follows_the_scope_decided_for_the_finished_answer( reads = _table_reads(monkeypatch, refuse=False) real = phase_answer._decide_answer_scope - def mismatched(*args: Any) -> phase_answer._AnswerScope: + def mismatched(*args: Any) -> ReadScope: scope = real(*args) assert scope.decision.composite is not early_composite decision = dataclasses.replace(scope.decision, spec_id="other", composite=early_composite) @@ -236,7 +257,7 @@ def test_a_raising_decision_for_the_finished_answer_keeps_it_as_a_single_table( _components(monkeypatch) real = phase_answer._decide_answer_scope - def mismatched(*args: Any) -> phase_answer._AnswerScope: + def mismatched(*args: Any) -> ReadScope: scope = real(*args) return dataclasses.replace( scope, decision=dataclasses.replace(scope.decision, spec_id="other") @@ -256,21 +277,21 @@ def raises(spec: Spec, **kwargs: Any) -> ScopeDecision: def test_a_raising_decision_for_an_undecided_answer_keeps_it_as_a_single_table(monkeypatch, caplog): - """An answer reaching `pipeline._with_scope_choices` with no decision is decided there, + """An answer judged with no decision (`phase_answer.finished_scope`) is decided there, guarded as every decision is: a decision that raises leaves the answer as one table that states no scope, and the error is logged.""" _components(monkeypatch) - real = pipeline._stating_unpublished_wholes + real = phase_answer.finished_scope - def undecided(result: RegionResult, state: ResolveState) -> RegionResult: - state.scope_decision = None - return real(result, state) + async def undecided(state: ResolveState, spec: Spec, **kwargs: Any) -> ScopeDecision | None: + return await real(state, spec, **{**kwargs, "decision": None}) def raises(*args: Any, **kwargs: Any) -> ScopeDecision: raise _not_servable() - monkeypatch.setattr(pipeline, "_stating_unpublished_wholes", undecided) - # The answer phase decides through its own `decide_scope`, so only the late decision raises. + monkeypatch.setattr(phase_answer, "finished_scope", undecided) + # The answer phase decides through `reading.build_part_scope`, so only the late decision + # raises. monkeypatch.setattr(scope_decision, "decide_scope", raises) with caplog.at_level(logging.WARNING, logger=scope_decision.logger.name): @@ -288,14 +309,23 @@ def _raising_decisions( monkeypatch: pytest.MonkeyPatch, error: Callable[[], Exception] = _not_servable ) -> list[str]: """Make every scope decision from here on raise `error()`, `CompositeNotServable` by - default, and list the `spec_id` of each one asked for.""" + default, and list the `spec_id` of each one asked for. Those are the answer phase's and + the candidates path's (`reading.build_part_scope`, called by `reading.decide_read_scope`) + and a late one (`scope_decision.decide_scope`). The build made by a reading election + (`reading.elect_reading`) passes through to the real one.""" asked: list[str] = [] + build = reading.build_part_scope def raises(spec_id: str, *args: Any, **kwargs: Any) -> ScopeDecision: asked.append(spec_id) raise error() - monkeypatch.setattr(phase_answer, "decide_scope", raises) + def raising_build(identity: Any, choice: Any, *, spec_id: str, facts: Any) -> ScopeDecision: + if spec_id == reading._ELECTION: + return build(identity, choice, spec_id=spec_id, facts=facts) + return raises(spec_id) + + monkeypatch.setattr(reading, "build_part_scope", raising_build) monkeypatch.setattr(scope_decision, "decide_scope", raises) return asked @@ -358,11 +388,7 @@ def record(self: PreviewReader) -> Any: return real(self) monkeypatch.setattr(PreviewReader, "observations", record) - - def raises(spec: Spec, **kwargs: Any) -> ScopeDecision: - raise CompositeNotServable("no component", ref="dfo-composite", pointer="/facets") - - monkeypatch.setattr(scope_decision, "decide_spec_scope", raises) + asked = _raising_decisions(monkeypatch) outcome: list[tuple[RegionResult | None, ScopeDecision | None]] = [] async def body(state: ResolveState) -> None: @@ -371,7 +397,7 @@ async def body(state: ResolveState) -> None: result = await resolve_candidates(state, rate_cache=RateTableCache()) if result is not None and result.observations_task is not None: result.observations_task.cancel() - outcome.append((result, state.scope_decision)) + outcome.append((result, None if result is None else result.scope_decision)) with caplog.at_level(logging.WARNING, logger=scope_decision.logger.name): _resolve_candidates_in_pass(monkeypatch, body) @@ -386,7 +412,10 @@ async def body(state: ResolveState) -> None: False, ) assert reads == ["observations"] - _assert_decision_error_logged(caplog) + # Each grounded candidate's decision raises, and the unit filter keeps one of them. + assert result.spec.spec_id in asked + errors = [r.exc_info[0] for r in caplog.records if r.exc_info is not None] + assert errors and set(errors) == {CompositeNotServable} @pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streamed"]) @@ -412,34 +441,51 @@ def test_a_raising_scope_decision_leaves_a_carve_out_refusal_standing(monkeypatc @pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streamed"]) -def test_a_candidates_survivor_restated_as_a_composite_reads_no_table_preview(monkeypatch, stream): - """The candidates path decides the survivor's scope before its preview launches - (`candidate_collapse._assemble_definite`). A survivor restated as a composite reads no - preview of its table: neither the bound-unit eager read nor the buffered read or the - streamed relaunch.""" - reads: list[str] = [] - monkeypatch.setattr(PreviewReader, "observations", lambda self: reads.append("observations")) - monkeypatch.setattr(PreviewReader, "relaunch", lambda self, labels: reads.append("relaunch")) - real = scope_decision.decide_spec_scope +def test_a_candidates_survivor_minted_as_a_composite_reads_no_table_preview(monkeypatch, stream): + """The candidates path decides each candidate's scope when its Spec is minted + (`candidate_grounding._candidate_answer`), before the survivor's preview launches + (`candidate_collapse._assemble_definite`). A survivor minted as a composite reads no + preview of its table, buffered or streamed, and its preview reads the composite's + components (`CompositeRelaunch`). + + The set is India's DAC2a readings with every recipient open, left with the donor reading + over the bilateral composite's population once the recipient reading's mint refuses + (`test_dac2a_candidate_readings`).""" + patch_india_coverage(monkeypatch) + refuse_the_recipient_reading_s_mint(monkeypatch) + # The survivor's table preview is read by the candidates path or by the answer phase. The + # composite's components read theirs through their own resubmits. + reads = _table_reads(monkeypatch, refuse=False) - def composite(spec: Spec, **kwargs: Any) -> ScopeDecision: - return dataclasses.replace(real(spec, **kwargs), composite=True) + class Recording(PreviewReader): + def observations(self) -> Any: + reads.append("observations") + return super().observations() - monkeypatch.setattr(scope_decision, "decide_spec_scope", composite) - outcome: list[RegionResult | None] = [] + def relaunch(self, labels: Any) -> Any: + reads.append("relaunch") + return super().relaunch(labels) - async def body(state: ResolveState) -> None: - assert state.bound_unit_outcome is not None - state.include_observations = True - state.stream_observations = stream - async with scoped_concurrency(): - outcome.append(await resolve_candidates(state, rate_cache=RateTableCache())) + monkeypatch.setattr(candidate_collapse, "PreviewReader", Recording) + survivors: list[RegionResult] = [] + assemble = candidate_collapse._assemble_definite - _resolve_candidates_in_pass(monkeypatch, body) + async def capture(*args: Any, **kwargs: Any) -> RegionResult: + result = await assemble(*args, **kwargs) + survivors.append(result) + return result - (result,) = outcome - assert result is not None + monkeypatch.setattr(candidate_collapse, "_assemble_definite", capture) + request = india_request(open_role="recipient", observations=True) + + events = offline_events(request, stream_slot_options=stream, **india_clients()) + + spec = _answered(events) + (result,) = survivors assert result.status == "definite" + assert result.spec.spec_id == spec.spec_id + pin = (spec.definition.identity.facets or {}).get("provenance") + assert pin is not None and pin.ref == _DAC2A_BILATERAL assert result.observations_task is None - assert result.observations_relaunch is None + assert isinstance(result.observations_relaunch, CompositeRelaunch) assert reads == [] diff --git a/tests/engine/test_composite_mint.py b/tests/engine/test_composite_mint.py new file mode 100644 index 000000000..345cbca55 --- /dev/null +++ b/tests/engine/test_composite_mint.py @@ -0,0 +1,157 @@ +"""`build_spec` minting a composite (`scope.composite.CompositeMint`) builds a Spec that states +the composite's identity and spec_id and counts the composite's components. + +Each case runs a part and records the `build_spec` calls that minted a Spec as the composite. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest + +from code_index.composites import component_identities, table_identity +from code_index.shared import get_index +from search.engine import same_shape, spec_build +from search.engine.region import ( + candidate_grounding, + grounding, + phase_answer, + phase_shared, + promote_ground, +) +from search.engine.region.codes import ROLE_SCOPE_DEFAULTED +from search.engine.retrieval.bound_values import served_members +from search.engine.spec_id import identity_spec_id +from search.models import BaseIdentity, Spec, Warning +from series_coverage.schema import RoleNames +from tests.engine._fake_coverage import FakeCoverageService, build_coverage_index +from tests.engine._harness import offline_resolve +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_dac2a_destination_populations import ( + _HELD, + _SCENARIOS, + _VARIABLE, + _graph_with_parties, + _patched_coverage, + _sdmx, +) +from tests.engine.test_promote_shape_sample import _USA, _YEARS + +_CALLERS = (phase_answer, promote_ground, candidate_grounding, grounding, phase_shared, same_shape) +"""Every module that builds a Spec.""" + + +@dataclass(frozen=True) +class _Built: + """One `build_spec` call and the Spec it built.""" + + args: tuple[Any, ...] + kwargs: dict[str, Any] + spec: Spec + + +def _recorded_builds(monkeypatch: pytest.MonkeyPatch) -> list[_Built]: + """Every `build_spec` call any caller makes from here on, in order.""" + built: list[_Built] = [] + + def recording(*args: Any, **kwargs: Any) -> Spec: + spec = spec_build.build_spec(*args, **kwargs) + built.append(_Built(args, kwargs, spec)) + return spec + + for module in _CALLERS: + monkeypatch.setattr(module, "build_spec", recording) + return built + + +def _minted(monkeypatch: pytest.MonkeyPatch, run: Callable[[], object]) -> list[_Built]: + """Each `build_spec` call `run` makes that mints a Spec as a composite.""" + built = _recorded_builds(monkeypatch) + run() + return [item for item in built if item.kwargs.get("composite") is not None] + + +def _headline_coverage(document: dict[str, Any], *, origin: str = _USA) -> FakeCoverageService: + """Series over 2005 to 2024 from `origin`, the United States unless named, for every + member a mint of `document`'s table or of any of its components serves.""" + index = get_index() + identities = [ + table_identity(index, document).identity, + *(c.identity for c in component_identities(index, document) or ()), + ] + members = sorted( + {m for doc in identities for m in served_members(BaseIdentity.model_validate(doc)) or ()} + ) + roles = dict.fromkeys(members, RoleNames(donor="financingProvider", recipient=None)) + rows = [(dcid, "facet", origin, "", _YEARS) for dcid in members] + return FakeCoverageService(build_coverage_index(rows, role_names=roles)) + + +@pytest.fixture +def _dac2a_coverage(monkeypatch: pytest.MonkeyPatch) -> None: + """The DAC2a fixture's coverage snapshot (`test_dac2a_destination_populations`).""" + _patched_coverage(monkeypatch, _HELD) + + +def _fresh(scenario: str) -> Callable[[], object]: + return lambda: offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + +@pytest.mark.parametrize( + ("run", "composites"), + [ + *(pytest.param(_fresh(name), 1, id=name) for name in ("H1", "H2", "H6")), + # All recipients spans both sides, so the part reads total ODA and mints no composite. + pytest.param(_fresh("H5"), 0, id="H5"), + ], +) +@pytest.mark.usefixtures("_dac2a_coverage") +def test_a_minted_dac2a_composite_states_the_composite_s_identity_and_components( + run: Callable[[], object], composites: int, monkeypatch: pytest.MonkeyPatch +) -> None: + minted = _minted(monkeypatch, run) + + assert len(minted) >= composites + assert bool(minted) == bool(composites) + for each in minted: + mint = each.kwargs["composite"] + assert each.spec.spec_id == mint.spec_id == identity_spec_id(mint.identity) + assert each.spec.definition.identity == mint.identity + assert len(mint.components()) == len(mint.served) + + +@pytest.mark.usefixtures("_dac2a_coverage") +def test_the_mint_states_the_defaulted_scope_count_its_role_axis_fact_counts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A `role_scope_defaulted` warning on the composite's open axis, keyed to the minted + Spec or to none, states the count of the parties the mint lists there, found by its code + and axis whatever its text says. A warning on an axis the composite counts nothing on + stands, as does one keyed to another reading beside the answer.""" + (first, *_rest) = _minted(monkeypatch, _fresh("H2")) + minted = first.spec + mint = first.kwargs["composite"] + on_axis = Warning( + code=ROLE_SCOPE_DEFAULTED, + severity="info", + message="anything", + params={"axis": "destination"}, + ) + keyed = on_axis.model_copy(update={"spec_id": minted.spec_id}) + off_axis = on_axis.model_copy(update={"params": {"axis": "origin"}}) + other = Warning(code="observations_scope_partial", severity="warn", message="partial") + elsewhere = on_axis.model_copy(update={"spec_id": "spec_another_reading"}) + + stated = mint.role_scope_warnings((on_axis, keyed, off_axis, other, elsewhere), minted) + + assert stated[0].message != on_axis.message + assert stated[1].message == stated[0].message + assert stated[2:] == (off_axis, other, elsewhere) diff --git a/tests/engine/test_composite_resubmit_mint.py b/tests/engine/test_composite_resubmit_mint.py new file mode 100644 index 000000000..bb79ba036 --- /dev/null +++ b/tests/engine/test_composite_resubmit_mint.py @@ -0,0 +1,329 @@ +"""A dataset_resubmit of a composite, and a fresh part whose read elected one, mint the Spec +as the composite (`spec_build.build_spec` with a `scope.composite.CompositeMint`), so each +states the composite's spec_id, warnings and counts with no restatement of the table answer +it read. + +Each resubmit case answers a composite under the spec_id of its identity. A fresh answer +equals a resubmit of its definition. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest + +from code_index.composites import table_identity +from code_index.declarations import DAC2A_INCLUSIVE +from code_index.shared import get_index +from search.engine import app as search_app +from search.engine import core as search_core +from search.engine.entry import dataset_resubmit +from search.engine.events import ResponseEvent +from search.engine.region import RegionResult, composite_answer, result +from search.engine.region.composite_answer import CompositeRelaunch +from search.engine.spec_id import identity_spec_id +from search.models import ( + BaseIdentity, + DatasetResubmitInput, + DefinitePart, + Resolution, + ResolveOptions, + ResolveRequest, + ResubmitIdentityEntry, +) +from tests.engine._fake_sdmx import FakeSdmxClient +from tests.engine._harness import _walk_arcs, offline_events, offline_resolve +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_composite_mint import _headline_coverage, _recorded_builds +from tests.engine.test_dac2a_destination_populations import ( + _AT_TOTAL, + _HELD, + _IMPUTED_ONLY, + _SCENARIOS, + _VARIABLE, + _answer, + _graph_with_parties, + _identity, + _materialise, + _patched_coverage, + _resolve_fresh, + _resolve_resubmit, + _resubmit, + _sdmx, +) +from tests.engine.test_promote_shape_sample import ( + _HEADLINE, + _USA, + _composites, + _member_node, + _SampleReader, +) +from tests.engine.test_whole_total_bind_e2e import _FRA +from tests.fixtures import FakeGraph + +_DAC1 = "provenance/OECDDAC1" + + +def _part(response: object) -> DefinitePart: + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, DefinitePart), part + return part + + +def _stated(part: DefinitePart) -> dict[str, Any]: + """The part's Spec, what its resolved part states beyond its dump, and its warnings.""" + return { + "spec": part.spec.model_dump(mode="json"), + "private": part.spec.resolved.__pydantic_private__, + "warnings": [warning.model_dump(mode="json") for warning in part.warnings], + } + + +def _dac2a( + monkeypatch: pytest.MonkeyPatch, + identity: BaseIdentity, + *, + held: dict[tuple[str, str, str], int] = _HELD, +) -> Callable[[], DefinitePart]: + """Resubmits of `identity` over the DAC2a fixture's series `held` + (`test_dac2a_destination_populations`).""" + _patched_coverage(monkeypatch, held) + + def resubmitted() -> DefinitePart: + return _part( + offline_resolve( + _resubmit(identity), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(held), + ) + ) + + return resubmitted + + +def _fresh_definition(monkeypatch: pytest.MonkeyPatch, scenario: str) -> BaseIdentity: + """The definition a fresh answer to `scenario` states over the DAC2a fixture.""" + _patched_coverage(monkeypatch, _HELD) + return _part( + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ).spec.definition.identity + + +def _headline( + monkeypatch: pytest.MonkeyPatch, *, origin: str = _USA, label: str = "United States" +) -> Callable[[], DefinitePart]: + """The OECD headline from `origin` over 2015 to 2020, net flows to 2017 and grant + equivalents from 2018.""" + document = {**_composites()[_HEADLINE], "entities": {"origin": [origin]}} + service = _headline_coverage(document, origin=origin) + monkeypatch.setattr(search_app, "get_coverage_service", lambda: service) + monkeypatch.setattr(search_core, "get_coverage_service", lambda: service) + graph = FakeGraph( + nodes={ + origin: {"label": label, "type": "Country"}, + **{row.dcid: _member_node(row) for row in get_index().statvars}, + } + ) + request = ResolveRequest( + input=DatasetResubmitInput( + entries=[ + ResubmitIdentityEntry( + variable_text="official development assistance", + identity=BaseIdentity.model_validate(document), + ) + ] + ) + ) + return lambda: _part( + offline_resolve( + request, + graph=graph, + sdmx=FakeSdmxClient(observations={}, availability={}), + materialise_graph=_SampleReader(serve=False, arcs=_walk_arcs(graph)), + ) + ) + + +def _assert_nothing_restates() -> None: + """No function restates a table answer as the composite. The rekeying, the recount and the + restated reading do not exist, so no part or resubmit can run them.""" + assert not hasattr(result, "with_restated_keys") + assert not hasattr(result, "recounted_scope") + assert not hasattr(composite_answer, "restated_reading") + + +@pytest.mark.parametrize( + "case", + [ + *( + pytest.param(lambda mp, name=name: _dac2a(mp, _fresh_definition(mp, name)), id=name) + for name in ("H1", "H2", "H6") + ), + pytest.param( + lambda mp: _dac2a(mp, _identity(DAC2A_INCLUSIVE)), id="inclusive-open-destination" + ), + # The imputed line holds a recipient the bilateral table lacks, so the composite + # counts two recipients where its table answer counts one. + pytest.param( + lambda mp: _dac2a(mp, _identity(DAC2A_INCLUSIVE), held=_IMPUTED_ONLY), + id="inclusive-imputed-only-recipient", + ), + pytest.param(_headline, id="oecd-headline"), + ], +) +def test_a_composite_resubmit_answers_as_the_composite_with_no_restatement_layer( + case: Callable[[pytest.MonkeyPatch], Callable[[], DefinitePart]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + _assert_nothing_restates() + answered = case(monkeypatch)() + + assert answered.spec.definition.identity.facets is not None + assert answered.spec.spec_id == identity_spec_id(answered.spec.definition.identity) + + +def test_a_streamed_composite_resubmit_launches_none_of_its_table_s_probes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The table identity a composite reads offers slot options to stream. The composite + launches none and reads its preview per component under its own spec_id.""" + _patched_coverage(monkeypatch, _HELD) + composite = _identity(DAC2A_INCLUSIVE) + document = composite.model_dump(mode="json", exclude_none=True) + table = BaseIdentity.model_validate(dict(table_identity(get_index(), document).identity)) + launched: list[RegionResult] = [] + launch = dataset_resubmit._launch_trailing_tasks + + def recording(region: RegionResult, **kwargs: bool) -> RegionResult: + launched.append(region) + return launch(region, **kwargs) + + monkeypatch.setattr(dataset_resubmit, "_launch_trailing_tasks", recording) + for identity in (table, composite): + materialise_graph, materialise_sdmx = _materialise() + request = _resubmit(identity, observations=True) + options = request.response_options.model_copy(update={"include_option_coverage": True}) + events = offline_events( + request.model_copy(update={"response_options": options}), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + materialise_graph=materialise_graph, + materialise_sdmx=materialise_sdmx, + stream_slot_options=True, + ) + + read, minted = launched + assert read.slot_options_relaunch is not None + assert minted.slot_options_relaunch is None + assert minted.place_options_relaunch is None + assert isinstance(minted.observations_relaunch, CompositeRelaunch) + assert [(e.kind, getattr(e, "spec_id", None)) for e in events if e.kind != "sentence"] == [ + ("response", None), + ("observations", identity_spec_id(composite)), + ("done", None), + ] + + +@pytest.mark.parametrize( + ("scenario", "held"), + [ + *(pytest.param(scenario, _HELD, id=scenario) for scenario in ("H1", "H2", "H5", "H6")), + pytest.param("D2", _AT_TOTAL, id="D2"), + pytest.param("D2-ranking", _AT_TOTAL, id="D2-ranking"), + ], +) +def test_a_fresh_answer_equals_a_resubmit_of_its_definition( + scenario: str, held: dict[tuple[str, str, str], int], monkeypatch: pytest.MonkeyPatch +) -> None: + """A fresh part reads its composite within its population and mints its Spec as the + composite, so it states the scope, headline, coverage, entities and warnings a resubmit + of its definition states.""" + _patched_coverage(monkeypatch, held) + _assert_nothing_restates() + fresh = _resolve_fresh(scenario, held=held) + + again = _resolve_resubmit(fresh.spec.definition.identity, held=held) + + assert again.spec.spec_id == fresh.spec.spec_id + assert _answer(again.spec, again) == _answer(fresh.spec, fresh) + + +def test_a_streamed_fresh_composite_launches_none_of_its_table_s_probes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh part whose read elected the bilateral composite streams no slot or place + option of the table it read, and reads its preview per component under the composite's + spec_id.""" + _patched_coverage(monkeypatch, _HELD) + materialise_graph, materialise_sdmx = _materialise() + request = _SCENARIOS["H1"]().model_copy( + update={ + "response_options": ResolveOptions( + include_observations=True, include_option_coverage=True + ) + } + ) + + events = offline_events( + request, + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + materialise_graph=materialise_graph, + materialise_sdmx=materialise_sdmx, + stream_slot_options=True, + ) + + (response,) = [e for e in events if isinstance(e, ResponseEvent)] + part = _part(response.response) + assert (part.spec.definition.identity.facets or {}).get("provenance") is not None + trailing = events[events.index(response) :] + assert [(e.kind, getattr(e, "spec_id", None)) for e in trailing if e.kind != "sentence"] == [ + ("response", None), + ("observations", part.spec.spec_id), + ("done", None), + ] + + +def test_the_oecd_headline_resubmitted_serves_net_flows_then_grant_equivalents( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The OECD headline over France from 2015 to 2020, posted at the resubmit door, is + minted as the composite. Its net flows component serves 2015 to 2017 and its grant + equivalent component 2018 to 2020, both from DAC1, and the Spec states both tables' + series in the years each serves.""" + minted = _recorded_builds(monkeypatch) + + part = _headline(monkeypatch, origin=_FRA, label="France")() + + (built,) = [each for each in minted if each.kwargs.get("composite") is not None] + mint = built.kwargs["composite"] + assert [ + ( + component.role, + [tuple(period) for period in component.periods], + component.provenance, + ) + for component in mint.components() + ] == [ + ("headline_net", [(2015, 2017)], _DAC1), + ("headline_ge", [(2018, 2020)], _DAC1), + ] + resolved = part.spec.resolved + assert part.spec.spec_id == mint.spec_id == identity_spec_id(mint.identity) + assert part.spec.definition.identity.entities == {"origin": [_FRA]} + assert (resolved.stat_vars.member_count, resolved.stat_vars.served_count) == (2, 2) + assert resolved.download is not None and resolved.download.series == 2 + assert resolved.coverage is not None and resolved.coverage.observation_count == 6 + assert resolved.headline is not None + assert [year.year for year in resolved.headline.recent_years] == [2020, 2019, 2018] diff --git a/tests/engine/test_conjoin_assemble_and_enrichment.py b/tests/engine/test_conjoin_assemble_and_enrichment.py index a9f3b4d04..f7e1c27f8 100644 --- a/tests/engine/test_conjoin_assemble_and_enrichment.py +++ b/tests/engine/test_conjoin_assemble_and_enrichment.py @@ -858,6 +858,7 @@ def test_window_refusal_keeps_original_enrichment_tasks_joinable_to_visible_part async def _body(): call_log: list[str] = [] spec_a = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -875,6 +876,7 @@ async def _body(): requested_window=TimeWindow(start_year=2015, end_year=2020), ) spec_b = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, diff --git a/tests/engine/test_conjoin_entities_and_facets.py b/tests/engine/test_conjoin_entities_and_facets.py index e912e6d37..bd2e954e4 100644 --- a/tests/engine/test_conjoin_entities_and_facets.py +++ b/tests/engine/test_conjoin_entities_and_facets.py @@ -78,6 +78,7 @@ def _spec_with_facet( """Like _spec, but with a door-posted facet binding (definition.identity.facets) and open roles, the concrete cases a same-shape merge must compare and carry forward.""" return build_spec( + composite=None, open_roles=open_roles, measure_elected_properties=frozenset(), shape=shape, @@ -105,6 +106,7 @@ def _spec_with_resolved_facets( State `unit_lexicon_version` after `build_spec`. """ spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -175,6 +177,7 @@ def _where_bound_spec(shape: Shape, recipient_dcid: str, sv_dcid: str): binding=BindingValue(value=SlotValue(ref=_gr(recipient_dcid), value_kind="entity")), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -202,6 +205,7 @@ def _where_bound_spec_with_provider( binding=BindingValue(value=SlotValue(ref=_gr(recipient_dcid), value_kind="entity")), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -508,6 +512,7 @@ def test_collapse_same_shape_reversed_provider_recipient_stays_distinct(): def _place_constraint_off_spec(constraint_val, sv_dcid, identity_entities): return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -571,6 +576,7 @@ def test_collapse_same_shape_divergent_entities_on_non_where_axis_stays_residual def _place_constraint_off_spec(constraint_val, sv_dcid, identity_entities): return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -605,6 +611,7 @@ def test_collapse_same_shape_window_survives_merge(): """The merged spec keeps the requested window in its identity and trace.""" window = TimeWindow(start_year=2015, end_year=2020) spec_a = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -622,6 +629,7 @@ def test_collapse_same_shape_window_survives_merge(): requested_window=window, ) spec_b = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -661,6 +669,7 @@ def test_collapse_same_shape_refuses_merge_on_window_disagreement(): window_a = TimeWindow(start_year=2015, end_year=2020) window_b = TimeWindow(start_year=2018, end_year=2020) spec_a = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -678,6 +687,7 @@ def test_collapse_same_shape_refuses_merge_on_window_disagreement(): requested_window=window_a, ) spec_b = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -714,6 +724,7 @@ def test_combine_regions_keeps_window_refusal_parts_and_effective_regions_distin window_a = TimeWindow(start_year=2015, end_year=2020) window_b = TimeWindow(start_year=2018, end_year=2020) spec_a = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -731,6 +742,7 @@ def test_combine_regions_keeps_window_refusal_parts_and_effective_regions_distin requested_window=window_a, ) spec_b = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, diff --git a/tests/engine/test_conjoin_five_tuple_and_slots.py b/tests/engine/test_conjoin_five_tuple_and_slots.py index c4d71c0ae..aa830d483 100644 --- a/tests/engine/test_conjoin_five_tuple_and_slots.py +++ b/tests/engine/test_conjoin_five_tuple_and_slots.py @@ -45,6 +45,7 @@ def _slot_options(prop: str, *dcids: str, truncated: bool = True) -> SlotOptions def _spec_with_extra_slots(shape, constraint_val: str, *extra_slots, sv_dcid: str = "test_sv"): """Build a spec with extra slots to test unbound-slot roster merging (#590).""" return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -399,6 +400,7 @@ def _purpose_region( else _set_slot("what", "crsPurpose", "crsPurpose", list(code)) ) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_ECON_SHAPE, @@ -655,6 +657,7 @@ def _named_member_region(slot: Slot, sv_dcid: str, name: str, index: int, *place slot_key = SlotKey(axis=slot.key.axis, property=slot.key.property, label=slot.key.label) shape = _PERSON_SHAPE.model_copy(update={"label": _FAMILY_LABEL, "slot_keys": [slot_key]}) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, diff --git a/tests/engine/test_conjoin_measure_merge.py b/tests/engine/test_conjoin_measure_merge.py index a0dd87ba9..9925af550 100644 --- a/tests/engine/test_conjoin_measure_merge.py +++ b/tests/engine/test_conjoin_measure_merge.py @@ -95,6 +95,7 @@ def _where_bound_spec( role=EntityRoleDirectional(role=_gr(_RECIPIENT_PROP), axis="destination"), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, diff --git a/tests/engine/test_conjoin_warnings_and_primary.py b/tests/engine/test_conjoin_warnings_and_primary.py index 268811a6a..a8f277b02 100644 --- a/tests/engine/test_conjoin_warnings_and_primary.py +++ b/tests/engine/test_conjoin_warnings_and_primary.py @@ -52,6 +52,7 @@ def test_same_shape_multislot_residual_warns(): def _two_slot_spec(val_a: str, val_b: str): return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape_two_slots, @@ -102,6 +103,7 @@ def test_combine_regions_keeps_multislot_refusal_parts_distinct(): def _two_slot_spec(val_a: str, val_b: str): return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape_two_slots, @@ -161,6 +163,7 @@ def test_collapse_same_shape_exact_coverage_downgrade_unions_has_data(): """ window = TimeWindow(start_year=2020, end_year=2020) spec_a = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, @@ -177,6 +180,7 @@ def test_collapse_same_shape_exact_coverage_downgrade_unions_has_data(): coverage=CoverageExact(has_data=False, observation_count=0, dimensions=None, window=window), ) spec_b = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_PERSON_SHAPE, diff --git a/tests/engine/test_conjunction.py b/tests/engine/test_conjunction.py index 98516c230..7cd8aa391 100644 --- a/tests/engine/test_conjunction.py +++ b/tests/engine/test_conjunction.py @@ -251,6 +251,7 @@ def test_same_shape_multislot_refusal_keeps_each_members_own_spec(): def _two_slot_spec(val_a: str, val_b: str): return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape_two_slots, diff --git a/tests/engine/test_coverage_facet_dates.py b/tests/engine/test_coverage_facet_dates.py index 82a66e1b7..4fe34d842 100644 --- a/tests/engine/test_coverage_facet_dates.py +++ b/tests/engine/test_coverage_facet_dates.py @@ -112,6 +112,7 @@ def _resolve( ), coverage=FakeCoverageService(index).snapshot(), discover_scope=discover, + within=None, ) assert isinstance(result, Materialised) return result diff --git a/tests/engine/test_dac2a_candidate_readings.py b/tests/engine/test_dac2a_candidate_readings.py new file mode 100644 index 000000000..dd6ad534d --- /dev/null +++ b/tests/engine/test_dac2a_candidate_readings.py @@ -0,0 +1,697 @@ +"""A fresh candidate set whose DAC2a readings settle on population composites elects each +reading at its own read, reads it once, within the population where the reading leaves its +destination open, and mints its Spec as the composite, so each candidate states what a +resubmit of its definition states. + +India holds DAC2a's total ODA on both sides here, so "official development assistance in +India" offers India as the donor and as the recipient. Each reading settles on the +bilateral composite, the donor reading at the bilateral population's total and the +recipient reading at India, a bilateral recipient.""" + +from __future__ import annotations + +import dataclasses +import logging +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +from code_index.composites import destination_population +from code_index.shared import get_index +from origin_catalogue import current_catalogue, scoped_reads +from search.engine.events import ResponseEvent +from search.engine.region import ( + RegionResult, + candidate_collapse, + candidate_grounding, + phase_place_materialise, + pipeline, + reading, + resubmit, +) +from search.engine.region.scope_decision import _withheld_warning +from search.engine.region.state import ResolveState +from search.models import ( + BaseIdentity, + CandidatesPart, + DefinitePart, + Entry, + NearestRealSpec, + NoDataPart, + Relaxation, + Resolution, + ResolveOptions, + ResolveRequest, +) +from search.scope.provider import Withheld +from series_coverage.schema import FacetMetadata +from tests.engine._builders import raise_answer_warnings, slow_related_reads +from tests.engine._fake_coverage import FakeCoverageService, build_coverage_index, patch_coverage +from tests.engine._harness import offline_events, offline_resolve +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_dac2a_destination_populations import ( + _AGENCY, + _BILATERAL, + _NAMES, + _YEARS, + _counted, + _fresh_request, + _identity, + _offer_the_bilateral_composite, + _patched_coverage, + _resolve_resubmit, + _sdmx, + _shared_fields, + _table_reads, +) +from tests.engine.test_dac2a_destination_populations import _HELD as _FRANCE_HELD +from tests.engine.test_whole_total_bind_e2e import ( + _ALL_RECIPIENTS, + _DAC2A, + _DAC2A_ATTRIBUTED, + _DAC2A_MEMBER, + _ETH, + _FRA, + _IND, + _OFFICIAL, + _VARIABLE, + _graph, +) +from tests.scope._axes import carried + +_DEVELOPING = "DACDevelopingCountries" +# India's receipts from France and the official donors, and its own ODA to Ethiopia, to an +# agency and to the bilateral population's total. +_HELD = { + (_DAC2A_MEMBER, _FRA, _IND): 100, + (_DAC2A_MEMBER, _OFFICIAL, _IND): 300, + (_DAC2A_MEMBER, _IND, _ETH): 40, + (_DAC2A_MEMBER, _IND, _AGENCY): 20, + (_DAC2A_MEMBER, _IND, _DEVELOPING): 45, + (_DAC2A_ATTRIBUTED, _IND, _ETH): 3, +} + + +_ROLE_NAMES = dict.fromkeys((_DAC2A_MEMBER, _DAC2A_ATTRIBUTED), _NAMES) +_FACETS = {"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")} + + +def _rows(held: dict[tuple[str, str, str], int]) -> list[tuple[str, str, str, str, list[int]]]: + return [(sv, "dac2a", donor, recipient, list(_YEARS)) for (sv, donor, recipient) in held] + + +def patch_india_coverage( + monkeypatch: pytest.MonkeyPatch, held: dict[tuple[str, str, str], int] = _HELD +) -> None: + """The coverage snapshot holding the series `held` in every year of the window.""" + patch_coverage(monkeypatch, _rows(held), role_names=_ROLE_NAMES, facet_metadata=_FACETS) + + +def read_the_answer_against( + monkeypatch: pytest.MonkeyPatch, held: dict[tuple[str, str, str], int] +) -> None: + """The answer's read reads a coverage snapshot holding the series `held` alone, while + the election and the role resolution before it read the one in force, as a table the + snapshot disagrees with would.""" + snapshot = FakeCoverageService( + build_coverage_index(_rows(held), role_names=_ROLE_NAMES, facet_metadata=_FACETS) + ).snapshot() + run = phase_place_materialise.run_materialise + + def reading_held(*args: Any, **kwargs: Any) -> Any: + return run(*args, **{**kwargs, "coverage": snapshot}) + + monkeypatch.setattr(phase_place_materialise, "run_materialise", reading_held) + + +@pytest.fixture(autouse=True) +def _coverage(monkeypatch: pytest.MonkeyPatch) -> None: + patch_india_coverage(monkeypatch) + + +def india_request( + *, + open_role: str | None = None, + carried_entries: tuple[Entry, ...] = (), + observations: bool = False, +) -> ResolveRequest: + """The request for official development assistance in India, India's role left to the + readings, with `open_role` open, the conversation's `carried_entries`, and the preview + asked for with `observations`.""" + request = _fresh_request( + [(None, "India")], message="official development assistance in India", open_role=open_role + ) + return request.model_copy( + update={ + "input": request.input.model_copy( + update={"carried_entries": list(carried_entries) or None} + ), + "response_options": ResolveOptions(include_observations=True) if observations else None, + } + ) + + +def india_clients(held: dict[tuple[str, str, str], int] = _HELD) -> dict[str, Any]: + """The graph, binder and SDMX doubles a request about India reads, over `held`.""" + graph = _graph(variable=_VARIABLE, members=(_DAC2A_MEMBER, _DAC2A_ATTRIBUTED)) + population = destination_population(get_index(), _DAC2A) + for dcid in (*population.members, population.total, _AGENCY, _ALL_RECIPIENTS): + graph._nodes.setdefault(dcid, {"label": dcid, "type": "Organization"}) + return {"graph": graph, "llm": _ScriptedLLM(_VARIABLE), "sdmx": _sdmx(held)} + + +def _india(request: ResolveRequest, held: dict[tuple[str, str, str], int] = _HELD) -> Any: + response = offline_resolve(request, **india_clients(held)) + assert isinstance(response, Resolution), response + (part,) = response.parts + return part + + +def _candidates(request: ResolveRequest | None = None) -> CandidatesPart: + part = _india(india_request() if request is None else request) + assert isinstance(part, CandidatesPart), part + return part + + +def _source(identity: BaseIdentity) -> str | None: + pin = (identity.facets or {}).get("provenance") + return None if pin is None else pin.ref + + +def _own(part: Any, spec: Any) -> dict[str, Any]: + """`_shared_fields` of `spec` in `part`, over the warnings keyed to it alone.""" + return _shared_fields( + SimpleNamespace( + spec=spec, + warnings=[warning for warning in part.warnings if warning.spec_id == spec.spec_id], + departures=[], + ) + ) + + +@pytest.mark.parametrize( + ("entries", "source"), + [ + pytest.param((), _BILATERAL, id="no-entry"), + # The conversation settled total ODA, contributions included, so each reading keeps it. + pytest.param((carried("provider:contributions"),), None, id="carried-total"), + ], +) +def test_each_dac2a_candidate_is_read_within_its_population_and_minted_as_its_composite( + entries: tuple[Entry, ...], source: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + """Each reading elects its provider reading at its own read, under the part's provider + entry, and is read once. The donor reading reads at the population's total and the + recipient reading at India. Each states what a resubmit of its definition states.""" + built: list[str] = [] + build = candidate_grounding.build_spec + + def recorded(**kwargs: Any) -> Any: + spec = build(**kwargs) + built.append(spec.spec_id) + return spec + + monkeypatch.setattr(candidate_grounding, "build_spec", recorded) + reads = _table_reads(monkeypatch) + + part = _candidates(india_request(carried_entries=entries)) + + assert _counted(reads) == [("answer", False), ("candidate", False)] + specs = part.candidates.specs + assert [_source(spec.definition.identity) for spec in specs] == [source, source] + assert all(spec.spec_id in built for spec in specs) + for spec in specs: + again = _resolve_resubmit(spec.definition.identity, held=_HELD) + assert _own(again, again.spec) == _own(part, spec) + + +@pytest.mark.parametrize( + ("entries", "within"), + [ + pytest.param((), True, id="population-composite"), + # The conversation settled total ODA, so the donor reading reads the plain table. + pytest.param((carried("provider:contributions"),), False, id="plain-table"), + ], +) +def test_a_set_whose_elected_reading_s_read_holds_no_row_answers_no_data( + entries: tuple[Entry, ...], within: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """India's own ODA goes to an agency alone, so India reads as the donor and as the + recipient, and the donor reading is elected first. Its read returns no row. Settled on + the bilateral composite, it reads within the population, which holds none. Settled on + the plain table, it reads a table holding no row India gives. Either way the part + answers no_data `no_observations` from that one read and reads no other reading. The + recipient reading is not offered in its place, since a reading without data offers only + rule 1's stand-ins and the riders (`phase_place_materialise._offered`). The coverage + the election reads offers the composite only where it serves a year, and holds the + agency row the plain read finds, so the election is made to offer the composite and + the read to read no row India gives, as a table the snapshot disagrees with would. + + The no_data answer's search for nearest-real suggestions reads relaxed queries of its + own (`grounding._relaxed_binding_suggestion`), which read no reading of the set and are + left out of the count.""" + held = {key: value for key, value in _HELD.items() if key[1] != _IND or key[2] == _AGENCY} + patch_india_coverage(monkeypatch, held) + _offer_the_bilateral_composite(monkeypatch) + read_the_answer_against( + monkeypatch, {key: value for key, value in held.items() if key[1] != _IND} + ) + reads = _table_reads(monkeypatch) + + part = _india(india_request(open_role="recipient", carried_entries=entries), held) + + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "no_observations" + readings = [read for read in _counted(reads) if read[0] != "nearest-real"] + assert readings == [("answer", within)] + + +def refuse_the_recipient_reading_s_mint(monkeypatch: pytest.MonkeyPatch) -> None: + """The mint of the composite at India as the recipient raises, so that reading is dropped + (`candidate_grounding.ground_candidates`) and the donor reading is left alone.""" + mint = resubmit.composite_mint + + def refusing(identity: BaseIdentity, **kwargs: Any) -> Any: + if identity.entities.get("destination") == [_IND]: + raise RuntimeError("mint failed") + return mint(identity, **kwargs) + + monkeypatch.setattr(resubmit, "composite_mint", refusing) + + +def test_a_set_left_with_one_composite_reading_answers_it_as_its_resubmit_does( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """With every recipient open, the donor reading reads India's ODA within the bilateral + population and the recipient reading reads India's receipts. The recipient reading's + composite cannot be minted, so it is dropped with an ERROR, and the set left with the + donor reading answers it definite, as a resubmit of its composite does, with its spec_id + and its scope compiled to the composite and the component reading the population.""" + refuse_the_recipient_reading_s_mint(monkeypatch) + reads = _table_reads(monkeypatch) + survivors: list[Any] = [] + finished = candidate_collapse.finished_scope + + async def judged(*args: Any, reading: Any, **kwargs: Any) -> Any: + decision = await finished(*args, reading=reading, **kwargs) + survivors.append((reading, decision)) + return decision + + monkeypatch.setattr(candidate_collapse, "finished_scope", judged) + results: list[Any] = [] + assemble = candidate_collapse._assemble_definite + + async def assembled(*args: Any, **kwargs: Any) -> Any: + result = await assemble(*args, **kwargs) + results.append(result.scope_decision) + return result + + monkeypatch.setattr(candidate_collapse, "_assemble_definite", assembled) + + with caplog.at_level(logging.ERROR, logger=reading.__name__): + part = _india(india_request(open_role="recipient")) + + assert isinstance(part, DefinitePart), part + # The finished answer is judged by the survivor's own reading + # (`phase_answer.finished_scope`), read within the population. + ((survivor, decision),) = survivors + assert survivor is not None and survivor.table is not None + # The result carries the decision its check returned, which the part states. + assert results == [decision] + assert decision is not None and decision.spec_id == part.spec.spec_id + assert _counted(reads) == [("answer", True), ("candidate", False)] + assert [r.levelno for r in caplog.records if r.name == reading.__name__] == [logging.ERROR] + assert _source(part.spec.definition.identity) == _BILATERAL + again = _resolve_resubmit(part.spec.definition.identity, held=_HELD) + assert again.spec.spec_id == part.spec.spec_id + assert _shared_fields(again) == _shared_fields(part) + choices = part.spec.resolved.scope_choices + assert choices is not None + assert choices.compiled.spec_id == part.spec.spec_id + (component,) = choices.compiled.components or () + population = destination_population(get_index(), _DAC2A) + assert component.identity.entities["destination"] == list(population.members) + + +def test_a_survivor_no_scope_states_fails_closed_launching_no_task( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A survivor whose rows were read within the bilateral population but whose Spec is + minted as the table, under a decision made for the composite, is stated by no scope. It + is judged once, before anything of it launches (`candidate_collapse._assemble_definite`, + `phase_answer.finished_scope`), so it fails closed with no warning about it and no + related read, option or preview of its own, and the related read started for the + elected shape before grounding is cancelled: no_data `facet_not_available` stands in its + place, describing the survivor (`ResolveState.no_data(candidate=)`), and the pass states + its departures, warnings and timing on it.""" + refuse_the_recipient_reading_s_mint(monkeypatch) + decide = candidate_grounding.decide_read_scope + + def as_the_table(*args: Any, **kwargs: Any) -> Any: + scoped = decide(*args, **kwargs) + if scoped.decision is None: + return scoped + return dataclasses.replace(scoped, minted=None) + + monkeypatch.setattr(candidate_grounding, "decide_read_scope", as_the_table) + launched: list[str] = [] + + def recording(name: str, launch: Any) -> Any: + def recorded(*args: Any, **kwargs: Any) -> Any: + launched.append(name) + return launch(*args, **kwargs) + + return recorded + + related = slow_related_reads(monkeypatch) + for name in ( + "start_answer_related_read", + "note_denominator_unverified", + "_unit_omitted_warning", + "related_read_for", + "_attach_candidate_slot_options", + "_make_slot_options_task", + "_relaunch_place_options", + "CompositeRelaunch", + ): + monkeypatch.setattr( + candidate_collapse, name, recording(name, getattr(candidate_collapse, name)) + ) + for name in ("relaunch", "observations"): + monkeypatch.setattr( + candidate_collapse.PreviewReader, + name, + recording(name, getattr(candidate_collapse.PreviewReader, name)), + ) + assembled: list[tuple[Any, RegionResult]] = [] + assemble = candidate_collapse._assemble_definite + + async def capture( + state: Any, specs: list[Any], grounded: Any, rate_cache: Any, **kwargs: Any + ) -> RegionResult: + result = await assemble(state, specs, grounded, rate_cache, **kwargs) + assembled.append((grounded[specs[0].spec_id].candidate, result)) + return result + + monkeypatch.setattr(candidate_collapse, "_assemble_definite", capture) + decorated: list[tuple[str, str | None]] = [] + departures = pipeline.with_requirement_departures + + def decorating(result: RegionResult, *args: Any) -> RegionResult: + decorated.append((result.status, result.no_data_reason)) + return departures(result, *args) + + monkeypatch.setattr(pipeline, "with_requirement_departures", decorating) + + events = offline_events( + india_request(open_role="recipient", observations=True), + stream_slot_options=True, + **india_clients(), + ) + + (response,) = [event.response for event in events if isinstance(event, ResponseEvent)] + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "facet_not_available" + ((survivor, result),) = assembled + assert survivor.reading is not None and survivor.reading.table is not None + assert result.refused_answer == (survivor.shape, tuple(survivor.bindings), survivor.uncovered) + # Only the elected shape's related read started, before grounding, and it was cancelled. + assert launched == ["start_answer_related_read"] + assert [read.task.cancelled() for read in related] == [True] + assert decorated == [("no_data", "facet_not_available")] + + +def test_a_survivor_that_fails_closed_raises_and_judges_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A survivor that fails closed at its check contributes no warning, departure fact or + judgement. Grounding the candidates restates `role_scope_defaulted` to count the + survivor's composite's parties (`candidate_grounding.restate_scope_warning`), and the + survivor's answer raises an omitted currency and an option coverage warning, each with a + departure fact, and is judged (`requirements.settle_answer`). Refused at its check, the + part holds what it had raised and judged before the candidates were grounded + (`ResolveState.mark_answer`, `phase_answer.failed_closed`).""" + refuse_the_recipient_reading_s_mint(monkeypatch) + + async def refused(*_args: Any, **_kwargs: Any) -> None: + return None + + monkeypatch.setattr(candidate_collapse, "finished_scope", refused) + settle = candidate_collapse.settle_answer + + def raising(state: Any, *args: Any, **kwargs: Any) -> Any: + raise_answer_warnings(state) + return settle(state, *args, **kwargs) + + monkeypatch.setattr(candidate_collapse, "settle_answer", raising) + held: dict[str, tuple[Any, ...]] = {} + + def holding(state: Any) -> tuple[Any, ...]: + return (tuple(state.warnings), dict(state.departure_facts), state.judgement) + + ground = candidate_collapse.ground_candidates + + def grounding(state: Any, candidates: Any) -> Any: + held["grounding"] = holding(state) + return ground(state, candidates) + + monkeypatch.setattr(candidate_collapse, "ground_candidates", grounding) + assemble = candidate_collapse._assemble_definite + + async def assembling(state: Any, *args: Any, **kwargs: Any) -> Any: + held["assembly"] = holding(state) + return await assemble(state, *args, **kwargs) + + monkeypatch.setattr(candidate_collapse, "_assemble_definite", assembling) + closed: list[Any] = [] + close = candidate_collapse.failed_closed + + def closing(state: Any, *args: Any, **kwargs: Any) -> Any: + held["check"] = holding(state) + result = close(state, *args, **kwargs) + closed.append((result, state.judgement)) + return result + + monkeypatch.setattr(candidate_collapse, "failed_closed", closing) + + offline_events( + india_request(open_role="recipient", observations=True), + stream_slot_options=True, + **india_clients(), + ) + + ((result, judgement),) = closed + warnings, facts, before = held["grounding"] + assert (result.status, result.no_data_reason) == ("no_data", "facet_not_available") + # Grounding restated the scope warning before the survivor's assembly started. + scoped = [w for w in warnings if w.code == "role_scope_defaulted"] + assert scoped + assert [w for w in held["assembly"][0] if w.code == "role_scope_defaulted"] != scoped + _checked, raised, judged = held["check"] + assert set(raised) - set(facts) + assert judged is not None + assert result.warnings == warnings + assert result.departure_facts == facts + assert judgement is before + + +def test_a_survivor_stated_as_no_composite_reads_as_its_table( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A survivor minted as a composite whose check returns a decision compiling none reads + as the table that decision states, as the answer phase's does + (`phase_answer._finalize_definite_result`): its preview streams from the table's read, + and nothing reads it per component.""" + refuse_the_recipient_reading_s_mint(monkeypatch) + finished = candidate_collapse.finished_scope + + async def as_no_composite(*args: Any, **kwargs: Any) -> Any: + decision = await finished(*args, **kwargs) + assert decision is not None and decision.composite + return dataclasses.replace(decision, composite=False, scope=None) + + monkeypatch.setattr(candidate_collapse, "finished_scope", as_no_composite) + results: list[RegionResult] = [] + assemble = candidate_collapse._assemble_definite + + async def assembling(*args: Any, **kwargs: Any) -> Any: + result = await assemble(*args, **kwargs) + results.append(result) + return result + + monkeypatch.setattr(candidate_collapse, "_assemble_definite", assembling) + + offline_events( + india_request(open_role="recipient", observations=True), + stream_slot_options=True, + **india_clients(), + ) + + (result,) = results + assert result.status == "definite" + assert not isinstance(result.observations_relaunch, candidate_collapse.CompositeRelaunch) + assert result.observations_task is not None + + +def test_each_candidate_is_minted_under_the_request_s_catalogue( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A candidate's composite is minted under the request's catalogue + (`reading.decide_read_scope`), as the definite answer's is, so a source scope its own + reads ran under narrows none of its figures.""" + scopes: list[frozenset[str] | None] = [] + mint = resubmit.composite_mint + + def recording(identity: BaseIdentity, **kwargs: Any) -> Any: + scopes.append(current_catalogue().scope) + return mint(identity, **kwargs) + + monkeypatch.setattr(resubmit, "composite_mint", recording) + decide = candidate_grounding.decide_read_scope + + def narrowed(*args: Any, **kwargs: Any) -> Any: + with scoped_reads(frozenset({_DAC2A})): + return decide(*args, **kwargs) + + monkeypatch.setattr(candidate_grounding, "decide_read_scope", narrowed) + states: list[ResolveState] = [] + assemble = candidate_collapse._assemble_candidate_set + + async def capture(state: Any, specs: list[Any], grounded: Any, truncated: bool) -> Any: + states.append(state) + return await assemble(state, specs, grounded, truncated) + + monkeypatch.setattr(candidate_collapse, "_assemble_candidate_set", capture) + + _candidates() + + (state,) = states + assert state.request_catalogue is not None + assert scopes == [state.request_catalogue.scope] * 2 + + +def test_a_refusal_showing_a_composite_candidate_carries_its_mint_on_its_own( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A candidate a refusal shows as its flagged match hands the composite it is minted as to + the refusal (`candidate_collapse._shown`), whose whole-total warnings read the identity + its rows were read for, and leaves the part's own answer's composite + (`ResolveState.minted`) unchanged.""" + seen: list[tuple[Any, Any, list[Any]]] = [] + assemble = candidate_collapse._assemble_candidate_set + + async def capture(state: Any, specs: list[Any], grounded: Any, truncated: bool) -> Any: + seen.append((state, grounded, specs)) + return await assemble(state, specs, grounded, truncated) + + monkeypatch.setattr(candidate_collapse, "_assemble_candidate_set", capture) + _candidates() + ((state, grounded, specs),) = seen + before = state.minted + spec = specs[0] + minted = grounded[spec.spec_id].minted + assert minted is not None + + matched = candidate_collapse._shown(state, grounded, spec) + + assert state.minted is before + assert matched == (minted,) + assert state.minted_as(spec, matched) is minted + + +def test_a_refusal_offering_a_composite_alternate_carries_its_mint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A refusal whose first nominated reading rides as its alternate match hands the + composite that reading is minted as to the refusal (`ResolveState.no_data` `matched`), + as a flagged match does, so the alternate's whole-total warnings read the identity its + rows were read for.""" + seen: list[tuple[Any, Any, list[Any]]] = [] + assemble = candidate_collapse._assemble_candidate_set + + async def capture(state: Any, specs: list[Any], grounded: Any, truncated: bool) -> Any: + seen.append((state, grounded, specs)) + return await assemble(state, specs, grounded, truncated) + + monkeypatch.setattr(candidate_collapse, "_assemble_candidate_set", capture) + _candidates() + ((state, grounded, specs),) = seen + minted = grounded[specs[0].spec_id].minted + assert minted is not None + # The donor and recipient readings stand in for no elected shape and break no family down. + assert {grounded[spec.spec_id].candidate.basis for spec in specs}.isdisjoint( + {"structural", "breakdown"} + ) + refusals: list[dict[str, Any]] = [] + no_data = ResolveState.no_data + + def recorded(self: ResolveState, reason: Any, **kwargs: Any) -> RegionResult: + refusals.append(kwargs) + return no_data(self, reason, **kwargs) + + monkeypatch.setattr(ResolveState, "no_data", recorded) + # The elected shape holds no data and no stand-in answers for it. + state.elected_has_data = False + + refused = candidate_collapse._handle_sparse_survivors(state, list(specs), grounded, None) + + assert isinstance(refused, RegionResult) + assert refused.alternate_match is specs[0] + (kwargs,) = refusals + assert kwargs.get("matched") == (minted,) + + +def test_each_candidate_carries_its_own_withheld_reading_warning() -> None: + """Two candidates whose readings the bilateral default cannot reach for the same reason + each carry the warning keyed to their own spec_id, though the messages match.""" + specs = [SimpleNamespace(spec_id=spec_id) for spec_id in ("spec_a", "spec_b")] + grounded = { + spec.spec_id: SimpleNamespace( + decision=SimpleNamespace( + warning=_withheld_warning(spec.spec_id, Withheld(why="option_not_covered")) + ) + ) + for spec in specs + } + + found = candidate_collapse._decision_warnings((), cast(Any, grounded), cast(Any, specs)) + + assert [(w.spec_id, w.message) for w in found] == [ + ("spec_a", found[0].message), + ("spec_b", found[0].message), + ] + + +def test_a_refusal_s_composite_readings_leave_the_pipeline_as_minted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A refusal whose flagged match and nearest real suggestion were minted as the bilateral + composite at their own builds leaves the part's scope step unchanged. Each reading was + elected at its read, so none is read again.""" + _patched_coverage(monkeypatch, _FRANCE_HELD) + composite = _resolve_resubmit(_identity(_BILATERAL)).spec + near = NearestRealSpec.model_validate( + { + **composite.model_dump(), + "relaxation": Relaxation( + axis="what", property_dcid="measurementDenominator", dropped_value_dcids=["x"] + ), + } + ) + refused = RegionResult( + variable_text=_VARIABLE, + status="no_data", + specs=(), + no_data_reason="no_observations", + warnings=(), + timing_by_step={}, + flagged_match=(composite,), + nearest_real=(near,), + ) + + stated = pipeline._with_scope_choices(refused) + + assert stated is refused diff --git a/tests/engine/test_dac2a_destination_populations.py b/tests/engine/test_dac2a_destination_populations.py index 9500553e7..e27256c8e 100644 --- a/tests/engine/test_dac2a_destination_populations.py +++ b/tests/engine/test_dac2a_destination_populations.py @@ -10,12 +10,9 @@ from __future__ import annotations import asyncio -import dataclasses import logging -import re -import threading -import time from collections.abc import Callable +from dataclasses import dataclass from typing import Any import pytest @@ -25,31 +22,21 @@ from code_index.declarations import DAC2A_INCLUSIVE from code_index.shared import get_index from dc_sdmx import SdmxClient as MaterialiseSdmxClient -from origin_catalogue import DFO, current_catalogue +from origin_catalogue import DFO from search.engine.clients.graph import Facet -from search.engine.concurrency import scoped_concurrency -from search.engine.core import ResolveRecord -from search.engine.degraded_reads import note_degraded_read, recording_degraded_reads -from search.engine.departures import measure_not_applied, warning_key -from search.engine.entry import dataset_resubmit -from search.engine.errors import EngineInputError from search.engine.events import ObservationsEvent from search.engine.quantity import NamedDenominator from search.engine.region import ( RegionResult, - composite_answer, + candidates, phase_answer, phase_place_materialise, pipeline, - result, + promote_materialise, scope_decision, ) -from search.engine.region.composite_answer import ( - ComponentReads, - as_merged_composite, - population_table_answer, -) -from search.engine.region.grounding import role_scope_defaulted_warning +from search.engine.region.state import ResolveState +from search.engine.retrieval import role_resolve from search.models import ( BaseIdentity, DatasetResubmitInput, @@ -60,6 +47,7 @@ ParsedEntity, ParsedInput, ParsedPart, + Relaxation, Resolution, ResolveOptions, ResolveRequest, @@ -67,9 +55,10 @@ Spec, Warning, ) +from search.render.heading import LATEST_YEAR_LABEL +from search.render.plan import render_sentence_plan from search.scope import composite as scope_composite from search.scope import provider -from search.scope.state import ScopeState from search.scope.tables import ScopeFacts from series_coverage.index import CoverageIndex from series_coverage.schema import FacetMetadata, RoleNames @@ -232,6 +221,58 @@ def _resubmit(identity: BaseIdentity, *, observations: bool = False) -> ResolveR ) +@dataclass(frozen=True) +class _TableRead: + """One read of a part's table: the step that made it (`answer` for a fresh part's read, + `resubmit` for a posted identity's, `candidate` for a reading beside the answer, + `nearest-real` for a suggestion's relaxed query), whether an open axis was discovered + within a population's members, and the recipients the read binds. A nearest-real read + names its places as subjects, not by role (`role_resolve.graph_confirm_resolve`), so its + recipients are None.""" + + step: str + within: bool + recipients: tuple[str, ...] | None + + +# Every function a part's table is read through, by the step that calls it. A nearest-real +# suggestion's relaxed query reads through `graph_confirm_resolve` by its module attribute +# (`grounding._relaxed_binding_suggestion`), which no other step reads it through. +_READ_SITES = { + "answer": (phase_place_materialise, "run_materialise"), + "resubmit": (promote_materialise, "run_materialise"), + "candidate": (candidates, "materialise"), + "nearest-real": (role_resolve, "graph_confirm_resolve"), +} + + +def _table_reads(monkeypatch: pytest.MonkeyPatch) -> list[_TableRead]: + """Every read of a part's table from here on, in order, recorded where each step calls + the read (`_READ_SITES`).""" + reads: list[_TableRead] = [] + + def counting(step: str, read: Callable[..., Any]) -> Callable[..., Any]: + def counted(*args: Any, within: Any, **kwargs: Any) -> Any: + # The role-keyed entities, the third positional argument of every read but the + # nearest-real one, which names its places as subject dcids. + recipients = ( + None if step == "nearest-real" else tuple(args[2].get("financingRecipient", ())) + ) + reads.append(_TableRead(step, bool(within), recipients)) + return read(*args, within=within, **kwargs) + + return counted + + for step, (module, name) in _READ_SITES.items(): + monkeypatch.setattr(module, name, counting(step, getattr(module, name))) + return reads + + +def _counted(reads: list[_TableRead]) -> list[tuple[str, bool]]: + """Each read's step and whether it was made within a population.""" + return [(read.step, read.within) for read in reads] + + def _part(response: Any) -> DefinitePart: assert isinstance(response, Resolution), response (part,) = response.parts @@ -239,6 +280,30 @@ def _part(response: Any) -> DefinitePart: return part +def _ended_state( + monkeypatch: pytest.MonkeyPatch, scenario: str, *, sdmx: FakeSdmxClient | None = None +) -> ResolveState: + """The state a fresh answer of `scenario` ends its pass with, read over `sdmx`, else the + test's held rows (`_sdmx`): its shape, bindings and place, and the read that elected its + reading.""" + captured: list[ResolveState] = [] + preview = pipeline._with_composite_preview + + async def capture(result: RegionResult, state: ResolveState) -> RegionResult: + captured.append(state) + return await preview(result, state) + + monkeypatch.setattr(pipeline, "_with_composite_preview", capture) + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx() if sdmx is None else sdmx, + ) + (state,) = captured + return state + + def _resolve_resubmit( identity: BaseIdentity, *, held: dict[tuple[str, str, str], int] = _HELD ) -> DefinitePart: @@ -309,7 +374,11 @@ def test_a_composite_preview_holds_the_country_row_and_inclusive_adds_106( def _fresh_request( - entities: list[tuple[str, str]], *, message: str, open_role: str | None + entities: list[tuple[str, str]], + *, + message: str, + open_role: str | None, + time_text: str = "2019-2021", ) -> ResolveRequest: return ResolveRequest( input=ParsedInput( @@ -318,7 +387,7 @@ def _fresh_request( variable_text=_VARIABLE, entities=[ParsedEntity(text=text, role_text=role) for role, text in entities], open_roles=[open_role] if open_role else [], - time_text="2019-2021", + time_text=time_text, ) ], user_message=message, @@ -348,6 +417,13 @@ def _fresh_request( message="official development assistance from all donors", open_role="recipient", ), + # H1 asking for the latest year. + "H1-latest": lambda: _fresh_request( + [_FRANCE], + message="official development assistance from France", + open_role="recipient", + time_text="latest", + ), # A part naming no destination, read over `_AT_TOTAL`: France's ODA, and the top donors. "D2": lambda: _fresh_request( [_FRANCE], message="official development assistance from France", open_role=None @@ -358,14 +434,19 @@ def _fresh_request( } -def _answer(spec: Spec, part: DefinitePart) -> dict[str, Any]: +def _answer( + spec: Spec, part: DefinitePart, *, words: frozenset[str] = frozenset() +) -> dict[str, Any]: + """What a fresh answer and a resubmit of its definition both state, without the warnings + whose codes `words` names, which the request's own words raise.""" resolved = spec.resolved dump = lambda model: None if model is None else model.model_dump(mode="json") # noqa: E731 return { "scope": dump(resolved.scope), "headline": dump(resolved.headline), "coverage": dump(resolved.coverage), - "warnings": sorted((w.code, w.message) for w in part.warnings), + "entities": [dump(entity) for entity in resolved.entities], + "warnings": sorted((w.code, w.message) for w in part.warnings if w.code not in words), } @@ -400,74 +481,56 @@ def test_a_fresh_answer_equals_a_resubmit_of_its_definition( assert _answer(again.spec, again) == _answer(fresh.spec, fresh) -# What a fresh part holds from the request's words or its door, which a resubmit of its -# definition holds none of: the parsed part it restates, the places each text bound, the -# scope its words and binders stated, and its timings. Departures quote the request's words. -_WORD_FIELDS = frozenset({"restatement", "entity_refs", "timing_ms"}) - - -def _read_fields(part: DefinitePart) -> dict[str, Any]: - """`part`'s dump without the fields its request's words decide, its departures without - the words they quote.""" - dump = part.model_dump(mode="json", exclude=set(_WORD_FIELDS)) - dump["spec"]["resolved"].pop("scope_choices") - dump["departures"] = [ - {key: value for key, value in departure.items() if key not in ("asked", "text")} - for departure in dump["departures"] - ] - return dump - - -# What a fresh region holds from the request's words or its door, which a resubmit region of -# its definition holds otherwise: the part's words and timings, the phrases and places its -# words bound, the scope its binders stated, the relaunch facts and deadline of the door, -# and the closure reading the composite's preview over the door's clients. -_WORD_REGION_FIELDS = frozenset( - { - "variable_text", - "timing_by_step", - "admitted_qualifier_phrases", - "scope_choices", - "entity_words", - "entity_refs", - "promote_relaunch", - "reads_deadline", - "observations_relaunch", +def _shared_fields(part: DefinitePart) -> dict[str, Any]: + """What a fresh part and a resubmit of its definition must both state: its definition + and spec_id, its scope, entities, headline, coverage, StatVar counts, measure label, + provenance facet, data years and download, its warnings, and its departures without the + words they quote.""" + spec = part.spec + resolved = spec.resolved + dump = lambda model: None if model is None else model.model_dump(mode="json") # noqa: E731 + return { + "spec_id": spec.spec_id, + "definition": dump(spec.definition), + "scope": dump(resolved.scope), + "entities": [dump(entity) for entity in resolved.entities], + "headline": dump(resolved.headline), + "coverage": dump(resolved.coverage), + "stat_vars": dump(resolved.stat_vars), + "measure_label": resolved.measure_label, + "provenance": dump(resolved.facets.get("provenance")), + "data_years": resolved.data_years, + "download": dump(resolved.download), + "warnings": [dump(warning) for warning in part.warnings], + "departures": [ + {key: value for key, value in dump(departure).items() if key not in ("asked", "text")} + for departure in part.departures + ], } -) -def _region_fields(region: RegionResult) -> dict[str, str]: - """Each field of `region` the request's words do not decide, as its text with object - addresses left out, so two closures over the same code compare equal.""" - return { - field.name: re.sub(r" at 0x[0-9a-f]+", "", repr(getattr(region, field.name))) - for field in dataclasses.fields(region) - if field.name not in _WORD_REGION_FIELDS - } +# The headline fields that say whether the request named the years it reads, which the door +# decides. A fresh `latest` request names no year, and a resubmit posts the year elected. +_DOOR_FACTS = ("named_window", "year_named") -@pytest.mark.parametrize("scenario", ["H1", "H2", "H6"]) +@pytest.mark.parametrize("scenario", ["H1", "H2", "H6", "H5", "H1-latest"]) def test_a_fresh_composite_answer_reads_every_field_a_resubmit_reads( scenario: str, monkeypatch: pytest.MonkeyPatch ) -> None: - """Every field of a fresh composite answer the rows it read decide, in its region and in - its part, its Spec whole, its warnings and its departures, is a resubmit's of its - definition: the fresh answer is that read with the request's words added, so no read - field keeps the whole table's.""" - regions: list[RegionResult] = [] - scope_choices, as_composite = pipeline._with_scope_choices, dataset_resubmit._as_composite - - def fresh_region(result: RegionResult, state: Any) -> RegionResult: - regions.append(scope_choices(result, state)) - return regions[-1] - - def resubmit_region(region: RegionResult, identity: BaseIdentity, **kwargs: Any) -> Any: - regions.append(as_composite(region, identity, **kwargs)) - return regions[-1] - - monkeypatch.setattr(pipeline, "_with_scope_choices", fresh_region) - monkeypatch.setattr(dataset_resubmit, "_as_composite", resubmit_region) + """Every field of a fresh composite answer the rows it read decide is a resubmit's of + its definition (`_shared_fields`), as for the plain table answer H5, the control. + + The fields the two doors state differently for the plain table too are left out: the + trace's `query_qualifiers`, `slot_filters` provenance and `default_reading`, which the + request's words decide, `available_at_place`, `inapplicable_slots`, and the region's + `observation_entities`, `served_members`, `preview_read` and `served_filter`, which + each door builds from its own read. + + The headline's `named_window` and `year_named` are door facts too (`_DOOR_FACTS`). A + fresh `latest` request names no year, and a resubmit's identity carries the year its + definition elected, so it names that year. Its figures and window of data are the + same.""" fresh = _part( offline_resolve( _SCENARIOS[scenario](), @@ -479,9 +542,52 @@ def resubmit_region(region: RegionResult, identity: BaseIdentity, **kwargs: Any) again = _resolve_resubmit(fresh.spec.definition.identity) - assert _read_fields(again) == _read_fields(fresh) - fresh_read, resubmitted = regions - assert _region_fields(resubmitted) == _region_fields(fresh_read) + shared, resubmitted = _shared_fields(fresh), _shared_fields(again) + if scenario == "H1-latest": + named = { + door: {key: shared["headline"].pop(key) for key in _DOOR_FACTS} + for door, shared in (("fresh", shared), ("resubmit", resubmitted)) + } + year = fresh.spec.definition.identity.requested_window.end_year + assert named == { + "fresh": {"named_window": None, "year_named": False}, + "resubmit": { + "named_window": {"start_year": year, "end_year": year}, + "year_named": True, + }, + } + assert fresh.spec.resolved.trace.requested_window == "latest" + # The fresh byline states the latest-year label, which the resubmit's lacks (apis#312). + assert any(token.text == LATEST_YEAR_LABEL for token in render_sentence_plan(fresh.spec)) + assert resubmitted == shared + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason="DevelopmentFinanceObservatory/apis#312: a composite identity keeps no latest window", +) +def test_a_latest_composite_answer_states_its_label_at_both_doors() -> None: + """A `latest` request answered as the bilateral composite states "Latest year with data" + in its byline, fresh and resubmitted. The composite's identity holds the year its lines + elect in place of `latest` (`provider.elect_latest_window`), since the Datasets mint + refuses a composite's `latest` window, so only the fresh answer's trace says the request + asked for the latest year (`heading._asks_latest`), and the resubmit states no label.""" + fresh = _part( + offline_resolve( + _SCENARIOS["H1-latest"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + again = _resolve_resubmit(fresh.spec.definition.identity) + + labelled = { + door: any(token.text == LATEST_YEAR_LABEL for token in render_sentence_plan(spec)) + for door, spec in (("fresh", fresh.spec), ("resubmit", again.spec)) + } + assert labelled == {"fresh": True, "resubmit": True} @pytest.mark.parametrize( @@ -555,168 +661,6 @@ def test_a_fresh_bilateral_reading_lists_no_agency_in_its_scope(scenario: str) - assert not {member.ref for member in scope.members} & _SPURIOUS -def _reads(coverage: CoverageIndex) -> ComponentReads: - """The fixture's clients and coverage, as a part's component reads hold the request's.""" - return ComponentReads( - graph=_graph_with_parties(), - sdmx=_sdmx(), - coverage=FakeCoverageService(coverage).snapshot(), - materialise_graph=None, - materialise_sdmx=None, - rate_cache=None, - deadline=None, - catalogue=current_catalogue(), - ) - - -def _in_request_scope[T](call: Callable[[], T]) -> T: - """`call`'s result, called inside a request's concurrency scope - (`concurrency.scoped_concurrency`), as a direct call below `core.resolve_events` needs.""" - - async def scoped() -> T: - async with scoped_concurrency(): - return call() - - return asyncio.run(scoped()) - - -def _definite(spec: Spec) -> RegionResult: - return RegionResult( - variable_text=_VARIABLE, - status="definite", - specs=(spec,), - no_data_reason=None, - warnings=(), - timing_by_step={}, - ) - - -def test_a_merged_composite_reads_its_table_within_the_population( - _coverage: CoverageIndex, -) -> None: - """A same-shape merge of bilateral composite answers rebuilds its table Spec from its - members' slots and plans; the composite it is restated as reads that table within the - population, as a resubmit of the merged composite does, so a merged table Spec counting - the agency leaves no trace. Two donors never fold, since a donor is no slot - (`same_shape._resolve_diff_key`), so the merge is driven with a merged table Spec read - over the whole table.""" - whole = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec - scope = whole.resolved.scope - assert scope is not None and {member.ref for member in scope.members} >= _SPURIOUS - member = _resolve_resubmit(_identity(_BILATERAL)).spec - - merged = as_merged_composite(_definite(whole), [_definite(member)], reads=_reads(_coverage)) - - assert merged.spec.spec_id == member.spec_id - resolved, again = merged.spec.resolved, member.resolved - assert (resolved.scope, resolved.headline, resolved.coverage) == ( - again.scope, - again.headline, - again.coverage, - ) - - -def test_a_merged_composite_whose_population_holds_no_row_answers_no_data( - _coverage: CoverageIndex, -) -> None: - """A same-shape merge whose merged table read holds no row within the population - answers that read's no_data, as a reading the read drops does, never the whole table's - read restated as the composite: Germany's total ODA in the window went to the agency - alone.""" - table = _identity(_DAC2A).model_copy(update={"facets": None}) - whole = _resolve_resubmit(table.model_copy(update={"entities": {"origin": [_DEU]}})).spec - member = _resolve_resubmit(_identity(_BILATERAL)).spec - - merged = as_merged_composite(_definite(whole), [_definite(member)], reads=_reads(_coverage)) - - assert (merged.status, merged.specs, merged.no_data_reason) == ( - "no_data", - (), - "no_observations", - ) - - -def test_a_candidate_restated_as_the_bilateral_composite_lists_no_agency_in_its_scope( - _coverage: CoverageIndex, -) -> None: - """A candidate set's DAC2a total ODA reading, resolved over every party, is read again - within the bilateral population before it is restated as the composite, as a fresh - definite answer is, so its scope and headline count the country alone.""" - table = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec - scope = table.resolved.scope - assert scope is not None and {member.ref for member in scope.members} >= _SPURIOUS - reads = _reads(_coverage) - candidates = RegionResult( - variable_text=_VARIABLE, - status="candidates", - specs=(table,), - no_data_reason=None, - warnings=(), - timing_by_step={}, - ) - - (restated,) = _readings(candidates, reads=reads, selection=ScopeState()).specs - - pin = (restated.definition.identity.facets or {}).get("provenance") - assert isinstance(pin, IdentityBindingValue) and pin.ref == _BILATERAL - scope = restated.resolved.scope - assert scope is not None - assert {member.ref for member in scope.members} == {_ETH} - headline = restated.resolved.headline - assert headline is not None and headline.role_counts["financingRecipient"] == 1 - - -@pytest.mark.parametrize("shown", ["candidate", "alternate"]) -def test_a_reading_s_population_read_keys_its_warnings_to_the_reading( - _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch, shown: str -) -> None: - """A reading pinning DAC2a's provenance is read within the bilateral population under the - composite's table identity, whose spec_id is not the reading's. Every warning that read - raises, keyed to its own Spec or to none, follows the reading to the composite it is - restated as, so none names a Spec the response does not hold, whether the reading is the - set's own candidate or the alternate a refusal offers.""" - table = _resolve_resubmit(_identity(_DAC2A)).spec - population_read = composite_answer.population_table_answer - - def keyed(spec: Spec, identity: BaseIdentity, *, reads: ComponentReads) -> RegionResult: - read = population_read(spec, identity, reads=reads) - assert read is not None and read.status == "definite" - assert read.spec.spec_id != spec.spec_id - own = Warning( - code="date_coverage_partial", - severity="info", - message="the read's", - spec_id=read.spec.spec_id, - ) - return dataclasses.replace(read, warnings=(*read.warnings, own)) - - monkeypatch.setattr(composite_answer, "population_table_answer", keyed) - region = ( - dataclasses.replace(_definite(table), status="candidates") - if shown == "candidate" - else RegionResult( - variable_text=_VARIABLE, - status="no_data", - specs=(), - no_data_reason="no_observations", - warnings=(), - timing_by_step={}, - alternate_match=table, - ) - ) - - read = _readings(region, reads=_reads(_coverage), selection=ScopeState()) - - (restated,) = (*read.specs, *([read.alternate_match] if read.alternate_match else ())) - assert (restated.definition.identity.facets or {}).get("provenance") == ( - IdentityBindingValue(ref=_BILATERAL) - ) - assert [w.spec_id for w in read.warnings if w.code == "date_coverage_partial"] == [ - restated.spec_id - ] - assert {w.spec_id for w in read.warnings} <= {None, restated.spec_id} - - def test_a_composite_whose_population_holds_no_row_answers_no_data_with_the_request_s_words( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -744,75 +688,43 @@ def test_a_composite_whose_population_holds_no_row_answers_no_data_with_the_requ @pytest.mark.parametrize( - ("error", "reason", "degraded"), + ("scenario", "within"), [ - pytest.param( - EngineInputError( - "no rows", - code="facet_not_available", - status=422, - pointer="/input/identity/entities/destination", - ), - "facet_not_available", - False, - id="refused", - ), - pytest.param( - dc_graph.GraphDeadlineExceededError("deadline"), "timed_out", True, id="deadline" - ), - pytest.param(dc_graph.GraphReadError("lost"), "no_observations", True, id="lost-read"), + pytest.param("H1", True, id="within-the-population"), + # The plain table answer, the control. + pytest.param("H5", False, id="whole-table"), ], ) -def test_a_population_read_that_raises_answers_no_data_with_the_request_s_words( - error: Exception, reason: str, degraded: bool, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + "error", + [ + pytest.param(dc_graph.GraphDeadlineExceededError("deadline"), id="deadline"), + pytest.param(dc_graph.GraphReadError("lost"), id="lost-read"), + ], +) +def test_a_read_within_the_population_that_raises_is_handled_as_any_table_read( + scenario: str, within: bool, error: Exception, monkeypatch: pytest.MonkeyPatch ) -> None: - """A fresh answer whose population read raises answers no_data with the read's reason - under the request's own words, degraded where the read was lost: never the whole - table's read of every party, and never a refusal pointing at an identity the request - did not post.""" + """A part's one read, made within the composite's population, that crosses its deadline + or loses its read raises the error a whole-table read raises, for the request's door to + answer as it answers any lost read.""" + read_within: list[bool] = [] - def raising(**_kwargs: Any) -> RegionResult: + def raising(*_args: Any, within: Any, **_kwargs: Any) -> Any: + read_within.append(bool(within)) raise error - monkeypatch.setattr(composite_answer, "resolve_spec_resubmit", raising) - record = ResolveRecord() - - response = offline_resolve( - _SCENARIOS["H1"](), - graph=_graph_with_parties(), - llm=_ScriptedLLM(_VARIABLE), - sdmx=_sdmx(), - record=record, - ) - - assert isinstance(response, Resolution), response - (part,) = response.parts - assert isinstance(part, NoDataPart), part - assert (part.variable_text, part.no_data.reason) == (_VARIABLE, reason) - assert record.done is not None and record.done.degraded_leg is degraded - - -def test_a_request_warning_follows_the_population_read_s_spec_id() -> None: - """A warning the pass raised about its whole-table answer before the population read is - keyed to the read's Spec, the one the part answers, so it never names a spec_id the - response does not show.""" - whole = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec - read = _resolve_resubmit( - _identity(_DAC2A).model_copy(update={"facets": None, "entities": {"origin": [_DEU]}}) - ).spec - assert whole.spec_id != read.spec_id - words = Warning( - code="query_qualifier_dropped", severity="info", message="the words", spec_id=whole.spec_id - ) - fresh = dataclasses.replace(_definite(whole), warnings=(words,)) + monkeypatch.setattr(phase_place_materialise, "run_materialise", raising) - grafted = composite_answer.with_population_read( - fresh, _definite(read), request_warnings=(words,) - ) + with pytest.raises(type(error)): + offline_resolve( + _SCENARIOS[scenario](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) - assert [(w.code, w.spec_id) for w in grafted.warnings] == [ - ("query_qualifier_dropped", read.spec_id) - ] + assert read_within == [within] def test_a_composite_answer_keeps_the_warnings_raised_before_its_read( @@ -860,59 +772,6 @@ def available(*args: Any) -> Any: monkeypatch.setattr(provider, "_bilateral_option", available) -def test_a_candidate_whose_population_holds_no_row_is_dropped_not_restated( - _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch -) -> None: - """A candidate set's reading settled on the bilateral composite whose population holds no - row in the window is dropped, never restated over the whole table's read of the agency, - and a set left with none refuses with the reason that read gives.""" - _offer_the_bilateral_composite(monkeypatch) - table = _resolve_resubmit( - _identity(_DAC2A).model_copy(update={"facets": None, "entities": {"origin": [_DEU]}}) - ).spec - candidates = dataclasses.replace(_definite(table), status="candidates") - - read = _readings(candidates, reads=_reads(_coverage), selection=ScopeState()) - - assert (read.status, read.specs, read.no_data_reason) == ("no_data", (), "no_observations") - - -@pytest.mark.parametrize("with_france", [False, True], ids=["none-left", "one-left"]) -def test_a_dropped_reading_takes_its_warnings_and_departures_with_it( - _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch, with_france: bool -) -> None: - """A candidate whose population holds no row is dropped with the warnings keyed to it - and the departure facts they disclose, whether the set refuses or is left with one - reading.""" - _offer_the_bilateral_composite(monkeypatch) - table = _identity(_DAC2A).model_copy(update={"facets": None}) - germany = _resolve_resubmit(table.model_copy(update={"entities": {"origin": [_DEU]}})).spec - france = _resolve_resubmit(table).spec - departed = Warning( - code="query_qualifier_dropped", - severity="degraded", - message="Germany's words", - spec_id=germany.spec_id, - ) - candidates = dataclasses.replace( - _definite(germany), - status="candidates", - specs=(france, germany) if with_france else (germany,), - warnings=(departed,), - departure_facts={ - warning_key(departed): measure_not_applied(asked="words", property_dcid=None) - }, - ) - - read = _in_request_scope( - lambda: _readings(candidates, reads=_reads(_coverage), selection=ScopeState()) - ) - - assert read.status == ("definite" if with_france else "no_data") - assert departed.code not in {w.code for w in read.warnings} - assert warning_key(departed) not in read.departure_facts - - def test_a_composite_answer_keeps_a_denominator_its_words_name_unverified( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -977,41 +836,6 @@ def test_an_inclusive_answer_counts_the_recipients_of_both_lines( assert "both recipients" in defaulted -def test_a_composite_recounts_the_defaulted_scope_warning_on_its_axis_whatever_it_says( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A table answer's `role_scope_defaulted` warning names its axis in `params`, so the - composite it is restated as states its own count on that axis whatever words the table's - warning holds, here a label the request spelled its own way; a warning on another axis - stands.""" - _patched_coverage(monkeypatch, _IMPUTED_ONLY) - composite = _resolve_fresh("H2", held=_IMPUTED_ONLY).spec - facts = [ - fact.model_copy(update={"member_count": 1}) - for fact in composite.resolved.role_axis_facts or () - ] - table = composite.model_copy( - update={ - "spec_id": "table", - "resolved": composite.resolved.model_copy(update={"role_axis_facts": facts}), - } - ) - recipients = role_scope_defaulted_warning( - "destination", - "No recipient was named, so the answer covers the one recipient with data from FR.", - ).model_copy(update={"spec_id": "table"}) - donors = role_scope_defaulted_warning("origin", "No provider was named.").model_copy( - update={"spec_id": "table"} - ) - - recounted = result.recounted_scope((recipients, donors), table, composite, own=False) - - assert [w.message for w in recounted] == [ - "No recipient was named, so the answer covers both recipients with data from France.", - donors.message, - ] - - def test_an_inclusive_answer_lists_the_parties_its_role_axis_fact_counts( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1069,15 +893,12 @@ def test_an_inclusive_answer_whose_component_parties_cannot_be_listed_says_so( inclusive answer keeps the open axes of its table read, which count the bilateral line's recipients alone, and logs a WARNING naming the composite and the component.""" _patched_coverage(monkeypatch, _IMPUTED_ONLY) - open_axes = scope_composite._open_axes + lists_parties = scope_composite._lists_parties - def unsettled(stated: Any, served: Any, declaration: Any, *, facts: Any) -> Any: - *listed, last = served - return open_axes( - stated, [*listed, dataclasses.replace(last, members=None)], declaration, facts=facts - ) + def unsettled(item: scope_composite.ServedComponent, coverage: CoverageIndex) -> bool: + return item.component.role != "imputed" and lists_parties(item, coverage) - monkeypatch.setattr(scope_composite, "_open_axes", unsettled) + monkeypatch.setattr(scope_composite, "_lists_parties", unsettled) with caplog.at_level(logging.WARNING, logger=scope_composite.__name__): part = _resolve_fresh("H2", held=_IMPUTED_ONLY) @@ -1088,106 +909,6 @@ def unsettled(stated: Any, served: Any, declaration: Any, *, facts: Any) -> Any: assert (record.levelno, record.args[0]) == (logging.WARNING, DAC2A_INCLUSIVE) -def test_a_candidate_set_left_with_one_reading_answers_as_its_resubmit( - _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch -) -> None: - """A candidate set whose other reading holds no row within its population is left with - one reading, so it answers that reading definite, as a resubmit of its composite does, - and not as a set of one, stating the scope a definite answer states: compiled to the - composite, with its components.""" - _offer_the_bilateral_composite(monkeypatch) - table = _identity(_DAC2A).model_copy(update={"facets": None}) - france = _resolve_resubmit(table).spec - germany = _resolve_resubmit(table.model_copy(update={"entities": {"origin": [_DEU]}})).spec - candidates = dataclasses.replace( - _definite(france), status="candidates", specs=(france, germany) - ) - - read = _in_request_scope( - lambda: _readings(candidates, reads=_reads(_coverage), selection=ScopeState()) - ) - - again = _resolve_resubmit(_identity(_BILATERAL)).spec - assert (read.status, [spec.spec_id for spec in read.specs]) == ("definite", [again.spec_id]) - assert (read.spec.resolved.scope, read.spec.resolved.headline) == ( - again.resolved.scope, - again.resolved.headline, - ) - assert read.scope_choices is not None - assert read.scope_choices.compiled.spec_id == again.spec_id - assert read.scope_choices.compiled.components - - -def test_population_reads_take_the_request_s_limit_and_report_their_lost_reads( - _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch, search_settings: Any -) -> None: - """Two or more population reads run in worker threads, and two parts' reads together run - at most the request's concurrency limit at once. A read one of them loses reaches the - request's record, so no cache stores the answer.""" - search_settings(max_variables=2, max_variable_concurrency=2) - lock, running, peak = threading.Lock(), [0], [0] - - def reading(spec: Any, identity: Any, *, reads: Any) -> None: - with lock: - running[0] += 1 - peak[0] = max(peak[0], running[0]) - time.sleep(0.05) - note_degraded_read(f"read of {spec}") - with lock: - running[0] -= 1 - - monkeypatch.setattr(composite_answer, "population_table_answer", reading) - parts: list[Any] = [ - {name: (name, None) for name in (f"{part}-first", f"{part}-second")} for part in ("a", "b") - ] - reads = _reads(_coverage) - - async def read() -> tuple[list[dict[str, Any]], list[str]]: - async with scoped_concurrency(): - with recording_degraded_reads() as record: - answered = await asyncio.gather( - *( - asyncio.to_thread(composite_answer._population_reads, part, reads=reads) - for part in parts - ) - ) - return answered, record.reads - - answered, lost = asyncio.run(read()) - - assert answered == [dict.fromkeys(part) for part in parts] - assert sorted(lost) == sorted(f"read of {name}" for part in parts for name in part) - assert peak[0] == 2 - - -def test_a_composite_naming_no_destination_is_read_again_at_its_population_total( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A composite whose identity names no destination binds the population's total in its - place (`RolePopulation.total`), which a whole-table answer of the same words never read, - so a fresh answer settled on it is read again at that total, as a resubmit of it reads - it.""" - assert destination_population(get_index(), _DAC2A).total == _DEVELOPING - reads = dataclasses.replace( - _reads(_patched_coverage(monkeypatch, _AT_TOTAL)), sdmx=_sdmx(_AT_TOTAL) - ) - composite = _identity(_BILATERAL).model_copy(update={"open_roles": []}) - table = _resolve_resubmit( - _identity(_DAC2A).model_copy(update={"facets": None}), held=_AT_TOTAL - ).spec - - read = population_table_answer(table, composite, reads=reads) - - assert read is not None and read.status == "definite" - assert read.spec.definition.identity.entities["destination"] == [_DEVELOPING] - resolved = read.spec.resolved - assert resolved.headline is not None - assert (resolved.headline.series, resolved.headline.role_counts["financingRecipient"]) == ( - 1, - 1, - ) - - @pytest.mark.parametrize( ("scenario", "axes"), [ @@ -1208,11 +929,14 @@ def test_a_fresh_answer_naming_no_destination_states_the_scope_of_its_own_identi """A fresh answer naming no destination reads the bilateral composite at the population's total, and that read never changes the part's identity or the scope decided on it: the answer and its scope's base name no destination, and France's part keeps its provider - switch to total ODA, contributions included.""" + switch to total ODA, contributions included. The one read names the population's + total, `DACDevelopingCountries`, and the composite identity names none.""" _patched_coverage(monkeypatch, _AT_TOTAL) + recorded = _table_reads(monkeypatch) part = _resolve_fresh(scenario, held=_AT_TOTAL) + assert recorded == [_TableRead("answer", False, (_DEVELOPING,))] identity = part.spec.definition.identity assert "destination" not in identity.entities assert (identity.facets or {}).get("provenance") == IdentityBindingValue(ref=_BILATERAL) @@ -1229,24 +953,6 @@ def test_a_fresh_answer_naming_no_destination_states_the_scope_of_its_own_identi assert ("provider:contributions" in offered) == (scenario == "D2") -def test_a_population_read_the_composite_refuses_answers_no_data( - _coverage: CoverageIndex, monkeypatch: pytest.MonkeyPatch -) -> None: - """A composite that refuses the read of its population answers no_data, so the caller - never states the whole table's read of every party under the composite's spec_id.""" - - def refusing(*_args: Any, **_kwargs: Any) -> Any: - raise EngineInputError("refused", code="requested_option_unavailable", status=422) - - monkeypatch.setattr(composite_answer, "composite_table_identity", refusing) - table = _resolve_resubmit(_identity(_DAC2A).model_copy(update={"facets": None})).spec - - read = population_table_answer(table, _identity(_BILATERAL), reads=_reads(_coverage)) - - assert read is not None - assert (read.status, read.specs, read.no_data_reason) == ("no_data", (), "facet_not_available") - - def test_a_fresh_bilateral_answer_s_component_carries_the_bound_population() -> None: """The compiled scope's component identity is the one the mint reads: its destination bound to the population's members, the list a client posting the component reads.""" @@ -1294,16 +1000,21 @@ def _years_short(monkeypatch: pytest.MonkeyPatch) -> None: ) -def _slot_refused(monkeypatch: pytest.MonkeyPatch) -> None: - """The bilateral composite refuses what the part binds at its aid type slot.""" - served_components = provider.served_components +def _composite_refuses(pointer: str) -> Callable[[pytest.MonkeyPatch], None]: + """A drift under which the bilateral composite refuses what the part binds at + `pointer`.""" - def refusing(document: dict[str, Any], *, facts: Any) -> Any: - if document.get("facets", {}).get("provenance", {}).get("ref") == _BILATERAL: - raise CompositeNotServable("refused", ref=_BILATERAL, pointer="/slots/0") - return served_components(document, facts=facts) + def drift(monkeypatch: pytest.MonkeyPatch) -> None: + served_components = provider.served_components - monkeypatch.setattr(provider, "served_components", refusing) + def refusing(document: dict[str, Any], *, facts: Any) -> Any: + if document.get("facets", {}).get("provenance", {}).get("ref") == _BILATERAL: + raise CompositeNotServable("refused", ref=_BILATERAL, pointer=pointer) + return served_components(document, facts=facts) + + monkeypatch.setattr(provider, "served_components", refusing) + + return drift def _decision_raises(monkeypatch: pytest.MonkeyPatch) -> None: @@ -1316,28 +1027,40 @@ def refusing(*_args: Any, **_kwargs: Any) -> Any: @pytest.mark.parametrize( - ("drift", "why"), + ("drift", "params"), [ pytest.param( lambda monkeypatch: _patched_coverage( monkeypatch, {**_HELD, (_DAC2A_MEMBER, _OFFICIAL, _UNCLASSIFIED): 50} ), - "destination_unclassified", + {"why": "destination_unclassified"}, id="population-unavailable", ), - pytest.param(_years_short, "option_not_covered", id="years-not-covered"), - pytest.param(_slot_refused, "dac2aAidType", id="slot-refused"), - pytest.param(_decision_raises, "scope_undecided", id="decision-raises"), + pytest.param(_years_short, {"why": "option_not_covered"}, id="years-not-covered"), + pytest.param( + _composite_refuses("/slots/0"), + {"why": "slot_not_served", "slot": "dac2aAidType"}, + id="slot-refused", + ), + pytest.param( + _composite_refuses("/requested_window"), + {"why": "option_not_covered"}, + id="window-refused", + ), + pytest.param(_decision_raises, {"why": "scope_undecided"}, id="decision-raises"), ], ) def test_a_ranking_over_every_donor_says_when_it_cannot_read_bilateral_destinations( - monkeypatch: pytest.MonkeyPatch, drift: Callable[[pytest.MonkeyPatch], Any], why: str + monkeypatch: pytest.MonkeyPatch, + drift: Callable[[pytest.MonkeyPatch], Any], + params: dict[str, str], ) -> None: """A fresh ranking over every donor has no provider axis to state why it reads total ODA, so where the bilateral default cannot apply, for a population no side declaration fully - places, a composite covering fewer years than the answer, a composite refusing a slot - the part binds, or a scope decision that raises, it warns - `bilateral_reading_unavailable` on its own spec_id with the reason.""" + places, a composite covering fewer years than the answer or none the window reaches, a + composite refusing a slot the part binds, or a scope decision that raises, it warns + `bilateral_reading_unavailable` on its own spec_id with the registered reason, and the + slot's property for a slot it cannot serve.""" drift(monkeypatch) part = _part( @@ -1350,7 +1073,7 @@ def test_a_ranking_over_every_donor_says_when_it_cannot_read_bilateral_destinati ) assert (part.spec.definition.identity.facets or {}).get("provenance") is None - assert _withheld(part) == [(part.spec.spec_id, {"why": why})] + assert _withheld(part) == [(part.spec.spec_id, params)] def test_a_part_with_a_provider_axis_states_an_unavailable_population_on_its_scope( @@ -1386,35 +1109,245 @@ def test_a_part_with_a_provider_axis_states_an_unavailable_population_on_its_sco assert (_withheld(donor), _withheld(every_donor)) == ([], []) -def test_a_candidate_over_every_donor_says_when_it_cannot_read_bilateral_destinations( +_TOTAL_YEARS = tuple(range(2017, 2022)) +# What each series holds when the bilateral destinations' rows end a year before the rows +# of the total spanning both sides, and the agency's, so the composite lacks the table's +# newest year. +_COMPOSITE_BEHIND = {key: _TOTAL_YEARS[:-1] if key[2] == _ETH else _TOTAL_YEARS for key in _HELD} +# The reverse. The total spanning both sides ends a year before the destinations, so the +# composite holds every year the table's newest five do. +_COMPOSITE_AHEAD = { + key: _TOTAL_YEARS[:-1] if key[2] in (_AGENCY, _ALL_RECIPIENTS) else _TOTAL_YEARS + for key in _HELD +} + + +def _ragged( + monkeypatch: pytest.MonkeyPatch, years: dict[tuple[str, str, str], tuple[int, ...]] +) -> FakeSdmxClient: + """Coverage and an SDMX double where each series holds the years `years` gives it.""" + patch_coverage( + monkeypatch, + [ + (sv, "dac2a", donor, recipient, list(span)) + for (sv, donor, recipient), span in years.items() + ], + role_names=dict.fromkeys((_DAC2A_MEMBER, _DAC2A_ATTRIBUTED), _NAMES), + facet_metadata={"dac2a": FacetMetadata(provenance_id=_DAC2A, unit="USD")}, + ) + observations: dict[str, dict] = {} + for (sv, donor, recipient), span in years.items(): + facet = Facet( + earliest_date=str(span[0]), + latest_date=str(span[-1]), + obs_count=len(span), + dates=tuple(str(y) for y in span), + values=(float(_HELD[sv, donor, recipient]),) * len(span), + provenance_id=_DAC2A, + unit="USD", + facet_id="dac2a", + ) + key = (("financingProvider", donor), ("financingRecipient", recipient)) + observations.setdefault(sv, {})[key] = [facet] + return FakeSdmxClient(observations=observations, availability={}) + + +def _trailing(scenario: str) -> ResolveRequest: + (part,) = _SCENARIOS[scenario]().input.parts + return ResolveRequest( + input=ParsedInput( + parts=[part.model_copy(update={"time_text": "the last five years"})], + user_message=_SCENARIOS[scenario]().input.user_message, + ) + ) + + +@pytest.mark.parametrize( + ("years", "composite"), + [ + # The settled range ends in 2021, a year the bilateral destinations lack. + pytest.param(_COMPOSITE_BEHIND, False, id="composite-behind"), + pytest.param(_COMPOSITE_AHEAD, True, id="composite-ahead"), + ], +) +@pytest.mark.parametrize("scenario", ["H1", "H6"]) +def test_a_trailing_request_reads_the_reading_its_settled_range_allows( + scenario: str, + years: dict[tuple[str, str, str], tuple[int, ...]], + composite: bool, monkeypatch: pytest.MonkeyPatch, ) -> None: - """A candidate set's reading over every donor that keeps total ODA because the bilateral - population is unavailable warns as a definite answer does, keyed to that reading.""" - coverage = _patched_coverage( - monkeypatch, {**_HELD, (_DAC2A_MEMBER, _OFFICIAL, _UNCLASSIFIED): 50} + """Series ending in different years settle "the last five years" on the newest years + the table holds, and the answer names the bilateral composite exactly when the + composite holds every year of that range; otherwise it states + `bilateral_reading_unavailable`. A resubmit of its definition answers the same.""" + sdmx = _ragged(monkeypatch, years) + + fresh = _part( + offline_resolve( + _trailing(scenario), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=sdmx, + ) ) - table = _resolve_resubmit( - _identity(_DAC2A).model_copy(update={"facets": None, "entities": {"origin": [_OFFICIAL]}}) - ).spec - candidates = dataclasses.replace(_definite(table), status="candidates") - - read = _readings(candidates, reads=_reads(coverage), selection=ScopeState()) - - assert [spec.spec_id for spec in read.specs] == [table.spec_id] - assert [ - (w.spec_id, w.params) for w in read.warnings if w.code == "bilateral_reading_unavailable" - ] == [(table.spec_id, {"why": "destination_unclassified"})] - - -def _readings( - region: RegionResult, *, reads: ComponentReads, selection: ScopeState -) -> RegionResult: - """`region` with its readings restated (`with_bilateral_readings`) under scope facts of - no operation, as a part with no answer phase decision reads them.""" - return composite_answer.with_bilateral_readings( - region, - reads=reads, - selection=selection, - facts=ScopeFacts(index=get_index(), coverage=reads.coverage), + again = _part( + offline_resolve( + _resubmit(fresh.spec.definition.identity), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=sdmx, + ) + ) + + pin = (fresh.spec.definition.identity.facets or {}).get("provenance") + assert (pin == IdentityBindingValue(ref=_BILATERAL)) == composite + choices = fresh.spec.resolved.scope_choices + assert choices is not None + if scenario == "H6": + # A ranking over every donor has no provider axis to state the reason on. + assert [params for _spec_id, params in _withheld(fresh)] == ( + [] if composite else [{"why": "option_not_covered"}] + ) + else: + (provider_choice,) = (c for c in choices.choices if c.axis == "provider") + bilateral = next(o for o in provider_choice.options if o.option_id == "provider:bilateral") + assert bilateral.availability.code == (None if composite else "option_not_covered") + # The trailing words' stale-year departure, which a resubmit of the definition does not + # raise, is left out. + words = frozenset({"date_coverage_partial"}) + assert again.spec.spec_id == fresh.spec.spec_id + assert _answer(again.spec, again, words=words) == _answer(fresh.spec, fresh, words=words) + + +# A row of the read-count test: given the test's monkeypatch, the reads its answer needs +# stood in for, it returns the answer's read over an SDMX double, made once the reads are +# counted. +type _AnswerOf = Callable[[pytest.MonkeyPatch], Callable[[FakeSdmxClient], Spec]] + + +def _answered(request_of: Callable[[], ResolveRequest]) -> _AnswerOf: + """The Spec a part posted as `request_of` answers.""" + + def prepare(_monkeypatch: pytest.MonkeyPatch) -> Callable[[FakeSdmxClient], Spec]: + def answer(sdmx: FakeSdmxClient) -> Spec: + return _part( + offline_resolve( + request_of(), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=sdmx, + ) + ).spec + + return answer + + return prepare + + +def _nearest_real(monkeypatch: pytest.MonkeyPatch) -> Callable[[FakeSdmxClient], Spec]: + """The level suggestion of a fresh H1 answer, its elected shape under the same bindings. + The fake graph resolves no relaxed query, so the relaxed read answers with the part's + own read of the population, stood in before the reads are counted.""" + answered: list[Any] = [] + monkeypatch.setattr(role_resolve, "graph_confirm_resolve", lambda **_kwargs: answered[0]) + + def suggest(sdmx: FakeSdmxClient) -> Spec: + state = _ended_state(monkeypatch, "H1", sdmx=sdmx) + answered.append(state.mat_result) + level = ( + state.shape_draft, + Relaxation( + axis="what", property_dcid="measurementDenominator", dropped_value_dcids=["x"] + ), + ) + near = asyncio.run(phase_place_materialise._nearest_real_for(state, level=level)) + assert near is not None + (suggestion,) = near + return suggestion + + return suggest + + +@pytest.mark.parametrize( + ("answer_of", "years", "source", "reads"), + [ + *( + pytest.param( + _answered(_SCENARIOS[scenario]), None, source, [("answer", True)], id=scenario + ) + for scenario, source in ( + ("H1", _BILATERAL), + ("H2", DAC2A_INCLUSIVE), + ("H6", _BILATERAL), + ) + ), + pytest.param( + _answered(lambda: _trailing("H1")), + _COMPOSITE_AHEAD, + _BILATERAL, + [("answer", True)], + id="trailing", + ), + pytest.param( + _answered(lambda: _resubmit(_identity(_BILATERAL))), + None, + _BILATERAL, + [("resubmit", True)], + id="resubmit", + ), + # The plain table answer, the control, makes one read of the whole table. + pytest.param(_answered(_SCENARIOS["H5"]), None, None, [("answer", False)], id="H5"), + # A suggestion beside the answer reads its relaxed query once, within the population. + pytest.param( + _nearest_real, + None, + _BILATERAL, + [("answer", True), ("nearest-real", True)], + id="nearest-real", + ), + ], +) +def test_a_part_settling_on_a_population_composite_reads_its_table_once_within_it( + answer_of: _AnswerOf, + years: dict[tuple[str, str, str], tuple[int, ...]] | None, + source: str | None, + reads: list[tuple[str, bool]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A part settling on a composite that binds a population reads its table once, within + the population, fresh or resubmitted, as the plain table answer reads its table once + whole. A nearest-real suggestion settling on the composite reads its own query once, + within the population, beside the answer's read.""" + sdmx = _sdmx() if years is None else _ragged(monkeypatch, years) + answer = answer_of(monkeypatch) + recorded = _table_reads(monkeypatch) + + spec = answer(sdmx) + + pin = (spec.definition.identity.facets or {}).get("provenance") + assert (pin.ref if isinstance(pin, IdentityBindingValue) else None) == source + assert _counted(recorded) == reads + + +def test_a_donor_with_no_total_oda_rows_answers_no_observations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A donor holding no 206 row has no observation in the table read, so the part answers + no_data with `no_observations`, as the table read alone would, where the composite's + population read has no row to hold either. The fixture offers no sibling reading, so + `_reelect_zero_observation_sibling` has nothing to elect.""" + held = {key: value for key, value in _HELD.items() if key[1] != _FRA} + held[_DAC2A_ATTRIBUTED, _FRA, _ETH] = 7 + _patched_coverage(monkeypatch, held) + response = offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(held), ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart) + assert part.no_data.reason == "no_observations" diff --git a/tests/engine/test_dac2a_suggestion_readings.py b/tests/engine/test_dac2a_suggestion_readings.py new file mode 100644 index 000000000..31457e0f0 --- /dev/null +++ b/tests/engine/test_dac2a_suggestion_readings.py @@ -0,0 +1,336 @@ +"""A nearest-real suggestion and a flagged match that settle on DAC2a's bilateral composite are +minted as it from their own build, and no population read follows them. + +Both elect their provider reading at the read they are built over (`reading.elect_at_read`) +and read within the population the reading binds, as the part's definite answer does, so the +composite spec_id they carry is the one a resubmit of their definition answers under. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Callable +from typing import Any + +import pytest + +from search.engine.errors import EngineInputError +from search.engine.region import ( + phase_place_materialise, + phase_shared, + reading, +) +from search.engine.region.state import ResolveState +from search.engine.retrieval import role_resolve +from search.engine.spec_id import identity_spec_id +from search.models import ( + IdentityBindingValue, + NearestRealSpec, + NoDataPart, + Relaxation, + Spec, + Warning, +) +from tests.engine._harness import offline_resolve +from tests.engine.test_dac2a_destination_populations import ( + _BILATERAL, + _FRA, + _KEN, + _VARIABLE, + _counted, + _coverage, # noqa: F401 (autouse fixture) + _ended_state, + _graph_with_parties, + _identity, + _resolve_resubmit, + _resubmit, + _ScriptedLLM, + _sdmx, + _table_reads, + _TableRead, + _years_short, +) + + +def _provenance(spec: Spec) -> str | None: + pin = (spec.definition.identity.facets or {}).get("provenance") + return pin.ref if isinstance(pin, IdentityBindingValue) else None + + +@pytest.fixture +def population_reads(monkeypatch: pytest.MonkeyPatch) -> list[_TableRead]: + """Every read of a part's table from here on (`_table_reads`). It holds the fresh H1 + answer's one read within the population and none by a suggestion or flagged match built + over it.""" + return _table_reads(monkeypatch) + + +# The fresh H1 answer's own read, the only table read a suggestion or match beside it leaves. +_ANSWER_READ = [("answer", True)] + + +@pytest.fixture +def fresh_state(monkeypatch: pytest.MonkeyPatch) -> Callable[[], ResolveState]: + """The state a fresh H1 answer (France, every recipient open, over the bilateral + composite) ends its pass with: its shape, bindings and place, and the read that elected + the composite.""" + + def run() -> ResolveState: + state = _ended_state(monkeypatch, "H1") + assert state.reading is not None and state.reading.table is not None + return state + + return run + + +def _level_suggestion(state: ResolveState, monkeypatch: pytest.MonkeyPatch) -> NearestRealSpec: + """The one level suggestion of `state`'s elected shape under the same bindings, the aid + type the bilateral default reads among them, read as the part's read is.""" + read = state.mat_result + monkeypatch.setattr(role_resolve, "graph_confirm_resolve", lambda **kwargs: read) + level = ( + state.shape_draft, + Relaxation(axis="what", property_dcid="measurementDenominator", dropped_value_dcids=["x"]), + ) + near = asyncio.run(phase_place_materialise._nearest_real_for(state, level=level)) + assert near is not None + (suggestion,) = near + return suggestion + + +def _coded(state: ResolveState, code: str) -> list[Warning]: + return [warning for warning in state.warnings if warning.code == code] + + +def test_a_flagged_match_settling_on_the_bilateral_composite_is_minted_as_it( + fresh_state: Callable[[], ResolveState], population_reads: list[Spec] +) -> None: + """The match carries the composite it is minted as on its own result, and its whole-total + warnings read the identity its rows were read for. The part's own answer's composite + (`ResolveState.minted`) stays as the answer phase minted it.""" + state = fresh_state() + answered = state.minted + assert answered is not None + + match = asyncio.run(phase_shared._elected_flagged_match(state)) + + flagged = match.spec + assert _provenance(flagged) == _BILATERAL + assert flagged.spec_id == identity_spec_id(flagged.definition.identity) + assert _counted(population_reads) == _ANSWER_READ + assert state.minted is answered + assert match.minted is not None + assert [minted.mint.spec_id for minted in match.matched] == [flagged.spec_id] + assert state.minted_as(flagged, match.matched) is match.minted + + +def test_a_flagged_match_minted_as_the_composite_states_the_scope_warning_as_its_own( + fresh_state: Callable[[], ResolveState], +) -> None: + """The part's read is the match's, so its `role_scope_defaulted` warning, counted over the + composite's parties, is unkeyed until the match is built and keyed to it afterwards.""" + state = fresh_state() + (stated,) = _coded(state, "role_scope_defaulted") + assert stated.spec_id is None + + match = asyncio.run(phase_shared._elected_flagged_match(state)) + + (keyed,) = _coded(state, "role_scope_defaulted") + assert (keyed.spec_id, keyed.params, keyed.message) == ( + match.spec.spec_id, + stated.params, + stated.message, + ) + + +def test_a_flagged_match_elects_and_decides_off_the_event_loop( + fresh_state: Callable[[], ResolveState], monkeypatch: pytest.MonkeyPatch +) -> None: + """The match's election and scope read coverage and scan families, so they run on a + worker thread, off the event loop the refusal is built on.""" + state = fresh_state() + state.reading = None + ran: list[tuple[str, bool]] = [] + + def on_loop() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + for name in ("elect_at_read", "decide_read_scope"): + real = getattr(phase_shared, name) + + def recorded(*args: Any, _real: Any = real, _name: str = name, **kwargs: Any) -> Any: + ran.append((_name, on_loop())) + return _real(*args, **kwargs) + + monkeypatch.setattr(phase_shared, name, recorded) + + asyncio.run(phase_shared._elected_flagged_match(state)) + + assert ran == [("elect_at_read", False), ("decide_read_scope", False)] + + +def test_a_flagged_match_whose_composite_refuses_its_read_says_it_reads_total_oda( + fresh_state: Callable[[], ResolveState], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A flagged match whose elected composite refuses its table read is read nowhere, so it + is the table's Spec, and it says it reads total ODA because its scope could not be + decided, keyed to its own spec_id. The refusal is logged once and noted as a degraded + read once, at the election. The match's identity chooses the bilateral composite, as the + part's did, and the reading's undecided choice differs from it by design, which is no + election fault (`reading._checked`), so the check notes nothing more.""" + state = fresh_state() + assert state.reading is not None + bilateral = state.reading.choice + state.reading = None + monkeypatch.setattr(reading, "choose_provider", lambda *_args: bilateral) + degraded: list[str] = [] + + def refusing(*_args: object, **_kwargs: object) -> object: + raise EngineInputError("refused", code="requested_option_unavailable", status=422) + + monkeypatch.setattr(reading, "composite_table_identity", refusing) + monkeypatch.setattr(reading, "note_degraded_read", degraded.append) + + with caplog.at_level(logging.ERROR, logger=reading.__name__): + match = asyncio.run(phase_shared._elected_flagged_match(state)) + + assert [r.levelno for r in caplog.records if r.name == reading.__name__] == [logging.ERROR] + assert degraded == ["reading election"] + + assert match.minted is None + assert _provenance(match.spec) != _BILATERAL + withheld = [ + w.params + for w in _coded(state, "bilateral_reading_unavailable") + if w.spec_id == match.spec.spec_id + ] + assert withheld == [{"why": "scope_undecided"}] + + +def test_a_resubmit_refusal_s_flagged_match_is_the_composite() -> None: + """A composite posted with a destination that holds no row in its population refuses + `no_observations` with the composite as its flagged match.""" + posted = _identity(_BILATERAL).model_copy( + update={"entities": {"origin": [_FRA], "destination": [_KEN]}, "open_roles": None} + ) + + response = offline_resolve( + _resubmit(posted), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=_sdmx() + ) + + (part,) = response.parts + assert isinstance(part, NoDataPart) + flagged = part.no_data.flagged_match + assert isinstance(flagged, Spec) + assert flagged.spec_id == identity_spec_id(posted) + assert _provenance(flagged) == _BILATERAL + + +def test_a_nearest_real_suggestion_settling_on_the_bilateral_composite_is_minted_as_it( + fresh_state: Callable[[], ResolveState], + population_reads: list[Spec], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The suggestion states its own `role_scope_defaulted` warning, keyed to it, as a + resubmit of its definition states it.""" + state = fresh_state() + read = state.mat_result + opened_within: list[Any] = [] + + def confirmed(**kwargs: Any) -> Any: + """The relaxed query's own read, answered as the part's read of the population.""" + opened_within.append(kwargs["within"]) + return read + + monkeypatch.setattr(role_resolve, "graph_confirm_resolve", confirmed) + + # The level relaxation reads the elected shape under the same bindings, the aid type the + # bilateral default reads among them. + level = ( + state.shape_draft, + Relaxation(axis="what", property_dcid="measurementDenominator", dropped_value_dcids=["x"]), + ) + near = asyncio.run(phase_place_materialise._nearest_real_for(state, level=level)) + + assert _counted(population_reads) == _ANSWER_READ + assert near is not None + (composite,) = near + assert _provenance(composite) == _BILATERAL + assert composite.spec_id == identity_spec_id(composite.definition.identity) + again = _resolve_resubmit(composite.definition.identity) + assert again.spec.spec_id == composite.spec_id + assert opened_within and all(within for within in opened_within) + own = [w for w in _coded(state, "role_scope_defaulted") if w.spec_id == composite.spec_id] + resubmitted = [w for w in again.warnings if w.code == "role_scope_defaulted"] + assert [(w.params, w.message) for w in own] == [(w.params, w.message) for w in resubmitted] + assert resubmitted + + +def test_each_reading_the_bilateral_default_cannot_reach_says_so_under_its_own_spec_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Over every donor with bilateral destinations covering fewer years than the answer, the + part reads total ODA, and a nearest-real suggestion and a flagged match of it, each + elected at its own read, each carry `bilateral_reading_unavailable` keyed to its own + spec_id, though their messages match.""" + _years_short(monkeypatch) + state = _ended_state(monkeypatch, "H6") + assert state.reading is not None and state.reading.table is None + + suggestion = _level_suggestion(state, monkeypatch) + match = asyncio.run(phase_shared._elected_flagged_match(state)) + + withheld = _coded(state, "bilateral_reading_unavailable") + assert suggestion.spec_id != match.spec.spec_id + assert [(w.spec_id, w.params) for w in withheld] == [ + (suggestion.spec_id, {"why": "option_not_covered"}), + (match.spec.spec_id, {"why": "option_not_covered"}), + ] + + +def test_a_netting_basis_relaxation_elects_the_bilateral_composite_as_its_resubmit_does( + fresh_state: Callable[[], ResolveState], + population_reads: list[Spec], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Dropping nettingBasis leaves an unbound slot in the suggestion's identity. The election + reads that slot as the mint does, so the suggestion settles on the bilateral composite, + reads within its population, passes the after-read monitor with no log, and a resubmit of + its definition answers under its spec_id.""" + state = fresh_state() + read = state.mat_result + opened_within: list[Any] = [] + + def confirmed(**kwargs: Any) -> Any: + opened_within.append(kwargs["within"]) + return read + + monkeypatch.setattr(role_resolve, "graph_confirm_resolve", confirmed) + + with caplog.at_level("WARNING"): + near = asyncio.run(phase_place_materialise._nearest_real_for(state)) + + assert _counted(population_reads) == _ANSWER_READ + assert near is not None + (relaxed,) = [ + spec + for spec in near + if any( + slot.property == "nettingBasis" and slot.binding.state == "unbound" + for slot in spec.definition.identity.slots + ) + ] + assert _provenance(relaxed) == _BILATERAL + again = _resolve_resubmit(relaxed.definition.identity) + assert again.spec.spec_id == relaxed.spec_id + assert any(within for within in opened_within) + monitor = [r for r in caplog.records if r.name == reading.__name__ and r.levelno >= 30] + assert monitor == [] diff --git a/tests/engine/test_degraded_read_sites.py b/tests/engine/test_degraded_read_sites.py index 6fdb1dd73..c0ae10c2f 100644 --- a/tests/engine/test_degraded_read_sites.py +++ b/tests/engine/test_degraded_read_sites.py @@ -57,6 +57,7 @@ from search.engine.retrieval.read_plan import PairPlan from search.engine.unit import refs from search.models import EntityRoleKey, Spec, Warning, omitted_observations +from search.scope.state import ScopeState from series_coverage.index import CoverageIndex from tests.engine import _fail_open_deadline from tests.engine._fail_open_deadline import FAIL_OPEN_REQUESTS, assert_fail_open_deadline @@ -141,7 +142,9 @@ def _relaxation_ctx(**changes: object) -> SimpleNamespace: "date_request": None, "deadline": None, "role_class_walks": object(), - "from_request": lambda *_warnings: None, + "coverage": None, + "scope_selection": ScopeState(), + "operations": (), **changes, } ) @@ -711,7 +714,7 @@ def suggest(self, **_kwargs): probe_places=["country/KEN"], roles=SimpleNamespace(obs_destination_dcid=None, origin_dcids=[]), ) - read = SimpleNamespace(place=place_ctx, plan=None, member_facts=None) + read = SimpleNamespace(place=place_ctx, plan=None, member_facts=None, composite=False) return [spec], [], {"near": read}, None def attach(self, spec, result, _warnings): @@ -755,6 +758,9 @@ def _nearest_real(monkeypatch: pytest.MonkeyPatch, recorder: _NearestRealRecorde measure_elected_properties=frozenset(), defaulted_properties=frozenset(), coverage=None, + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, unsettled_place=None, pin_served_provenance=False, include_observations=True, @@ -762,7 +768,6 @@ def _nearest_real(monkeypatch: pytest.MonkeyPatch, recorder: _NearestRealRecorde rate_cache=None, warnings=[], departure_facts={}, - from_request=lambda *_warnings: None, ) return materialise._nearest_real_for(cast(ResolveState, state)) diff --git a/tests/engine/test_discover_graph_confirm.py b/tests/engine/test_discover_graph_confirm.py index 478338f7f..df8c4e71e 100644 --- a/tests/engine/test_discover_graph_confirm.py +++ b/tests/engine/test_discover_graph_confirm.py @@ -188,6 +188,7 @@ def test_graph_confirm_resolve_specificity_sort() -> None: }, ) ).snapshot(), + within=None, ) assert isinstance(result, Materialised), f"Expected Materialised but got {result!r}" # sv/Specific must appear before sv/General (more specific first) diff --git a/tests/engine/test_discover_role_resolve.py b/tests/engine/test_discover_role_resolve.py index 3e605aec1..d29b27264 100644 --- a/tests/engine/test_discover_role_resolve.py +++ b/tests/engine/test_discover_role_resolve.py @@ -161,6 +161,7 @@ def test_custom_labels_used_when_given(self): obs_label="years", sdmx=FakeSdmxClient(observations={}, availability={}), coverage_snapshot=_debtor_coverage(sv, "country/ETH"), + within=None, ) assert isinstance(result, Materialised) # The variables dimension counts resolved members. @@ -177,6 +178,7 @@ def test_default_labels_are_sources_observations(self): graph=FakeGraph(nodes={}, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}, availability={}), coverage_snapshot=_debtor_coverage(sv, "country/ETH"), + within=None, ) assert isinstance(result, Materialised) # "variables" is a third, always-appended dimension -- see the @@ -246,6 +248,7 @@ def _resolve(self, sv_dcid: str, arcs: dict, bindings: list[SlotBindingDraft]): graph=FakeGraph(nodes={}, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}, availability={}), coverage_snapshot=_debtor_coverage(sv_dcid, self._RECIPIENT), + within=None, ) def test_unresolved_bound_property_excludes_sv(self): @@ -379,6 +382,7 @@ def test_no_recipient_still_reports_variable_not_resolved(self): graph=FakeGraph(nodes={}, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}, availability={}), coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) assert result.reason == "variable_not_resolved" @@ -405,6 +409,7 @@ def test_no_arc_facts_still_reports_variable_not_resolved(self): graph=FakeGraph(nodes={}, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}, availability={}), coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) assert result.reason == "variable_not_resolved" diff --git a/tests/engine/test_entity_resolution.py b/tests/engine/test_entity_resolution.py index 5a41dd0dc..d03669f5c 100644 --- a/tests/engine/test_entity_resolution.py +++ b/tests/engine/test_entity_resolution.py @@ -1188,6 +1188,7 @@ def test_region_expansion_issues_one_post(self): date_request=None, place_dcids=self._COUNTRIES, coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_grounding_identity_entities.py b/tests/engine/test_grounding_identity_entities.py index 970e379ce..2d85f7954 100644 --- a/tests/engine/test_grounding_identity_entities.py +++ b/tests/engine/test_grounding_identity_entities.py @@ -406,6 +406,37 @@ def test_a_scope_warning_naming_every_member_is_restated_to_count_the_answers_sc assert [w.message for w in warnings] == [restated] +def test_a_scope_warning_is_found_by_its_code_and_axis_whatever_its_text(): + """The warning on the discovered axis is restated whatever its text says, and the + companion axis's warning, whose text another reader could have worded like the + discovered axis's, stays as it was.""" + discovered = _discovered_scope("destination", 3) + mat = _materialised( + discovered_scope=discovered, + scope_member_dcids=discovered.member_dcids, + also_defaulted_axis="origin", + ) + on_axis, companion = role_scope_defaulted_warnings( + mat, boundary_roles=[], label_of=_EchoLabels() + ) + reworded = on_axis.model_copy(update={"message": "Recipients were counted elsewhere."}) + lookalike = companion.model_copy(update={"message": on_axis.message}) + warnings = [lookalike, reworded] + + restate_role_scope_warnings( + warnings, + mat, + members=discovered.member_dcids[:1], + boundary_roles=[], + label_of=_EchoLabels(), + ) + + assert [w.message for w in warnings] == [ + on_axis.message, + "No recipient was named, so the answer covers the one recipient with data.", + ] + + def test_a_scope_warning_is_removed_when_the_answer_lists_no_member(): """Both flow axes are open and the facet filter leaves no recipient served: the answer resolved across no recipient, so the recipient warning goes and the provider warning, diff --git a/tests/engine/test_index_member_source.py b/tests/engine/test_index_member_source.py index 043cb8e86..00b5d87c2 100644 --- a/tests/engine/test_index_member_source.py +++ b/tests/engine/test_index_member_source.py @@ -249,6 +249,7 @@ def test_standard_resolve_confirms_an_index_sourced_member_never_in_the_recall_p graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -434,6 +435,7 @@ def test_the_resolver_reads_the_dataset_a_source_binding_names(): ), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=FakeCoverageService(coverage).snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_member_population.py b/tests/engine/test_member_population.py index 2d630fc36..bd10f7d2c 100644 --- a/tests/engine/test_member_population.py +++ b/tests/engine/test_member_population.py @@ -65,6 +65,7 @@ def test_standard_resolve_confirmed_population_exceeds_the_elected_winner(): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -95,6 +96,7 @@ def test_set_binding_collected_count_matches_population_confirmed(): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -158,6 +160,7 @@ def test_index_sourced_population_is_exact(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -204,6 +207,7 @@ def test_recall_pool_population_is_inexact(): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_merged_composite_mint.py b/tests/engine/test_merged_composite_mint.py new file mode 100644 index 000000000..6138ee115 --- /dev/null +++ b/tests/engine/test_merged_composite_mint.py @@ -0,0 +1,120 @@ +"""A same-shape merge of DAC2a parts that settle on the bilateral composite mints the +composite over the merged identity (`spec_build.build_spec(composite=)`), so it reads no +population a second time and answers as a resubmit of its merged definition does. + +The fixture is France's DAC2a 206 gross and net disbursements to Ethiopia, the European +Commission and `DACAllRecipients`, and its imputed 106 to Ethiopia, every destination open: +the bilateral composite reads the country alone.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from search.models import ( + BaseIdentity, + DatasetResubmitInput, + DefinitePart, + Resolution, + ResolveOptions, + ResolveRequest, + ResubmitIdentityEntry, +) +from tests.engine._harness import offline_resolve +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_dac2a_destination_populations import ( + _AGENCY, + _BILATERAL, + _DAC2A_ATTRIBUTED, + _DAC2A_MEMBER, + _ETH, + _FRA, + _HELD, + _VARIABLE, + _answer, + _counted, + _identity, + _patched_coverage, + _ref, + _sdmx, + _table_reads, +) +from tests.engine.test_whole_total_bind_e2e import _ALL_RECIPIENTS, _graph +from tests.fixtures import FakeGraph + +_DAC2A_NET = _DAC2A_MEMBER.replace("_Gross_", "_Net_") +_NET_HELD = { + **_HELD, + (_DAC2A_NET, _FRA, _ETH): 90, + (_DAC2A_NET, _FRA, _AGENCY): 190, + (_DAC2A_NET, _FRA, _ALL_RECIPIENTS): 450, +} + + +def _netting(identity: BaseIdentity, basis: str) -> BaseIdentity: + """`identity` with its netting basis bound to `basis`.""" + slots = [ + slot.model_dump(mode="json") + if slot.property != "nettingBasis" + else {"property": "nettingBasis", "binding": _ref(basis)} + for slot in identity.slots + ] + return BaseIdentity.model_validate( + {**identity.model_dump(mode="json", exclude_none=True), "slots": slots} + ) + + +def _graph_holding_net() -> FakeGraph: + """The whole-total fixture graph finding DAC2a's gross and net members and the imputed + one, holding the agency and the total spanning both sides, which a part's reads post.""" + graph = _graph(variable=_VARIABLE, members=(_DAC2A_MEMBER, _DAC2A_ATTRIBUTED, _DAC2A_NET)) + for dcid in (_AGENCY, _ALL_RECIPIENTS): + graph._nodes.setdefault(dcid, {"label": dcid, "type": "Organization"}) + return graph + + +def _resubmit(*identities: BaseIdentity) -> Any: + return offline_resolve( + ResolveRequest( + input=DatasetResubmitInput( + entries=[ + ResubmitIdentityEntry(variable_text=_VARIABLE, identity=identity) + for identity in identities + ] + ), + response_options=ResolveOptions(include_observations=False), + ), + graph=_graph_holding_net(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(_NET_HELD), + ) + + +def test_a_merged_bilateral_composite_answers_as_a_resubmit_of_its_definition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A gross and a net part the merge folds answer the bilateral composite over both + bases, minted over the merged identity from its members' facets, with no read of the + population beyond each member's own: its scope, entities, headline, coverage and + warnings are a resubmit's of its merged definition.""" + _patched_coverage(monkeypatch, _NET_HELD) + bilateral = _identity(_BILATERAL) + reads = _table_reads(monkeypatch) + gross, net = _parts(_resubmit(_netting(bilateral, "Gross"), _netting(bilateral, "Net"))) + + assert _counted(reads) == [("resubmit", True), ("resubmit", True)] + + assert gross.spec == net.spec + (again,) = _parts(_resubmit(gross.spec.definition.identity)) + assert again.spec.spec_id == gross.spec.spec_id + assert _answer(gross.spec, gross) == _answer(again.spec, again) + scope = gross.spec.resolved.scope + assert scope is not None and [member.ref for member in scope.members] == [_ETH] + + +def _parts(response: Any) -> list[DefinitePart]: + assert isinstance(response, Resolution), response + parts = [part for part in response.parts if isinstance(part, DefinitePart)] + assert len(parts) == len(response.parts) + return parts diff --git a/tests/engine/test_observability.py b/tests/engine/test_observability.py index ccf375c83..5d374bb81 100644 --- a/tests/engine/test_observability.py +++ b/tests/engine/test_observability.py @@ -160,6 +160,7 @@ def _minimal_spec(): role=EntityRoleSubject(), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape_model, diff --git a/tests/engine/test_provenance.py b/tests/engine/test_provenance.py index 79811c1ec..7c7db0a21 100644 --- a/tests/engine/test_provenance.py +++ b/tests/engine/test_provenance.py @@ -5,7 +5,8 @@ import pytest from code_index.composites import DECLARED_COMPOSITES -from code_index.declarations import CRS, CRS_INCLUSIVE +from code_index.declarations import CRS, CRS_INCLUSIVE, DAC2A_BILATERAL +from code_index.shared import get_index from origin_catalogue import BASE, DFO, UN_SYSTEM, ProvenanceRow, SourceRef from search.engine.region.refs import provenance_labels from search.models import DataCommonsRef, ProvenanceRef @@ -146,7 +147,7 @@ def test_a_composite_facet_keeps_what_its_component_tables_share( bind_catalogue(restrictive_catalogue(provenances=rows)) composite = DECLARED_COMPOSITES[CRS_INCLUSIVE] - ref = composite_facet_ref(composite) + ref = composite_facet_ref(composite, index=get_index()) assert (ref.ref, ref.label, ref.documentation) == (CRS_INCLUSIVE, composite.label, None) assert ( @@ -155,6 +156,17 @@ def test_a_composite_facet_keeps_what_its_component_tables_share( ) == expected +def test_the_bilateral_composite_facet_reads_apart_from_its_plain_table(): + """The facet label of `DAC2A_BILATERAL`, whose declared label is its table's, carries the + composite's measure noun, as its bundle does.""" + composite = DECLARED_COMPOSITES[DAC2A_BILATERAL] + + ref = composite_facet_ref(composite, index=get_index()) + + assert ref.label == f"{composite.label} {composite.measure_noun}" + assert ref.label != composite.label + + class _LabelSpyGraph(FakeGraph): """Record every `node_labels_batch` and `node_arc_targets_many` call.""" diff --git a/tests/engine/test_qualifier_suppression_and_provenance.py b/tests/engine/test_qualifier_suppression_and_provenance.py index 1260d7734..9b28a1ce5 100644 --- a/tests/engine/test_qualifier_suppression_and_provenance.py +++ b/tests/engine/test_qualifier_suppression_and_provenance.py @@ -450,8 +450,9 @@ def test_spec_id_is_byte_identical_with_and_without_provenance_populated(): "coverage": CoverageBare(has_data=True), "open_roles": (), } - without = build_spec(measure_elected_properties=frozenset(), **common) + without = build_spec(composite=None, measure_elected_properties=frozenset(), **common) with_provenance = build_spec( + composite=None, measure_elected_properties=frozenset(), **common, text_recognized_properties=frozenset({"constraintProp"}), @@ -478,6 +479,7 @@ def test_measure_elected_properties_stamps_forced_slot_and_leaves_the_rest(): StatVar(ref=_gr("sv"), shape_id=shape.shape_id, slot_values=[], constraint_properties=[]) ] spec = build_spec( + composite=None, open_roles=(), shape=shape, slots=slots, @@ -514,6 +516,7 @@ def test_measure_elected_and_text_recognized_overlap_raises(): with pytest.raises(ValueError, match="sharedProp"): build_spec( + composite=None, open_roles=(), shape=shape, slots=slots, diff --git a/tests/engine/test_read_plan_grid.py b/tests/engine/test_read_plan_grid.py index 3a7f30ec5..3846b6058 100644 --- a/tests/engine/test_read_plan_grid.py +++ b/tests/engine/test_read_plan_grid.py @@ -696,6 +696,7 @@ def test_a_standard_answer_counts_the_points_its_preview_serves( date_request=date_request, place_dcids=places, coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(mat, Materialised) # Every plan that serves nothing yields the carrier for data outside its years, and @@ -784,6 +785,7 @@ def test_a_standard_window_between_recorded_and_projected_years_reads_nothing( date_request=DateRequest(window=TimeWindow(start_year=2020), latest=False), place_dcids=[_KEN], coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(mat, Materialised) assert mat.has_data is False and mat.available_at_place is True @@ -819,6 +821,7 @@ def test_a_scope_read_elects_the_facet_with_points_in_its_years(monkeypatch) -> date_request=DateRequest(window=TimeWindow(start_year=2021), latest=False), place_dcids=[_KEN, _NOR, _uga], coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(mat, Materialised) and mat.has_data plan = mat.read_plan @@ -869,6 +872,7 @@ def test_a_series_missing_an_id_or_a_provenance_answers_only_inside_its_years( date_request=DateRequest(window=window, latest=False), place_dcids=[_KEN], coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(mat, Materialised) assert mat.has_data is has_data diff --git a/tests/engine/test_reading_decision.py b/tests/engine/test_reading_decision.py new file mode 100644 index 000000000..f25b6c15b --- /dev/null +++ b/tests/engine/test_reading_decision.py @@ -0,0 +1,869 @@ +"""A part's provider reading elected before its read (`reading.elect_reading`), applied to +the identity the answer phase mints after the read (`scope_decision.build_part_scope`), +states the scope decided over that identity itself (`scope_decision.decide_scope`), which +the answer phase checks it against. Where the range the answer settles on would read a +composite the years read withheld, the answer says so. A place set reads with the election +of the place it settles on, and a composite that refuses its table read answers no_data. + +The fixture is `test_dac2a_destination_populations`': France's 206 to Ethiopia, to the +European Commission and to `DACAllRecipients`, and its imputed 106 to Ethiopia. +""" + +from __future__ import annotations + +import asyncio +import copy +import dataclasses +import logging +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +from code_index.composites import destination_population +from code_index.declarations import DAC2A_INCLUSIVE +from code_index.shared import get_index +from search.engine.bind import SlotBindingDraft +from search.engine.errors import EngineInputError +from search.engine.identity_edit import TableRead +from search.engine.region import ( + phase_answer, + phase_place_materialise, + pipeline, + reading, + resubmit, + scope_decision, +) +from search.engine.region.candidates import _Candidate +from search.engine.region.place_context import empty_place_context +from search.engine.region.reading import ReadingDecision, elect_reading +from search.engine.region.scope_decision import ( + READ_WINDOW_NOT_COVERED, + ProviderChoice, + ScopeDecision, + build_part_scope, +) +from search.engine.unit.conversion import ConversionPeriod +from search.models import ( + BaseIdentity, + DefinitePart, + IdentityBindingValue, + NoDataPart, + Resolution, + ResolveRequest, +) +from search.scope.state import ScopeState +from search.scope.tables import ScopeFacts +from tests.engine._builders import slow_related_reads +from tests.engine._fake_sdmx import FakeSdmxClient +from tests.engine._harness import offline_resolve +from tests.engine.test_aid_dataset_bind_e2e import _ScriptedLLM +from tests.engine.test_dac2a_destination_populations import ( + _AGENCY, + _BILATERAL, + _COMPOSITE_AHEAD, + _COMPOSITE_BEHIND, + _DAC2A, + _ETH, + _FRANCE, + _HELD, + _SCENARIOS, + _TOTAL_YEARS, + _fresh_request, + _graph_with_parties, + _identity, + _part, + _patched_coverage, + _ragged, + _sdmx, + _trailing, + _withheld, +) +from tests.engine.test_whole_total_bind_e2e import _VARIABLE + + +@pytest.fixture(autouse=True) +def _coverage(monkeypatch: pytest.MonkeyPatch) -> None: + _patched_coverage(monkeypatch, _HELD) + + +@dataclasses.dataclass(frozen=True) +class _Decided: + """The scope decided over an identity the answer phase minted: what it read and what + it decided.""" + + spec_id: str + identity: BaseIdentity + facts: ScopeFacts + decision: ScopeDecision + + +def _elect_and_decide( + request: ResolveRequest, sdmx: FakeSdmxClient, monkeypatch: pytest.MonkeyPatch +) -> tuple[list[ReadingDecision | None], list[_Decided]]: + """The reading each table read of `request` elects from the part's state before it + reads, and the scope decided over each identity the answer phase mints after it + (`scope_decision.decide_scope`), which the answer phase checks its applied choice + against (`reading._checked`).""" + elected: list[ReadingDecision | None] = [] + decided: list[_Decided] = [] + task, checked = phase_place_materialise._materialise_task, reading._checked + + def electing_task(state: Any) -> Any: + unsettled = state.unsettled_place + assert unsettled is None or unsettled.decided, "the fixture settles its place first" + elected.append( + elect_reading( + state.shape_draft, + list(state.bindings), + state.place, + date_request=state.date_request, + selection=copy.deepcopy(state.scope_selection), + coverage=state.coverage, + operations=state.operations, + ) + ) + return task(state) + + def recording_check( + applied: ScopeDecision, + identity: BaseIdentity, + elected: ReadingDecision, + *, + selection: ScopeState, + facts: ScopeFacts, + ) -> ScopeDecision: + spec_id = applied.spec_id + decision = scope_decision.decide_scope( + spec_id, identity, selection=copy.deepcopy(selection), facts=facts + ) + decided.append(_Decided(spec_id, identity, facts, decision)) + return checked(applied, identity, elected, selection=selection, facts=facts) + + monkeypatch.setattr(phase_place_materialise, "_materialise_task", electing_task) + monkeypatch.setattr(reading, "_checked", recording_check) + offline_resolve(request, graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=sdmx) + return elected, decided + + +def _stated(decision: ScopeDecision) -> dict[str, Any]: + """What a scope decision states of its answer.""" + return { + "scope": None if decision.scope is None else decision.scope.model_dump(mode="json"), + "composite": decision.composite, + "warning": None if decision.warning is None else decision.warning.model_dump(mode="json"), + } + + +def _assert_elected_scope_is_decided( + elected: list[ReadingDecision | None], decided: list[_Decided] +) -> ReadingDecision: + """The one reading elected before the read, applied to the identity the answer phase + minted, states the scope it decided over that identity.""" + (reading,) = elected + (answer,) = decided + assert reading is not None + applied = build_part_scope( + answer.identity, reading.choice, spec_id=answer.spec_id, facts=answer.facts + ) + assert _stated(applied) == _stated(answer.decision) + return reading + + +def _composite(decision: ScopeDecision) -> str | None: + """The provenance the decision's scope compiles to, where it is a composite.""" + if not decision.composite: + return None + assert decision.scope is not None + pin = (decision.scope.compiled.identity.facets or {}).get("provenance") + return pin.ref if isinstance(pin, IdentityBindingValue) else None + + +_TO_AGENCY = lambda: _fresh_request( # noqa: E731 + [_FRANCE, ("to", "European Commission")], + message="official development assistance from France to the European Commission", + open_role=None, +) + + +@pytest.mark.parametrize( + ("request_of", "source", "population"), + [ + pytest.param(_SCENARIOS["H1"], _BILATERAL, True, id="H1"), + pytest.param(_SCENARIOS["H2"], DAC2A_INCLUSIVE, True, id="H2"), + # All recipients spans both sides, so the default reads total ODA there. + pytest.param(_SCENARIOS["H5"], None, False, id="H5"), + # A ranking over every donor has no provider axis and still reads the composite. + pytest.param(_SCENARIOS["H6"], _BILATERAL, True, id="H6"), + # An agency is a multilateral destination, which the composite holds no row for. + pytest.param(_TO_AGENCY, None, False, id="agency"), + ], +) +def test_the_reading_elected_before_the_read_states_the_scope_decided_after_it( + request_of: Any, source: str | None, population: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + elected, decided = _elect_and_decide(request_of(), _sdmx(), monkeypatch) + + reading = _assert_elected_scope_is_decided(elected, decided) + (answer,) = decided + assert _composite(answer.decision) == source + assert (reading.table is not None) == (source is not None) + assert bool(reading.table and reading.table.populations) == population + if request_of is _TO_AGENCY: + assert answer.identity.entities.get("destination") == [_AGENCY] + + +@pytest.mark.parametrize( + ("years", "source"), + [ + # The bilateral destinations lack 2021, the newest year of the settled range. + pytest.param(_COMPOSITE_BEHIND, None, id="composite-behind"), + pytest.param(_COMPOSITE_AHEAD, _BILATERAL, id="composite-ahead"), + ], +) +def test_a_reading_elected_over_the_read_window_states_the_settled_range_s_scope( + years: dict[tuple[str, str, str], tuple[int, ...]], + source: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ "The last five years" reads a window wider than the range the answer settles on + after the read (`extract.read_window`), and the reading elected over the wider window + states the settled range's scope.""" + sdmx = _ragged(monkeypatch, years) + + elected, decided = _elect_and_decide(_trailing("H1"), sdmx, monkeypatch) + + _assert_elected_scope_is_decided(elected, decided) + (answer,) = decided + settled = answer.identity.requested_window + assert settled.kind == "range" + assert (settled.start_year, settled.end_year) == (2017, 2021) + assert _composite(answer.decision) == source + + +def _pin(part: DefinitePart) -> str | None: + """The provenance `part`'s definition pins.""" + pin = (part.spec.definition.identity.facets or {}).get("provenance") + return pin.ref if isinstance(pin, IdentityBindingValue) else None + + +def _faults(caplog: pytest.LogCaptureFixture) -> list[str]: + """The ERROR records the reading decision logged, a fault in the reading election.""" + return [ + record.getMessage() + for record in caplog.records + if record.levelno >= logging.ERROR and record.name == reading.__name__ + ] + + +# The bilateral destinations' rows start a year after the rest of the table's, so "the last +# five years" settles on 2017 to 2021, which the composite covers, but reads 2016 too. +_WINDOW_GAP = { + key: _TOTAL_YEARS if key[2] == _ETH else (_TOTAL_YEARS[0] - 1, *_TOTAL_YEARS) for key in _HELD +} + + +def _unwithheld(monkeypatch: pytest.MonkeyPatch) -> None: + """Every election from here on withholds nothing, as an election whose document lost the + bilateral default would. That diverges the pre-read identity from the minted one + (`reading._pre_read_identity`).""" + elect = reading.elect_reading + + def unwithheld(*args: Any, **kwargs: Any) -> ReadingDecision | None: + elected = elect(*args, **kwargs) + if elected is None: + return None + return dataclasses.replace( + elected, choice=dataclasses.replace(elected.choice, withheld=None) + ) + + monkeypatch.setattr(reading, "elect_reading", unwithheld) + + +def _degraded(monkeypatch: pytest.MonkeyPatch, module: Any) -> list[str]: + """The subject of each degraded read `module` notes from here on.""" + noted: list[str] = [] + monkeypatch.setattr(module, "note_degraded_read", noted.append) + return noted + + +@pytest.mark.parametrize( + ("withheld", "stated", "faulted"), + [ + pytest.param(True, [{"why": READ_WINDOW_NOT_COVERED}], False, id="option-not-covered"), + # Withheld for no reason, the table election is no window gap, so it is a fault. + pytest.param(False, [], True, id="withheld-nothing"), + ], +) +def test_a_reading_the_read_window_withholds_and_the_settled_range_allows_says_so( + withheld: bool, + stated: list[dict[str, str]], + faulted: bool, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """The reading is elected over the years the read reads, so a composite missing a year + the settled range leaves out is withheld, and the answer reads total ODA saying the + bilateral destinations do not cover every year read, with no fault logged. An election + of the table that withheld nothing is a different case. It is logged at ERROR and + noted as a degraded read.""" + sdmx = _ragged(monkeypatch, _WINDOW_GAP) + if not withheld: + _unwithheld(monkeypatch) + degraded = _degraded(monkeypatch, reading) + + with caplog.at_level(logging.WARNING, logger=reading.__name__): + part = _part( + offline_resolve( + _trailing("H6"), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=sdmx + ) + ) + + window = part.spec.definition.identity.requested_window + assert (window.kind, window.start_year, window.end_year) == ("range", 2017, 2021) + assert _pin(part) is None + assert [params for _spec_id, params in _withheld(part)] == stated + assert bool(_faults(caplog)) is faulted + assert degraded == (["reading election"] if faulted else []) + + +def test_an_election_that_raises_building_its_document_reads_total_oda_saying_so( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """An election that raises before its choice, here electing the dataset its document + names, is logged at ERROR and noted as a degraded read, and the part reads the table as + total ODA saying its scope could not be decided.""" + + def raising(*_args: object) -> object: + raise RuntimeError("index fault") + + monkeypatch.setattr(reading, "_elected_dataset", raising) + degraded = _degraded(monkeypatch, scope_decision) + + with caplog.at_level(logging.ERROR, logger=scope_decision.logger.name): + part = _part( + offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + assert _pin(part) is None + assert [params for _spec_id, params in _withheld(part)] == [{"why": "scope_undecided"}] + assert any( + record.exc_info is not None and record.exc_info[0] is RuntimeError + for record in caplog.records + ) + assert "scope decision" in degraded + + +def _decided_as_the_table(monkeypatch: pytest.MonkeyPatch, *, spec_id: str | None) -> None: + """The answer phase's scope from here on compiles no composite and mints none, so the + finished answer is the table's Spec. The decision is made for another answer, or, with + `spec_id` None, for the answer itself, the table's Spec (`ReadScope.answer`).""" + decide = phase_answer._decide_answer_scope + + def as_table(*args: Any) -> Any: + scope = decide(*args) + assert scope.decision is not None + decision = dataclasses.replace( + scope.decision, + spec_id=scope.answer.spec_id if spec_id is None else spec_id, + composite=False, + scope=None, + ) + return dataclasses.replace(scope, decision=decision, minted=None) + + monkeypatch.setattr(phase_answer, "_decide_answer_scope", as_table) + + +@pytest.mark.parametrize( + ("within", "decided_for"), + [ + pytest.param(True, "other", id="read-within"), + # The table's decision states the answer's spec_id, but its rows were read within the + # population, so no table label stands over them. + pytest.param(True, None, id="read-within-as-table"), + pytest.param(False, "other", id="table-read"), + ], +) +def test_a_finished_answer_decided_again_chooses_no_composite( + within: bool, + decided_for: str | None, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A finished answer no decision made before its read states is decided again from its + Spec, which here, the table of H1, chooses the bilateral composite. That decision chooses + nothing. It is logged at ERROR and noted as a degraded read, and the answer fails + closed. Rows read within the composite's population and not minted as it answer no_data, + `facet_not_available`, before anything is decided, whatever the decision made before the + read states, and before the answer's related read is awaited, which is cancelled. Rows + read from the whole table read total ODA, saying its scope could not be decided, and + state their related read.""" + if not within: + # The read elects no reading, so it reads the whole table. + monkeypatch.setattr(reading, "elect_reading", lambda *args, **kwargs: None) + _decided_as_the_table(monkeypatch, spec_id=decided_for) + degraded = _degraded(monkeypatch, scope_decision) + decided_again: list[str] = [] + decide_spec_scope = scope_decision.decide_spec_scope + + def deciding(spec: Any, **kwargs: Any) -> Any: + decided_again.append(spec.spec_id) + return decide_spec_scope(spec, **kwargs) + + monkeypatch.setattr(scope_decision, "decide_spec_scope", deciding) + related = slow_related_reads(monkeypatch) + awaited: list[Any] = [] + with_related = phase_answer.with_related + + async def awaiting(spec: Any, read: Any) -> Any: + awaited.append(read) + return await with_related(spec, read) + + monkeypatch.setattr(phase_answer, "with_related", awaiting) + + with caplog.at_level(logging.ERROR, logger=scope_decision.logger.name): + response = offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + if within: + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "facet_not_available" + else: + assert isinstance(part, DefinitePart), part + assert _pin(part) is None + assert part.spec.resolved.scope_choices is None + assert [params for _spec_id, params in _withheld(part)] == [{"why": "scope_undecided"}] + # Rows read within the population fail closed before anything is decided again. + assert decided_again == ([] if within else [part.spec.spec_id]) + # Each read but the answer's last is replaced as its years settle, and cancelled. + *replaced, read = related + assert all(earlier.task.cancelled() for earlier in replaced) + assert awaited == ([] if within else [read]) + assert read.task.cancelled() is within + logged = [r.levelno for r in caplog.records if r.name == scope_decision.logger.name] + assert logged == [logging.ERROR] + assert degraded == ["scope decision"] + + +def test_a_finished_answer_that_fails_closed_raises_and_judges_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An answer that fails closed at its check contributes no warning, departure fact or + judgement. Over the trailing years, grounding departs `date_coverage_partial` for the + stale years it settles on (`phase_answer._ground_elected_answer`), the Spec minted as + the composite rewords `role_scope_defaulted` to count the composite's parties + (`CompositeMint.role_scope_warnings`), and the carve-outs judge the answer + (`requirements.settle_answer`). Refused at its check, the part holds what it had raised + and judged before grounding (`ResolveState.mark_answer`, `phase_answer.failed_closed`).""" + + async def refused(*_args: Any, **_kwargs: Any) -> None: + return None + + monkeypatch.setattr(phase_answer, "finished_scope", refused) + held: dict[str, tuple[Any, ...]] = {} + + def holding(state: Any) -> tuple[Any, ...]: + return (tuple(state.warnings), dict(state.departure_facts), state.judgement) + + ground = phase_answer._ground_elected_answer + + async def grounding(state: Any) -> Any: + held["grounding"] = holding(state) + return await ground(state) + + monkeypatch.setattr(phase_answer, "_ground_elected_answer", grounding) + assemble = phase_answer._assemble_definite_answer + + async def assembling(state: Any, *args: Any) -> Any: + held["assembly"] = holding(state) + return await assemble(state, *args) + + monkeypatch.setattr(phase_answer, "_assemble_definite_answer", assembling) + closed: list[Any] = [] + close = phase_answer.failed_closed + + def closing(state: Any, *args: Any, **kwargs: Any) -> Any: + held["check"] = holding(state) + result = close(state, *args, **kwargs) + closed.append((result, state.judgement)) + return result + + monkeypatch.setattr(phase_answer, "failed_closed", closing) + + offline_resolve( + _trailing("H1"), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=_sdmx() + ) + + ((result, judgement),) = closed + warnings, facts, before = held["grounding"] + assert (result.status, result.no_data_reason) == ("no_data", "facet_not_available") + # Grounding departed for the stale years before assembly started. + assert [w.code for w in held["assembly"][0]] == [ + *(w.code for w in warnings), + "date_coverage_partial", + ] + checked, _facts, judged = held["check"] + assert [w for w in checked if w.code == "role_scope_defaulted"] != [ + w for w in warnings if w.code == "role_scope_defaulted" + ] + assert judged is not None + assert result.warnings == warnings + assert result.departure_facts == facts + assert judgement is before + + +def test_a_finished_answer_is_judged_by_the_reading_its_own_rows_were_read_under( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A finished answer is judged by the reading its own rows were read under (the + `reading` its producer passes `phase_answer.finished_scope`), which on the candidates + path is the survivor's, not the elected shape's (`ResolveState.reading`). Here the + elected shape read the whole table and the answer's rows were read within the bilateral + population. Not minted as the composite and stated by no decision, it fails closed, + noted as a degraded read.""" + captured: list[tuple[Any, Any]] = [] + composite_preview = pipeline._with_composite_preview + + async def capture(result: Any, state: Any) -> Any: + captured.append((result, state)) + return await composite_preview(result, state) + + monkeypatch.setattr(pipeline, "_with_composite_preview", capture) + readings: list[Any] = [] + finished = phase_answer.finished_scope + + async def judged(*args: Any, reading: Any, **kwargs: Any) -> Any: + readings.append(reading) + return await finished(*args, reading=reading, **kwargs) + + monkeypatch.setattr(phase_answer, "finished_scope", judged) + offline_resolve( + _SCENARIOS["H1"](), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=_sdmx() + ) + monkeypatch.setattr(phase_answer, "finished_scope", finished) + ((result, state),) = captured + (answer_reading,) = readings + assert answer_reading is not None and answer_reading.table is not None + state.reading = None + degraded = _degraded(monkeypatch, scope_decision) + + decision = asyncio.run( + phase_answer.finished_scope( + state, result.spec, decision=None, minted=None, reading=answer_reading + ) + ) + + assert decision is None + assert degraded == ["scope decision"] + + +def test_a_definite_answer_carries_the_scope_decision_it_states( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The definite answer's producer carries the decision its check returned on the result + (`RegionResult.scope_decision`), and the pass states that decision's scope, whatever the + pass's state holds by then. A definite result carrying none raises.""" + captured: list[Any] = [] + scope_choices = pipeline._with_scope_choices + + def capture(result: Any) -> Any: + captured.append(result) + return scope_choices(result) + + monkeypatch.setattr(pipeline, "_with_scope_choices", capture) + response = offline_resolve( + _SCENARIOS["H1"](), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=_sdmx() + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, DefinitePart), part + (result,) = captured + decision = result.scope_decision + assert decision is not None and decision.composite + assert decision.spec_id == part.spec.spec_id + assert part.spec.resolved.scope_choices == decision.scope + with pytest.raises(TypeError): + scope_choices(dataclasses.replace(result, scope_decision=None)) + + +def _narrowed(start: int, end: int) -> Any: + """`phase_answer._over_conversion_period` narrowing every answer to `start` to `end`, as + a conversion whose tables cover those years alone narrows it (`_narrowed_to_period`).""" + + def over_period( + state: Any, mat_result: Any, filtered: Any, admission: Any, *, settled: Any + ) -> Any: + period = ConversionPeriod( + served=(_TOTAL_YEARS[0], _TOTAL_YEARS[-1]), + years=(start, end), + limited_by="exchange_rates", + ) + mat_result, filtered, _shown = phase_answer._narrowed_to_period( + state, mat_result, admission, period, requested=None + ) + return mat_result, filtered + + return over_period + + +@pytest.mark.parametrize( + ("years", "source", "withheld"), + [ + pytest.param(dict.fromkeys(_HELD, _TOTAL_YEARS), _BILATERAL, [], id="composite-every-year"), + # The bilateral destinations lack 2021, the latest year, but cover 2018 to 2020. + pytest.param( + _COMPOSITE_BEHIND, + None, + [{"why": READ_WINDOW_NOT_COVERED}], + id="composite-behind", + ), + ], +) +def test_a_latest_request_narrowed_to_a_range_keeps_the_reading_it_read( + years: dict[tuple[str, str, str], tuple[int, ...]], + source: str | None, + withheld: list[dict[str, str]], + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A `latest` request elects its reading over the latest year, and a conversion's period + narrows its answer to a range after the read (`_narrowed_to_period`). The answer keeps + the reading it read. Where the range would have read the composite the latest year + withheld, it says so, and no fault is logged.""" + sdmx = _ragged(monkeypatch, years) + monkeypatch.setattr(phase_answer, "_over_conversion_period", _narrowed(2018, 2020)) + (parsed,) = _SCENARIOS["H6"]().input.parts + request = _SCENARIOS["H6"]().model_copy(deep=True) + request.input.parts[0] = parsed.model_copy(update={"time_text": "latest"}) + + with caplog.at_level(logging.WARNING, logger=reading.__name__): + part = _part( + offline_resolve( + request, graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=sdmx + ) + ) + + window = part.spec.definition.identity.requested_window + assert (window.kind, window.start_year, window.end_year) == ("range", 2018, 2020) + assert _pin(part) == source + assert [params for _spec_id, params in _withheld(part)] == withheld + assert _faults(caplog) == [] + + +def test_a_place_set_settling_on_its_members_reads_with_their_election( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A place set read at its node and then at its members elects at each, and the read + the set settles on, the members', carries the members' election and reads within the + population that election names, where the node's read elected none.""" + population = destination_population(get_index(), _DAC2A) + document = _identity(_DAC2A).model_dump(mode="json", exclude_none=True) + document.pop("facets") + document.pop("open_roles") + document["entities"] = {**document["entities"], "destination": [population.total]} + table = TableRead( + identity=BaseIdentity.model_validate(document), populations={"destination": population} + ) + choice = ProviderChoice( + states_scope=True, + bilateral=_BILATERAL, + withheld=None, + selection=ScopeState(), + provider=None, + ) + node, members = empty_place_context(), empty_place_context() + elected = {id(node): None, id(members): ReadingDecision(choice=choice, table=table)} + read_within: list[Any] = [] + + def settle(_place: Any, read: Any) -> Any: + read(node, None) + return members, read(members, None) + + def materialise(*_args: Any, within: Any, **_kwargs: Any) -> Any: + read_within.append(within) + return within + + monkeypatch.setattr(phase_place_materialise, "settle_sets", settle) + monkeypatch.setattr(reading, "elect_reading", lambda *a, **k: elected[id(a[2])]) + monkeypatch.setattr(phase_place_materialise, "run_materialise", materialise) + state = SimpleNamespace( + shape_draft=None, + bindings=(), + default_presence=None, + unsettled_place=None, + place=node, + graph=None, + sdmx=None, + date_request=None, + deadline=None, + coverage=None, + include_observations=False, + stream_observations=False, + scope_selection=ScopeState(), + operations=(), + ) + + async def read() -> Any: + return await phase_place_materialise._materialise_task(state) + + settled = asyncio.run(read()) + + assert settled.place is members + assert settled.reading is elected[id(members)] + assert read_within == [None, {"destination": frozenset(population.members)}] + assert settled.outcome == read_within[-1] + + +@pytest.mark.parametrize("elections", [1, 2], ids=["read-once", "read-twice"]) +def test_a_composite_that_refuses_its_table_read_answers_facet_not_available( + elections: int, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A composite the part's scope compiles to that refuses its table read is logged at + ERROR and noted as a degraded read, so no cache stores the answer, and the part reads + nothing and answers no_data, `facet_not_available`. A part whose reads elect the refused + composite twice logs and notes it once (`reading.noting_refusals_once`).""" + + def refusing(*_args: object, **_kwargs: object) -> object: + raise EngineInputError("refused", code="requested_option_unavailable", status=422) + + monkeypatch.setattr(reading, "composite_table_identity", refusing) + elect = reading.elect_reading + + def electing(*args: Any, **kwargs: Any) -> Any: + for _ in range(elections - 1): + elect(*args, **kwargs) + return elect(*args, **kwargs) + + monkeypatch.setattr(reading, "elect_reading", electing) + degraded = _degraded(monkeypatch, reading) + + with caplog.at_level(logging.ERROR, logger=reading.__name__): + response = offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "facet_not_available" + assert [r.levelno for r in caplog.records if r.name == reading.__name__] == [logging.ERROR] + assert degraded == ["reading election"] + + +@pytest.mark.parametrize( + ("code", "raises"), + [ + # The answer's mint refuses a reversed window, so no election is made for it. + pytest.param("request_invalid", False, id="reversed-window"), + pytest.param("requested_option_unavailable", True, id="other-refusal"), + ], +) +def test_a_reversed_window_elects_no_reading( + code: str, raises: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + def refusing(*_args: object, **_kwargs: object) -> object: + raise EngineInputError("refused", code=code) + + monkeypatch.setattr(reading, "elect_reading", refusing) + + def elect() -> ReadingDecision | None: + return reading.elect_at_read( + cast(Any, None), + (), + empty_place_context(), + date_request=None, + selection=ScopeState(), + coverage=cast(Any, None), + operations=(), + ) + + if raises: + with pytest.raises(EngineInputError): + elect() + else: + assert elect() is None + + +def test_a_stand_in_is_adopted_with_its_own_read_and_election( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A zero-observation stand-in answers through its own read, which elected its provider + reading at its place as the elected shape's read does (`candidates._materialise_alt_shapes`), + so it is adopted with that read and that election and is read no second time.""" + choice = ProviderChoice( + states_scope=True, + bilateral=_BILATERAL, + withheld=None, + selection=ScopeState(), + provider=None, + ) + elected = ReadingDecision(choice=choice, table=None) + + def reread(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("a stand-in is read once") + + monkeypatch.setattr(phase_place_materialise, "run_materialise", reread) + place = empty_place_context() + stand_in = _Candidate( + shape=cast(Any, None), + bindings=[_AID_TYPE_206, _DISBURSEMENT], + mat=cast(Any, "its own read"), + place=place, + walked=None, + date_request=None, + reading=elected, + ) + + adopted = phase_place_materialise._adopted_read(stand_in) + + assert (adopted.outcome, adopted.reading, adopted.place) == ("its own read", elected, place) + + +_AID_TYPE_206 = SlotBindingDraft( + axis="what", property_dcid="dac2aAidType", kind="value", value_dcids=["DAC2aAidType206"] +) +_DISBURSEMENT = SlotBindingDraft( + axis="how", property_dcid="financialFlowType", kind="value", value_dcids=["Disbursement"] +) + + +def test_a_composite_that_cannot_be_minted_answers_no_data_not_the_table( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A part read within the bilateral composite's population whose composite cannot be + minted answers no_data, `facet_not_available`, and logs at ERROR, since the table's + label would state the population's rows as total ODA.""" + + def raising(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("mint failed") + + monkeypatch.setattr(resubmit, "composite_mint", raising) + + with caplog.at_level(logging.ERROR, logger=reading.__name__): + response = offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "facet_not_available" + assert _faults(caplog) diff --git a/tests/engine/test_region_residuals_e2e.py b/tests/engine/test_region_residuals_e2e.py index 42b8edbb4..6dd589e68 100644 --- a/tests/engine/test_region_residuals_e2e.py +++ b/tests/engine/test_region_residuals_e2e.py @@ -398,7 +398,7 @@ def wrapper(**kwargs): return wrapper - for module in (scope_decision, dataset_resubmit): + for module in (scope_decision.scope_build, dataset_resubmit): monkeypatch.setattr(module, "build_part_scope", recording(module.build_part_scope)) first = _ask({}).spec diff --git a/tests/engine/test_request_warning_sites.py b/tests/engine/test_request_warning_sites.py deleted file mode 100644 index 2fd2b0629..000000000 --- a/tests/engine/test_request_warning_sites.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Every write to a fresh pass's warnings is declared, so a definite answer read again within -its composite's population (`pipeline._within_population`) keeps exactly the warnings the -request's words raised. - -`ResolveState.request_warnings` snapshots the pass's warnings before it interprets its read, -and a warning raised from the request's words after that is recorded through -`ResolveState.from_request`. The population read replaces every other warning about the -answer (`composite_answer.with_population_read`), so a write after the snapshot that is -neither derived from the read's rows nor recorded through `from_request` would drop a warning -the request's words raised. The sweep reads every write site under `search/engine` from the -source and holds it to `_SITES`: a new site fails until it is declared, and a site declared -`request` fails unless its function calls `state.from_request`. -""" - -from __future__ import annotations - -import ast -import pathlib -from collections import Counter -from typing import cast - -_ENGINE = pathlib.Path(__file__).resolve().parents[2] / "src" / "search" / "engine" - -# Callables that only read the list they are passed. -_READERS = frozenset( - {"tuple", "len", "list", "set", "frozenset", "sorted", "enumerate", "any", "all", "iter"} -) -_MUTATORS = frozenset({"append", "extend", "insert", "remove", "clear", "pop", "sort", "reverse"}) - -type Site = tuple[str, str, str] - - -def _is_warnings(node: ast.AST, *, module: str) -> bool: - """Whether `node` is a pass's warnings list: `state.warnings`, or `self.warnings` inside - the state module.""" - if not (isinstance(node, ast.Attribute) and node.attr == "warnings"): - return False - owner = node.value - names = {"state", "self"} if module == "region/state" else {"state"} - return isinstance(owner, ast.Name) and owner.id in names - - -def _site_ops(node: ast.AST, *, module: str, writers: frozenset[str]) -> list[str]: - """The writes `node` makes to a pass's warnings, one op each: a list method, a call that - is passed the list, a call of a state method that writes it (`writers`), an alias of the - list, or an assignment to it or an item of it.""" - ops: list[str] = [] - if isinstance(node, ast.Call): - func = node.func - if isinstance(func, ast.Attribute) and _is_warnings(func.value, module=module): - if func.attr in _MUTATORS: - ops.append(func.attr) - elif ( - isinstance(func, ast.Attribute) - and isinstance(func.value, ast.Name) - and func.value.id == "state" - and func.attr in writers - ): - ops.append(f"state.{func.attr}") - passed = [*node.args, *(keyword.value for keyword in node.keywords)] - reader = isinstance(func, ast.Name) and func.id in _READERS - if not reader and any(_is_warnings(arg, module=module) for arg in passed): - ops.append(f"passed:{ast.unparse(func)}") - if ( - isinstance(node, ast.Assign | ast.AnnAssign) - and node.value is not None - and _is_warnings(node.value, module=module) - ): - ops.append("alias") - targets: list[ast.expr] = [] - if isinstance(node, ast.Assign | ast.Delete): - targets = list(node.targets) - elif isinstance(node, ast.AugAssign | ast.AnnAssign): - targets = [node.target] - for target in targets: - held = target.value if isinstance(target, ast.Subscript) else target - if _is_warnings(held, module=module): - ops.append("assign") - return ops - - -def _functions(tree: ast.Module) -> list[ast.FunctionDef | ast.AsyncFunctionDef]: - return [ - node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) - ] - - -def _trees() -> dict[str, ast.Module]: - """Every module under `search/engine`, parsed, by its path below it without suffix.""" - return { - path.relative_to(_ENGINE).with_suffix("").as_posix(): ast.parse( - path.read_text(), filename=str(path) - ) - for path in sorted(_ENGINE.rglob("*.py")) - } - - -def _scanned(trees: dict[str, ast.Module]) -> dict[Site, int]: - """Every write site under `search/engine`, by module, innermost function and op, with - how many times it occurs there.""" - sites: Counter[Site] = Counter() - - def visit(node: ast.AST, module: str, function: str, writers: frozenset[str]) -> None: - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - function = node.name - for op in _site_ops(node, module=module, writers=writers): - sites[(module, function, op)] += 1 - for child in ast.iter_child_nodes(node): - visit(child, module, function, writers) - - visit(trees["region/state"], "region/state", "", frozenset()) - writers = frozenset(function for module, function, _ in sites if module == "region/state") - for module, tree in trees.items(): - if module != "region/state": - visit(tree, module, "", writers) - return dict(sites) - - -_R, _Q, _B, _P, _M = "read", "request", "before", "resubmit", "method" - -# Each write site, by module, function and op, with the kind of each occurrence in source -# order: -# - before: runs before the pass snapshots its warnings (`ResolveState.request_warnings`); -# - read: derived from the rows the read returned, which the population read replaces; -# - request: raised from the request's words after the snapshot, so the function records it -# through `state.from_request`; -# - keeps_key: rewrites a held warning's params or drops held warnings, raising none, so a -# snapshot warning keeps its key (`departures.warning_key`); -# - resubmit: the resubmit pass (`_PromoteState`), which takes no snapshot; -# - method: a state method, whose callers are the sites. -_SITES: dict[Site, tuple[str, ...]] = { - ("region/candidate_collapse", "_assemble_candidate_set", "append"): (_Q,), - ("region/candidate_collapse", "_assemble_candidate_set", "extend"): (_R,), - ("region/candidate_collapse", "_assemble_definite", "append"): (_R,), - ("region/candidate_collapse", "_assemble_definite", "passed:_attach_observations"): (_R, _R), - ("region/candidate_collapse", "_assemble_definite", "passed:install_place_warnings"): (_Q,), - ("region/candidate_collapse", "assemble_candidates", "state.state_reelected_reading"): ( - "keeps_key", - ), - ("region/candidate_collapse", "_handle_sparse_survivors", "append"): (_Q, _Q), - ( - "region/candidate_collapse", - "_offer_place_warnings", - "passed:stamp_offered_place_warnings", - ): (_Q,), - ("region/candidate_grounding", "_warn_for_candidates", "extend"): (_R, _R, _R), - ( - "region/candidate_grounding", - "restate_scope_warning", - "passed:restate_role_scope_warnings", - ): (_R,), - ("region/candidate_options", "_attach_candidate_entity_options", "append"): (_R,), - ("region/candidate_options", "_attach_candidate_facet_options", "append"): (_R,), - ("region/candidate_options", "_attach_candidate_slot_options", "append"): (_R,), - ("region/phase_answer", "_bound_unit_refusal", "append"): (_R,), - ("region/phase_answer", "_finalize_definite_result", "passed:_attach_observations"): (_R, _R), - ("region/phase_answer", "_assemble_definite_answer", "state.state_reelected_reading"): ( - "keeps_key", - ), - ("region/phase_answer", "_gather_definite_coverage", "append"): (_R, _R, _R), - # A member's family total is absent from the rows read; the answer is no_data. - ("region/phase_answer", "_ground_elected_answer", "state.depart"): (_R,), - ("region/phase_answer", "_ground_elected_answer", "extend"): (_R,), - ( - "region/phase_answer", - "_install_filtered_answer", - "passed:restate_role_scope_warnings", - ): (_R,), - # The years the request asked beside those the conversion tables cover; the answer's - # identity holds the narrowed window, so its population read departs nothing. - ("region/phase_answer", "_narrowed_to_period", "state.depart"): (_Q,), - # A trailing request's stale and passed-over years name the span its words asked, then - # the deflator omission the read's plan gives. - ("region/phase_answer", "_over_conversion_period", "state.depart"): (_Q, _Q, _Q, _R), - ("region/phase_answer", "_start_assemble_definite", "append"): (_R,), - ("region/phase_election", "_install_settled", "extend"): (_B,), - ("region/phase_election", "_repair_bindings", "append"): (_B, _B), - ("region/phase_election", "_repair_bindings", "extend"): (_B, _B), - ("region/phase_election", "_vet_election", "append"): (_B,), - ("region/phase_place_materialise", "_install_shape", "passed:install_place_warnings"): (_B,), - ("region/phase_place_materialise", "_unbind_refuted_claims", "state.forget_bound"): ( - "keeps_key", - ), - ("region/phase_place_materialise", "_nearest_real_for", "append"): (_R,), - ("region/phase_place_materialise", "_nearest_real_for", "extend"): (_R,), - ("region/phase_place_materialise", "_nearest_real_for", "passed:_attach_observations"): (_R,), - ( - "region/phase_place_materialise", - "_nearest_real_for", - "passed:stamp_offered_place_warnings", - ): (_Q,), - ("region/phase_place_materialise", "_note_named_places", "append"): (_B,), - ( - "region/phase_place_materialise", - "_phase_materialise", - "passed:interpret_materialised", - ): (_R,), - ("region/phase_place_materialise", "_phase_materialise", "passed:settle_place"): (_R,), - ("region/phase_place_materialise", "_phase_place_bindings", "append"): (_B,), - ("region/phase_place_materialise", "_reelect_zero_observation_sibling", "append"): (_B,), - ("region/phase_place_materialise", "note_denominator_unverified", "append"): (_Q,), - ("region/phase_rank_bind", "_phase_bind", "append"): (_B,), - ("region/phase_rank_bind", "_phase_prune_and_rank", "extend"): (_B,), - ("region/phase_rank_bind", "_read_availability", "append"): (_B,), - ("region/phase_shared", "_elected_flagged_match", "extend"): (_R,), - # The raw-text door's unit phrase the answer does not apply. - ("region/phase_shared", "withdraw_typed_unit", "state.depart"): (_Q,), - ("region/promote_finalize", "_bound_unit_refusal", "append"): (_P,), - ("region/promote_ground", "_ground_and_build_spec", "state.depart"): (_P,), - ( - "region/promote_ground", - "_install_narrowed_answer", - "passed:restate_role_scope_warnings", - ): (_P,), - ("region/promote_ground", "_narrowed_to_period", "append"): (_P,), - ("region/promote_ground", "_narrowed_to_period", "state.depart"): (_P,), - ("region/promote_ground", "_over_conversion_period", "append"): (_P,), - ("region/promote_ground", "_over_conversion_period", "state.depart"): (_P,), - ("region/promote_ground", "_phase_ground_build", "state.depart"): (_P,), - ("region/promote_materialise", "_phase_materialise", "passed:interpret_materialised"): (_P,), - ("region/promote_materialise", "_phase_materialise", "passed:settle_place"): (_P,), - ("region/promote_places", "_phase_where_places", "extend"): (_P,), - ("region/promote_reread", "_phase_reread_arcs", "append"): (_P,), - ("region/promote_shape", "_phase_build_shape", "alias"): (_P,), - ("region/state", "depart", "append"): (_M,), - ("region/state", "forget_bound", "assign"): (_M,), - ("region/state", "state_reelected_reading", "assign"): (_M,), - ("region/unrealisable", "unbind_unrealisable", "state.forget_bound"): ("keeps_key",), -} - - -def _function(trees: dict[str, ast.Module], module: str, name: str) -> ast.AST: - (function,) = [node for node in _functions(trees[module]) if node.name == name] - return function - - -def test_every_write_to_a_pass_s_warnings_is_declared() -> None: - """A write site the table does not declare, or declares a different number of times, - fails until its kind is declared.""" - declared = {site: len(kinds) for site, kinds in _SITES.items()} - - assert _scanned(_trees()) == declared - - -def test_a_request_site_records_its_warning_through_from_request() -> None: - """A function declaring a warning raised from the request's words after the snapshot - calls `state.from_request`, so a population read keeps the warning.""" - trees = _trees() - unrecorded = [ - (module, function) - for (module, function, _), kinds in _SITES.items() - if _Q in kinds - and not any( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == "from_request" - and isinstance(node.func.value, ast.Name) - and node.func.value.id == "state" - for node in ast.walk(_function(trees, module, function)) - ) - ] - - assert unrecorded == [] - - -def test_a_resubmit_site_writes_the_resubmit_pass_s_warnings() -> None: - """A site declared `resubmit` writes a `_PromoteState`, the resubmit pass's state, - which takes no snapshot.""" - trees = _trees() - misplaced = [ - (module, function) - for (module, function, _), kinds in _SITES.items() - if _P in kinds - and not any( - arg.arg == "state" - and arg.annotation is not None - and ast.unparse(arg.annotation) == "_PromoteState" - for arg in cast("ast.FunctionDef", _function(trees, module, function)).args.args - ) - ] - - assert misplaced == [] diff --git a/tests/engine/test_resolved_scope.py b/tests/engine/test_resolved_scope.py index a2250591f..7c1f4f398 100644 --- a/tests/engine/test_resolved_scope.py +++ b/tests/engine/test_resolved_scope.py @@ -181,11 +181,13 @@ def _minimal_build_spec_kwargs() -> dict: def test_scope_is_never_folded_into_identity_or_spec_id(): """Scope belongs to resolved data and does not change `spec_id` or identity.""" kwargs = _minimal_build_spec_kwargs() - bare = build_spec(measure_elected_properties=frozenset(), **kwargs) + bare = build_spec(composite=None, measure_elected_properties=frozenset(), **kwargs) scope = build_scope( "observationAbout", ["country/A1", "country/A2"], {"country/A1": "Testland One"}, {} ) - scoped = build_spec(measure_elected_properties=frozenset(), **kwargs, scope=scope) + scoped = build_spec( + composite=None, measure_elected_properties=frozenset(), **kwargs, scope=scope + ) assert scoped.spec_id == bare.spec_id assert scoped.definition.identity == bare.definition.identity diff --git a/tests/engine/test_role_axis_facts.py b/tests/engine/test_role_axis_facts.py index c68655873..52cde2679 100644 --- a/tests/engine/test_role_axis_facts.py +++ b/tests/engine/test_role_axis_facts.py @@ -73,6 +73,7 @@ def _bound_destination_open_origin_result(monkeypatch) -> Materialised: sdmx=FakeSdmxClient(observations={}), # never read on this branch discover_scope=True, coverage_snapshot=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) return result @@ -134,6 +135,7 @@ def test_bound_destination_open_origin_makes_no_extra_sdmx_call(monkeypatch): sdmx=sdmx, discover_scope=True, coverage=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -173,6 +175,7 @@ def test_both_axes_open_derives_the_same_fact_from_role_index_by_sv(monkeypatch) sdmx=sdmx, discover_scope=True, coverage_snapshot=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_role_bearing_consolidation.py b/tests/engine/test_role_bearing_consolidation.py index 46b9f546d..40ea6b231 100644 --- a/tests/engine/test_role_bearing_consolidation.py +++ b/tests/engine/test_role_bearing_consolidation.py @@ -113,6 +113,7 @@ def test_declaring_shape_uses_the_sdmx_branch_only(self): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) from search.engine.retrieval.retrieve import Materialised @@ -134,6 +135,7 @@ def test_non_declaring_shape_routes_through_the_generic_branch(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) from search.engine.retrieval.retrieve import Materialised diff --git a/tests/engine/test_role_bearing_resolve.py b/tests/engine/test_role_bearing_resolve.py index 5a432da7d..45b08dad7 100644 --- a/tests/engine/test_role_bearing_resolve.py +++ b/tests/engine/test_role_bearing_resolve.py @@ -107,6 +107,7 @@ def test_declared_names_key_the_facet_grouping(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -134,6 +135,7 @@ def test_materialise_confirms_against_every_named_destination(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), # coverage supplies role facets coverage=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -166,6 +168,7 @@ def test_materialise_headline_reflects_both_named_destinations(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), # coverage supplies role facets coverage=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -197,6 +200,7 @@ def test_materialise_windowed_to_a_year_with_no_data_reports_no_data(monkeypatch sdmx=FakeSdmxClient(observations={}), date_request=DateRequest(window=TimeWindow(start_year=2015, end_year=2015), latest=False), coverage=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -227,6 +231,7 @@ def test_materialise_windowed_to_a_year_with_data_reports_the_exact_count(monkey sdmx=FakeSdmxClient(observations={}), date_request=DateRequest(window=TimeWindow(start_year=2022, end_year=2022), latest=False), coverage=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -263,6 +268,7 @@ def test_materialise_windowed_count_sums_across_donor_recipient_rows_sharing_a_f sdmx=FakeSdmxClient(observations={}), date_request=DateRequest(window=TimeWindow(start_year=2021, end_year=2022), latest=False), coverage=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -288,6 +294,7 @@ def test_materialise_confirms_when_only_the_second_named_origin_has_data(monkeyp graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), # coverage supplies role facets coverage=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -308,6 +315,7 @@ def test_no_matching_coverage_reads_no_sdmx_series(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -332,6 +340,7 @@ def test_a_shape_declaring_no_role_is_refused_before_any_read(): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert sdmx.series_calls == [] @@ -349,6 +358,7 @@ def test_unplaceable_declared_roles_refuse_role_incompatible(): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) @@ -369,6 +379,7 @@ def test_materialise_forks_on_the_declaration_alone(): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(role_result, Materialised) assert role_result.available_at_place is False @@ -383,6 +394,7 @@ def test_materialise_forks_on_the_declaration_alone(): graph=graph, sdmx=FakeSdmxClient(observations={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(non_role_result, Materialised) @@ -408,6 +420,7 @@ def test_materialise_reads_entities_by_the_declared_names(): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations=observations), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(correctly_keyed, Materialised) assert correctly_keyed.sv_dcids == [_SV_DCID] @@ -419,6 +432,7 @@ def test_materialise_reads_entities_by_the_declared_names(): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations=observations), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(wrongly_keyed, NoDataDraft) assert wrongly_keyed.reason == "variable_not_resolved" @@ -480,6 +494,7 @@ def test_no_data_draft_sv_dcids_excludes_constraint_mismatched_members(): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -533,6 +548,7 @@ def test_no_data_draft_sv_dcids_preserves_specificity_order(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -562,6 +578,7 @@ def test_no_data_draft_sv_dcids_preserves_specificity_order(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=coverage, + within=None, ) assert isinstance(with_data, Materialised) @@ -617,6 +634,7 @@ def test_member_refining_another_over_an_unbound_constraint_is_dropped(monkeypat graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -674,6 +692,7 @@ def test_admitted_counts_the_bindings_admitted_population_named_destination(monk graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -708,6 +727,7 @@ def test_admitted_counts_the_bindings_admitted_population_discovered_scope(monke sdmx=sdmx, discover_scope=True, coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -737,6 +757,7 @@ def test_named_destination_resolve_reports_the_shapes_own_member_count(monkeypat graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -788,6 +809,7 @@ def test_no_break_collects_every_destination_confirmed_member(monkeypatch): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -876,6 +898,7 @@ def test_a_refresh_that_times_out_keeps_frozen_rows_and_reports_unverified(monke sdmx=FakeSdmxClient(observations={}), deadline=123.0, coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) @@ -922,6 +945,7 @@ def test_the_refresh_opens_a_deadline_scope_matching_the_callers_own_deadline( sdmx=sdmx, deadline=555.5, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1024,6 +1048,7 @@ def test_an_open_creditor_elects_from_the_index_family_beyond_the_recalled_shape sdmx=FakeSdmxClient(observations={}), open_axes=frozenset({"origin"}), coverage_snapshot=coverage, + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_role_class_shapes.py b/tests/engine/test_role_class_shapes.py index c9d054c2a..90cd337de 100644 --- a/tests/engine/test_role_class_shapes.py +++ b/tests/engine/test_role_class_shapes.py @@ -104,6 +104,7 @@ ResubmitIdentityPayload, RoleAxis, ) +from search.scope.state import ScopeState from series_coverage.schema import ABSENT_ENTITY, RoleNames from series_coverage.shared import CoverageSnapshot from statvar_members import MemberEnumerator, member_digest @@ -2917,6 +2918,8 @@ def test_an_alternate_whose_values_no_member_holds_together_unbinds_a_minimal_se sdmx=FakeSdmxClient(observations={}, availability={}), date_request=None, coverage=_one_role_coverage(_RELAX_NODES, _RELAX_OBS), + selection=ScopeState(), + operations=(), ) (candidate,) = read.candidates diff --git a/tests/engine/test_role_currentness.py b/tests/engine/test_role_currentness.py index 16d8164dd..092ab86ad 100644 --- a/tests/engine/test_role_currentness.py +++ b/tests/engine/test_role_currentness.py @@ -104,6 +104,7 @@ def _resolve( sdmx=sdmx, discover_scope=False, coverage=coverage if coverage is not None else empty_coverage_snapshot(), + within=None, ) @@ -616,6 +617,7 @@ def node_observation_properties_batch(self, dcids: list[str]) -> dict[str, tuple graph=graph, sdmx=sdmx, coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -696,6 +698,7 @@ def test_slot_bindings_do_not_change_the_verdict() -> None: graph=FakeGraph(raise_on_call=True), sdmx=_sdmx(_SV_DECLARING), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_role_grid_search.py b/tests/engine/test_role_grid_search.py index 125f6a886..50b443e77 100644 --- a/tests/engine/test_role_grid_search.py +++ b/tests/engine/test_role_grid_search.py @@ -681,6 +681,7 @@ def test_a_member_elected_under_one_open_key_states_the_mints_count_for_both_axe }, ) ).snapshot(), + within=None, ) assert resolution.authority.settled diff --git a/tests/engine/test_role_resolve_financing_roles.py b/tests/engine/test_role_resolve_financing_roles.py index ad83013da..a75156112 100644 --- a/tests/engine/test_role_resolve_financing_roles.py +++ b/tests/engine/test_role_resolve_financing_roles.py @@ -59,6 +59,7 @@ def test_materialise_resolves_a_financing_role_bearing_shape(monkeypatch) -> Non graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), # coverage supplies role facets coverage=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -93,6 +94,7 @@ def test_materialise_two_destinations_sharing_one_facet_id_report_one_source( graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -128,6 +130,7 @@ def test_materialise_refuses_role_not_resolved_for_an_unplaceable_role() -> None graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) @@ -256,6 +259,7 @@ def test_zero_observation_role_bearing_resolve_returns_the_flagged_carrier(monke graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(with_data, Materialised) assert with_data.has_data is True @@ -269,6 +273,7 @@ def test_zero_observation_role_bearing_resolve_returns_the_flagged_carrier(monke graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=FakeCoverageService(_auto_coverage_index_2).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -293,6 +298,7 @@ def test_zero_observation_carrier_reports_a_capped_population_as_inexact() -> No graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_role_scope_discovery.py b/tests/engine/test_role_scope_discovery.py index 1142c1746..e75da23cd 100644 --- a/tests/engine/test_role_scope_discovery.py +++ b/tests/engine/test_role_scope_discovery.py @@ -148,6 +148,7 @@ def _resolve(shape: ShapeDraft, sdmx: FakeSdmxClient, **kwargs): discover_scope=True, coverage_snapshot=coverage_snapshot, **kwargs, + within=None, ) @@ -177,6 +178,7 @@ def test_discover_scope_off_by_default_still_variable_not_resolved(): graph=FakeGraph(raise_on_call=True), sdmx=sdmx, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) @@ -268,6 +270,7 @@ def test_multiple_origins_carry_the_full_opposite_role_filter(monkeypatch): sdmx=sdmx, discover_scope=True, coverage=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -294,6 +297,7 @@ def test_origin_only_signature_without_discovery_stays_role_incompatible(): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}, availability={}), coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(direct, NoDataDraft) assert direct.reason == "role_incompatible" @@ -305,6 +309,7 @@ def test_origin_only_signature_without_discovery_stays_role_incompatible(): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(via_materialise, NoDataDraft) assert via_materialise.reason == "role_not_resolved" @@ -344,6 +349,7 @@ def test_origin_only_signature_discovers_the_origin_axis(monkeypatch, seed): sdmx=FakeSdmxClient(observations=observations, availability={}), discover_scope=True, coverage_snapshot=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(direct, Materialised) assert direct.discovered_scope.role_name == "financingProvider" @@ -358,6 +364,7 @@ def test_origin_only_signature_discovers_the_origin_axis(monkeypatch, seed): sdmx=FakeSdmxClient(observations=observations, availability={}), discover_scope=True, coverage=FakeCoverageService(_auto_coverage_index_1).snapshot(), + within=None, ) assert isinstance(via_materialise, Materialised) @@ -390,6 +397,7 @@ def test_origin_only_signature_confirms_at_its_bound_origin(seed, discover_scope coverage_snapshot=FakeCoverageService( build_coverage_index(rows, role_names=role_names) ).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -443,6 +451,7 @@ def test_a_bound_origin_confirms_the_donor_total_beside_an_optional_recipient(di sdmx=FakeSdmxClient(observations={}, availability={}), discover_scope=discover_scope, coverage_snapshot=FakeCoverageService(coverage).snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -470,6 +479,7 @@ def test_origin_only_signature_with_named_subject_skips_discovery(): sdmx=sdmx, discover_scope=True, coverage_snapshot=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) diff --git a/tests/engine/test_role_twin_resolution.py b/tests/engine/test_role_twin_resolution.py index d3e5cceca..547ef6bc0 100644 --- a/tests/engine/test_role_twin_resolution.py +++ b/tests/engine/test_role_twin_resolution.py @@ -318,6 +318,7 @@ def test_debtor_without_creditor_resolves_to_the_total() -> None: sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), discover_scope=True, + within=None, ) assert isinstance(result, Materialised) @@ -341,6 +342,7 @@ def test_an_open_creditor_resolves_to_the_breakdown_and_opens_the_creditor_axis( coverage=_twin_coverage().snapshot(), discover_scope=True, open_axes=frozenset({"origin"}), + within=None, ) assert isinstance(result, Materialised) @@ -366,6 +368,7 @@ def test_an_open_creditor_on_a_recipient_only_family_still_answers_the_total( discover_scope=True, open_axes=open_axes, coverage_snapshot=_twin_coverage().snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -384,6 +387,7 @@ def test_bound_creditor_resolves_to_the_breakdown() -> None: graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -404,6 +408,7 @@ def test_admission_requires_a_bound_origin_the_shape_declares_no_role_for() -> N graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage_snapshot=_twin_coverage().snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) @@ -429,6 +434,7 @@ def test_breakdown_with_an_extra_unbound_constraint_gives_way_to_the_total() -> graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -457,6 +463,7 @@ def test_a_lender_named_member_of_other_constraint_values_gives_way_to_the_total graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -483,6 +490,7 @@ def test_members_whose_role_sets_do_not_nest_still_refuse() -> None: sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), discover_scope=True, + within=None, ) assert isinstance(result, NoDataDraft) diff --git a/tests/engine/test_role_twin_unit_roles.py b/tests/engine/test_role_twin_unit_roles.py index 35ecd2ba8..10d48bb60 100644 --- a/tests/engine/test_role_twin_unit_roles.py +++ b/tests/engine/test_role_twin_unit_roles.py @@ -109,6 +109,7 @@ def total_result(twin_shape: ShapeDraft) -> Materialised: graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), + within=None, ) assert isinstance(result, Materialised) assert result.sv_dcids == [_TOTAL] @@ -242,6 +243,7 @@ def test_candidate_path_unit_decision_hands_sdmx_the_survivors_roles( _destination_place(twin_shape), None, state.date_request, + reading=None, ) ], ) @@ -271,7 +273,12 @@ def _capture(sv_dcids, *, role_bindings: dict[str, list[str]], **_: object): with pytest.raises(_StopAfterRequest): _prepare_candidate_context( - state, [_Candidate(twin_shape, [], total_result, own_places, None, state.date_request)] + state, + [ + _Candidate( + twin_shape, [], total_result, own_places, None, state.date_request, reading=None + ) + ], ) assert seen == [{"financingRecipient": [_DESTINATION, "country/UGA"]}] @@ -340,6 +347,7 @@ def _decide( graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}), coverage=_twin_coverage().snapshot(), + within=None, ) assert isinstance(total, Materialised) request = IdentityUnitRequest( @@ -351,7 +359,13 @@ def _decide( state, [ _Candidate( - twin_shape, [], total, _destination_place(twin_shape), None, state.date_request + twin_shape, + [], + total, + _destination_place(twin_shape), + None, + state.date_request, + reading=None, ) ], ) diff --git a/tests/engine/test_same_shape.py b/tests/engine/test_same_shape.py index 0096c4d8c..b28fa0f42 100644 --- a/tests/engine/test_same_shape.py +++ b/tests/engine/test_same_shape.py @@ -76,6 +76,7 @@ def _role_bound_region( ) facts = (RoleAxisFactInput(role="origin", member_dcids=origin_dcids),) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -241,6 +242,7 @@ def _where_bound_spec_with_member_count( role=EntityRoleDirectional(role=_gr(_RECIPIENT_PROP), axis="destination"), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -326,6 +328,7 @@ def _roster_variant_spec( role=EntityRoleDirectional(role=_gr(_RECIPIENT_PROP), axis="destination"), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -415,6 +418,7 @@ def _what_bound_spec_with_member_count( binding=BindingValue(value=SlotValue(ref=_gr(value_dcid), value_kind="enum_value")), ) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -507,6 +511,7 @@ def _g7_region(recipient_dcid: str, *, earliest_index: int) -> RegionResult: *(Entity(ref=_gr(dcid), role=donor) for dcid in sorted(_G7.member_dcids)), ] spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, diff --git a/tests/engine/test_set_ref.py b/tests/engine/test_set_ref.py index ed6e9e00d..699b47178 100644 --- a/tests/engine/test_set_ref.py +++ b/tests/engine/test_set_ref.py @@ -255,6 +255,7 @@ def _entity_slot(axis: Axis, val: str) -> Slot: def _region(shape: Shape, slot: Slot, sv: str = "sv/X", index: int = 0) -> RegionResult: spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, diff --git a/tests/engine/test_setaxes_multi.py b/tests/engine/test_setaxes_multi.py index 8d15dd81e..d5e7e07d6 100644 --- a/tests/engine/test_setaxes_multi.py +++ b/tests/engine/test_setaxes_multi.py @@ -103,6 +103,7 @@ def test_only_the_member_asserting_both_axes_survives(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -143,6 +144,7 @@ def test_discordant_members_are_not_combined_as_a_false_co_answer(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, BoundValuesUnrealisable) @@ -168,6 +170,7 @@ def test_member_matching_neither_axis_explicitly_is_not_collected(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) diff --git a/tests/engine/test_shape_label_measure_agreement.py b/tests/engine/test_shape_label_measure_agreement.py index 58b758e4c..49aebd11d 100644 --- a/tests/engine/test_shape_label_measure_agreement.py +++ b/tests/engine/test_shape_label_measure_agreement.py @@ -97,6 +97,7 @@ def test_served_gross_rows_publish_a_composed_label_not_either_members_own_name( netting_slot = value_slot("how", "nettingBasis", "netting basis", "Gross", "gross") spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -169,6 +170,7 @@ def test_a_reading_with_no_partition_is_labelled_by_the_one_series_it_serves( ) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -218,6 +220,7 @@ def test_an_unlabelled_partition_with_nothing_to_compose_falls_back_to_a_served_ values=(("measuredProperty", "grams"),), members=("SV_grams",) ) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -260,6 +263,7 @@ def test_a_group_agreeing_on_an_open_axis_names_it_but_never_a_varying_one(): donor_slot = value_slot("what", "syntheticDonor", "donor", "syntheticDonorValue") spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -317,6 +321,7 @@ def test_a_unit_request_keeps_the_native_price_basis_in_the_axis_values(): ) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, @@ -367,6 +372,7 @@ def test_a_served_member_name_never_moves_spec_id(): def spec_serving(name: str | None): return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, diff --git a/tests/engine/test_skills.py b/tests/engine/test_skills.py index 420e18cd0..aef8b79f1 100644 --- a/tests/engine/test_skills.py +++ b/tests/engine/test_skills.py @@ -87,6 +87,7 @@ def _spec_with( equivalent_readings: list[Spec] | None = None, ) -> Spec: return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=shape, diff --git a/tests/engine/test_slot_expansion.py b/tests/engine/test_slot_expansion.py index 6f857d3ba..5e6a5a684 100644 --- a/tests/engine/test_slot_expansion.py +++ b/tests/engine/test_slot_expansion.py @@ -213,6 +213,7 @@ def test_a_parent_code_crs_ask_returns_data_instead_of_no_data(): graph=FakeGraph(raise_on_call=True), sdmx=_role_sdmx(), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -267,6 +268,7 @@ def test_standard_path_reads_a_member_in_the_subtree(): graph=_graph_with_crs_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -296,6 +298,7 @@ def test_standard_path_refuses_a_sibling_outside_the_subtree(): graph=_graph_with_crs_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, BoundValuesUnrealisable) diff --git a/tests/engine/test_slot_expansion_wire.py b/tests/engine/test_slot_expansion_wire.py index 87843ec8b..20b4fd8f4 100644 --- a/tests/engine/test_slot_expansion_wire.py +++ b/tests/engine/test_slot_expansion_wire.py @@ -130,6 +130,7 @@ def test_apply_declaration_version_stamps_a_plain_spec_the_index_resolves(): ] shape = shape_five_tuple("FinancialFlow", "amount", "Nominal") spec = build_spec( + composite=None, open_roles=(), shape=shape, slots=slots, diff --git a/tests/engine/test_slot_summable.py b/tests/engine/test_slot_summable.py index 8b3cf33f2..cdc20d6f6 100644 --- a/tests/engine/test_slot_summable.py +++ b/tests/engine/test_slot_summable.py @@ -296,6 +296,7 @@ class TestBuildSpec: def _spec(self, slots: list[Slot], *, pop: str = "EconomicActivity", mq: str | None = None): shape = shape_five_tuple(pop, "amount", mq) return build_spec( + composite=None, open_roles=(), shape=shape, slots=slots, diff --git a/tests/engine/test_spec_build.py b/tests/engine/test_spec_build.py index 70838e102..ae90d2c44 100644 --- a/tests/engine/test_spec_build.py +++ b/tests/engine/test_spec_build.py @@ -154,6 +154,7 @@ def _shape(self): def _build(self, stat_vars: list[StatVar], **kwargs) -> Spec: return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=self._shape(), @@ -522,6 +523,7 @@ def _spec(self, origin_dcids: list[str], *, named_groups: tuple[NamedGroup, ...] entities = [Entity(ref=GraphRef(ref=d, label=d), role=origin) for d in origin_dcids] entities.append(Entity(ref=GraphRef(ref="country/ETH", label="Ethiopia"), role=destination)) return build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=build_shape_model( diff --git a/tests/engine/test_standard_resolve.py b/tests/engine/test_standard_resolve.py index 03613d26e..685e089a8 100644 --- a/tests/engine/test_standard_resolve.py +++ b/tests/engine/test_standard_resolve.py @@ -27,6 +27,7 @@ from search.engine.stated_words_read import StatedWordsOutput from search.engine.time_bind import parse_time_text from search.models import CoverageBare, TimeWindow +from search.scope.state import ScopeState from tests.engine._binders import elected_reply from tests.engine._fake_coverage import empty_coverage_snapshot from tests.engine._fake_sdmx import FakeSdmxClient @@ -63,6 +64,7 @@ def test_a_shape_with_no_declarations_resolves_by_subject_probe(): graph=FakeGraph(raise_on_call=True), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) @@ -200,6 +202,7 @@ def test_values_no_member_holds_refuse(self, shape, bindings, conflicts): graph=_graph_with_data_for(*shape.sv_arc_facts), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, BoundValuesUnrealisable) @@ -217,6 +220,7 @@ def test_a_value_a_member_holds_reads_it(self, binding): graph=_graph_with_male_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -372,6 +376,7 @@ def test_unresolved_bound_property_excludes_sole_member(self): graph=_graph_with_unresolved_gender_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -398,6 +403,7 @@ def test_degenerate_value_binding_refuses_despite_positive_data_for_broadened_me graph=_graph_with_male_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) @@ -417,6 +423,7 @@ def test_degenerate_set_binding_also_refuses(self): graph=_graph_with_male_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, NoDataDraft) @@ -590,6 +597,8 @@ def test_an_alternate_reads_without_values_no_member_holds(self): sdmx=FakeSdmxClient(observations={}, availability={}), date_request=None, coverage=empty_coverage_snapshot(), + selection=ScopeState(), + operations=(), ) assert unsupported is None @@ -634,6 +643,7 @@ def test_a_resubmit_keeps_the_posted_value_a_member_holds(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(refusal, BoundValuesUnrealisable) @@ -962,6 +972,7 @@ def test_both_members_with_data_are_both_collected_in_score_order(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -990,6 +1001,7 @@ def test_only_the_member_with_data_is_collected(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1080,6 +1092,7 @@ def test_unbound_entity_probe_preserves_insertion_order_and_single_match( graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1127,6 +1140,7 @@ def test_no_entity_bound_member_is_representative(shape, bindings, graph, expect graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1154,6 +1168,7 @@ def test_explicit_member_wins_over_bare_aggregate_when_both_have_data(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1179,6 +1194,7 @@ def test_asserting_member_flags_unavailable_when_only_silent_sibling_has_data(se graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1227,6 +1243,7 @@ def test_explicit_member_wins_with_defaulted_scope(self, place_dcids): sdmx=FakeSdmxClient(observations={}, availability={}), place_dcids=place_dcids, coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1251,6 +1268,7 @@ def test_unbound_axis_keeps_original_insertion_order_with_no_place(self): graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1316,6 +1334,7 @@ def test_split_ownership_no_member_asserts_both_degrades_honestly(self): graph=_graph_with_data_for(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1361,6 +1380,7 @@ def test_split_ownership_no_member_asserts_both_degrades_honestly_with_defaulted sdmx=FakeSdmxClient(observations={}, availability={}), place_dcids=["country/A1"], coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1390,6 +1410,7 @@ def test_single_member_resolve_reports_the_shapes_own_member_count(self): graph=_graph_with_male_data(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) dims = result.coverage.dimensions @@ -1414,6 +1435,7 @@ def test_multi_member_set_binding_resolve_counts_every_collected_member(self): graph=_graph_with_data_for(_SET_SV_A, _SET_SV_B), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) dims = result.coverage.dimensions @@ -1478,6 +1500,7 @@ def _resolve_at_recipient(shape: ShapeDraft, bindings: list[SlotBindingDraft], g graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) @@ -1593,6 +1616,7 @@ def test_un_family_without_a_place_names_every_kept_member_without_a_notice(): graph=_graph_with_data_for(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -1681,6 +1705,7 @@ def _resolve_in_years(graph: FakeGraph): sdmx=FakeSdmxClient(observations={}, availability={}), date_request=DateRequest(window=TimeWindow(start_year=2022, end_year=2024), latest=False), coverage=empty_coverage_snapshot(), + within=None, ) @@ -1821,6 +1846,7 @@ def test_the_no_place_path_elects_the_root(self): graph=_graph_with_data_for(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -2297,6 +2323,7 @@ def test_the_no_place_path_names_every_kept_member(self): graph=_graph_with_data_for(), sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), + within=None, ) assert isinstance(result, Materialised) @@ -2508,6 +2535,7 @@ def facet(facet_id: str, years: range) -> dict: sdmx=FakeSdmxClient(observations={}, availability={}), date_request=date_request, coverage=empty_coverage_snapshot(), + within=None, ) @@ -3339,6 +3367,7 @@ def _sources_in( sdmx=FakeSdmxClient(observations={}, availability={}), coverage=empty_coverage_snapshot(), date_request=date_request, + within=None, ) assert isinstance(result, Materialised) return result diff --git a/tests/engine/test_streaming.py b/tests/engine/test_streaming.py index b06f0f61f..1b670bc0c 100644 --- a/tests/engine/test_streaming.py +++ b/tests/engine/test_streaming.py @@ -954,6 +954,7 @@ def _same_shape_region_with_obs_task( """Build a same-shape result with a pending observations task and relaunch callback.""" stat_var_ref = GraphRef(ref=sv_dcid, label=sv_dcid) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_SAME_SHAPE, @@ -1086,6 +1087,7 @@ def _same_shape_region_with_place_options_task( """Build a same-shape result with a pending place-options task and relaunch callback.""" stat_var_ref = GraphRef(ref=sv_dcid, label=sv_dcid) spec = build_spec( + composite=None, open_roles=(), measure_elected_properties=frozenset(), shape=_SAME_SHAPE, diff --git a/tests/engine/test_window_answer.py b/tests/engine/test_window_answer.py index 6faaa6c8f..cca9b0775 100644 --- a/tests/engine/test_window_answer.py +++ b/tests/engine/test_window_answer.py @@ -951,7 +951,6 @@ def test_a_trailing_answer_states_the_years_its_words_cover_once( monkeypatch.setattr(phase_answer, "deflator_omission_departure", lambda *_a, **_k: None) departed: list[DepartureFact] = [] state = SimpleNamespace( - from_request=lambda *_warnings: None, unit_request=object(), unit_elected_selections={}, date_request=settled.date_request, @@ -989,7 +988,6 @@ def start(_state: SimpleNamespace, *, date_request: DateRequest, **_kwargs: obje monkeypatch.setattr(related_read_module, "_elected_measure_family", lambda *_a: (None, ())) monkeypatch.setattr(related_read_module, "start_related_read", start) state = SimpleNamespace( - from_request=lambda *_warnings: None, unit_request=object(), unit_elected_selections={}, date_request=_trailing("the last five years"), @@ -1037,7 +1035,6 @@ async def decide(_state: object, _mat: object) -> RegionResult: else DateRequest(window=TimeWindow(start_year=2015, end_year=2020), latest=False) ) state = SimpleNamespace( - from_request=lambda *_warnings: None, mat_result=_two_flows(range(2000, 2025)), date_request=request, related_read=None, @@ -1079,7 +1076,6 @@ async def decide(_state: object, _mat: object) -> RegionResult: related_read_module, "start_related_read", lambda *_a, **_k: started.append(_k) ) state = SimpleNamespace( - from_request=lambda *_warnings: None, mat_result=dataclasses.replace(_two_flows(range(2000, 2025)), sv_dcids=[_FLOW]), date_request=request, related_read=held, diff --git a/tests/materialise/test_bundle.py b/tests/materialise/test_bundle.py index 7610b688a..035ea2e73 100644 --- a/tests/materialise/test_bundle.py +++ b/tests/materialise/test_bundle.py @@ -13,6 +13,8 @@ import pytest import yaml +from code_index.declarations import DAC2A_BILATERAL, DAC2A_INCLUSIVE +from code_index.shared import get_index from materialise.bundle import ( DATAPACKAGE_PROFILE_URL, ArtifactEntry, @@ -27,7 +29,7 @@ SourceProvenance, assemble_bundle, ) -from materialise.bundle_sources import source_provenance +from materialise.bundle_sources import composite_summary, source_provenance from materialise.errors import MaterialisationUsageError from materialise.license_type import LicenceIds, licence_ids @@ -576,3 +578,16 @@ def test_a_data_cell_missing_from_the_codes_table_fails_validation(tmp_path): descriptor_path = _write_bundle(tmp_path, _standalone(codes=(_CODES[0],))) report = frictionless.validate(str(descriptor_path)) assert {error_type for (error_type,) in report.flatten(["type"])} == {"foreign-key"} + + +def test_the_inclusive_composite_states_the_bilateral_reading_beside_its_bilateral_component_only(): + index = get_index() + reading = "without core contributions to multilateral agencies" + + bilateral = composite_summary(index, index.composite(DAC2A_BILATERAL)) + inclusive = composite_summary(index, index.composite(DAC2A_INCLUSIVE)) + + assert bilateral.endswith(f"{reading}.") + assert not inclusive.endswith(f"{reading}.") + assert inclusive.count(reading) == 1 + assert inclusive.index(reading) < inclusive.index("imputed") diff --git a/tests/models/test_edge_cases.py b/tests/models/test_edge_cases.py index 14c5ef168..f3475df9e 100644 --- a/tests/models/test_edge_cases.py +++ b/tests/models/test_edge_cases.py @@ -278,6 +278,7 @@ def test_specs_differing_only_in_inapplicable_slots_share_spec_id(self): stat_vars, [], coverage, + composite=None, open_roles=(), inapplicable_slots=[], measure_elected_properties=frozenset(), @@ -288,6 +289,7 @@ def test_specs_differing_only_in_inapplicable_slots_share_spec_id(self): stat_vars, [], coverage, + composite=None, open_roles=(), inapplicable_slots=["SomeProperty"], measure_elected_properties=frozenset(), diff --git a/tests/platform/test_registry_binding.py b/tests/platform/test_registry_binding.py index 626f67e9b..fd51accb7 100644 --- a/tests/platform/test_registry_binding.py +++ b/tests/platform/test_registry_binding.py @@ -13,6 +13,8 @@ from __future__ import annotations +import json +import os from collections.abc import Callable from typing import get_args @@ -44,8 +46,12 @@ from presentation_api.doors.frames import build_frames_door from presentation_api.doors.shares import build_shares_door from search.engine.config import SEARCH_RESOLVE_DOORS +from search.engine.region.codes import BILATERAL_READING_UNAVAILABLE +from search.engine.region.scope_decision import _withheld_warning from search.models import NoDataReason +from search.scope.provider import WITHHELD_WHYS, Withheld, WithheldWhy from tests.code_registry import registered_endpoints, registered_status_codes +from tests.spec_kit_bundle import spec_kit_dir _REGISTRY = load_registry() @@ -362,3 +368,29 @@ def test_no_served_code_resolves_to_a_retired_registry_entry(): code for code in all_served if code in _REGISTRY and _REGISTRY[code].lifecycle == "retired" ) assert not retired, f"served code(s) resolve to a retired registry entry: {retired}" + + +def _registered_warning_reasons(code: str) -> frozenset[str]: + """The reasons dfo-spec-kit's warning-reason registry lists for `code`, read from the kit + checkout `DFO_SPEC_KIT_DIR` names, else the sibling checkout. A missing registry skips, + and fails when `DFO_SPEC_KIT_REQUIRED` is set.""" + path = spec_kit_dir() / "registries" / "warning-reasons.json" + if not path.is_file(): + if os.environ.get("DFO_SPEC_KIT_REQUIRED"): + raise RuntimeError(f"DFO_SPEC_KIT_REQUIRED is set but {path} does not exist") + pytest.skip("warning-reason registry not found -- set DFO_SPEC_KIT_DIR") + registry = json.loads(path.read_text()) + assert registry["format_version"] == 1 + return frozenset(entry["reason"] for entry in registry["entries"] if entry["code"] == code) + + +def test_the_bilateral_reading_warning_states_the_registered_reasons_and_no_other(): + """Every `why` Search can put on `bilateral_reading_unavailable` (`WITHHELD_WHYS`) is a + reason the registry lists for it, every listed reason is one Search can state, and each + has a message, `slot_not_served` alone naming the slot in `params`.""" + assert _registered_warning_reasons(BILATERAL_READING_UNAVAILABLE) == WITHHELD_WHYS + for why in get_args(WithheldWhy): + slot = "crsTypeOfFinance" if why == "slot_not_served" else None + warning = _withheld_warning("spec", Withheld(why=why, slot=slot)) + expected = {"why": why} if slot is None else {"why": why, "slot": slot} + assert (warning.code, warning.params) == (BILATERAL_READING_UNAVAILABLE, expected) diff --git a/tests/render/test_render_byline.py b/tests/render/test_render_byline.py index c42838e8a..4e83fb52a 100644 --- a/tests/render/test_render_byline.py +++ b/tests/render/test_render_byline.py @@ -18,6 +18,7 @@ ResolvedFacet, Spec, StatVar, + TimeWindow, UnitConversion, ) from search.render.heading import LATEST_YEAR_LABEL, heading_from_parts @@ -289,14 +290,28 @@ def test_a_unit_facet_with_no_currency_reads_the_registry_currency(ref, expected assert [t.text for t in plan if t.axis == "unit"] == expected -def test_a_latest_request_states_its_year_and_opens_the_byline_with_the_label(): - sentence, plan = _render( +@pytest.mark.parametrize( + "identity_window", + [ + pytest.param("latest", id="identity-latest"), + # A composite's identity holds the latest year it elected (`elect_latest_window`), + # and its trace the request's `latest`. + pytest.param(TimeWindow(start_year=2024, end_year=2024), id="composite-elected-year"), + ], +) +def test_a_latest_request_states_its_year_and_opens_the_byline_with_the_label(identity_window): + spec = _spec( + measured_property=_ref("disbursements", "Disbursements"), measurement_qualifier=_NOMINAL, facets=_unit("CurrencyUSD"), stat_vars=_served("2024", "2024"), resolved_sources=[_CRS], - requested_window="latest", + requested_window=identity_window, ) + trace = spec.resolved.trace.model_copy(update={"requested_window": "latest"}) + spec = spec.model_copy(update={"resolved": spec.resolved.model_copy(update={"trace": trace})}) + plan = render_sentence_plan(spec) + sentence = sentence_from_plan(plan) assert sentence == ( "Disbursements in 2024. Latest year with data · Current US dollars · " "OECD Creditor Reporting System (CRS)." diff --git a/tests/scope/test_basis.py b/tests/scope/test_basis.py index 57edcf3fa..b31d4c99a 100644 --- a/tests/scope/test_basis.py +++ b/tests/scope/test_basis.py @@ -1463,7 +1463,7 @@ def sharing(state: ResolveState, candidates: list[Any]) -> Any: offers: list[tuple[tuple[str, ...], UnitConversion | None, UnitConversion | None]] = [] stated = pipeline._with_scope_choices - def captured(result: RegionResult, state: ResolveState) -> RegionResult: + def captured(result: RegionResult) -> RegionResult: assert result.status == "definite", result.status # The offer a merge restates, read under the request's conversion catalogue. bare = result.spec.resolved.model_copy(update={"unit_conversion": None}) @@ -1483,7 +1483,7 @@ def captured(result: RegionResult, state: ResolveState) -> RegionResult: restated.resolved.unit_conversion, ) ) - return stated(result, state) + return stated(result) monkeypatch.setattr(candidate_collapse, "ground_candidates", sharing) monkeypatch.setattr(candidate_grounding, "_finish_candidate", counted) diff --git a/tests/scope/test_parsed_door.py b/tests/scope/test_parsed_door.py index b79d9bdff..e5bd42eab 100644 --- a/tests/scope/test_parsed_door.py +++ b/tests/scope/test_parsed_door.py @@ -102,7 +102,7 @@ def _write_records( assemble = phase_answer._assemble_definite_answer seen: list[ScopeState] = [] - async def recording(state): + async def recording(state, *args): selection = state.scope_selection seen.append(selection) sector = next( @@ -134,7 +134,7 @@ async def recording(state): entry=entries[0] if entries else None, ) ) - return await assemble(state) + return await assemble(state, *args) monkeypatch.setattr(phase_answer, "_assemble_definite_answer", recording) return seen diff --git a/tests/scope/test_provider.py b/tests/scope/test_provider.py index 672265640..1f8f93514 100644 --- a/tests/scope/test_provider.py +++ b/tests/scope/test_provider.py @@ -766,17 +766,19 @@ def _total() -> ScopeState: @pytest.mark.parametrize( - ("slots", "why"), + ("slots", "slot"), [ pytest.param( - {"crsCooperationModality": "CRSCooperationModalityA01"}, "modality", id="modality" + {"crsCooperationModality": "CRSCooperationModalityA01"}, + "crsCooperationModality", + id="modality", ), pytest.param( - {"crsTypeOfFinance": "CRSTypeOfFinance110"}, "finance_type", id="finance-type" + {"crsTypeOfFinance": "CRSTypeOfFinance110"}, "crsTypeOfFinance", id="finance-type" ), ], ) -def test_a_bound_modality_or_instrument_keeps_a_total_bilateral_with_the_reason(slots, why): +def test_a_bound_modality_or_instrument_keeps_a_total_bilateral_with_the_reason(slots, slot): facts = _facts() identity = _bilateral_identity(**slots) table = provider_rule(identity, None, facts) @@ -789,7 +791,7 @@ def test_a_bound_modality_or_instrument_keeps_a_total_bilateral_with_the_reason( "unavailable", "provider_not_offered", ) - assert inclusive.availability.detail == {"why": why} + assert inclusive.availability.detail == {"why": "slot_not_served", "slot": slot} assert (record.active, record.provenance, record.rule) == ( "provider:bilateral", "default", @@ -3390,7 +3392,11 @@ def refusing(document, *, facts): option.availability.status, option.availability.code, option.availability.detail, - ) == ("unavailable", "provider_not_offered", {"why": "dac2aAidType"}) + ) == ( + "unavailable", + "provider_not_offered", + {"why": "slot_not_served", "slot": "dac2aAidType"}, + ) def test_a_fresh_dac2a_scope_reads_its_bilateral_option_once(monkeypatch: pytest.MonkeyPatch): From 446e829fbaa4ca936bee8b3d080e559c2034fc86 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 20:17:05 +0100 Subject: [PATCH 07/12] fix(search): fail closed alike on every producer of an unminted reading A flagged match whose composite cannot be minted, and a candidate set whose elected reading cannot, now answer facet_not_available as the definite path does. The definite carve-out refusals state the decision's bilateral_reading_unavailable warning. A provider option refused at an entity, provenance or shape member marks only that option composite_not_servable and the scope still builds. Duplicate candidates are skipped before their composite is minted, and _suggest_nearest_real takes the part's scope selection, operations and catalogue as required arguments. --- .../engine/region/candidate_collapse.py | 3 + .../engine/region/candidate_grounding.py | 63 +++++++++++----- src/search/engine/region/grounding.py | 8 +- src/search/engine/region/phase_answer.py | 19 ++++- .../engine/region/phase_place_materialise.py | 6 ++ src/search/engine/region/phase_shared.py | 21 +++++- src/search/engine/region/reading.py | 6 +- src/search/models/scope.py | 3 +- src/search/scope/provider.py | 32 +++++--- .../test_candidate_collapse_rule_one.py | 4 +- tests/engine/test_dac2a_candidate_readings.py | 60 +++++++++++++++ .../engine/test_dac2a_suggestion_readings.py | 26 +++++++ .../engine/test_nearest_real_index_sourced.py | 4 + tests/engine/test_nearest_real_relaxation.py | 73 +++++++++++++++++++ tests/engine/test_pinned_code_refusal.py | 2 +- tests/engine/test_qualifier_guard.py | 2 +- tests/engine/test_reading_decision.py | 38 ++++++++++ tests/engine/test_role_bearing_resolve.py | 4 + tests/engine/test_slot_expansion_wire.py | 7 ++ tests/scope/test_provider.py | 60 +++++++++++++++ 20 files changed, 402 insertions(+), 39 deletions(-) diff --git a/src/search/engine/region/candidate_collapse.py b/src/search/engine/region/candidate_collapse.py index a204a4766..fa1469295 100644 --- a/src/search/engine/region/candidate_collapse.py +++ b/src/search/engine/region/candidate_collapse.py @@ -1018,6 +1018,9 @@ async def assemble_candidates( # Grounding reads the graph synchronously for every scope member, so it runs off the event # loop, where the request deadline and other requests keep running. grounded = await asyncio.to_thread(ground_candidates, state, candidates) + if grounded.failed is not None: + # A candidate read within a composite's population that no scope states. + return failed_closed(state, mark, candidate=grounded.failed) unique_specs = grounded.specs # The stand-in that re-election installed is the elected candidate here. for spec in unique_specs: diff --git a/src/search/engine/region/candidate_grounding.py b/src/search/engine/region/candidate_grounding.py index 33b0aac8b..98c8248e6 100644 --- a/src/search/engine/region/candidate_grounding.py +++ b/src/search/engine/region/candidate_grounding.py @@ -796,21 +796,16 @@ def spec_id(self) -> str: return self.identity.spec_id if self.minted is None else self.minted.mint.spec_id -def _candidate_answer( +def _candidate_identity( state: ResolveState, cand_index: int, cand: _Candidate, groundwork: _CandidateGroundwork, prep: _CandidatePrep, -) -> _CandidateAnswer: - """The identity candidate `cand_index`'s Spec will be minted from and the scope decided - for it, read before the rest of its Spec is assembled, so a candidate whose `spec_id` an - earlier one holds can be skipped at no further cost. - - The scope applies the provider reading the candidate's own read elected - (`_Candidate.reading`), as the definite answer's applies its read's - (`reading.decide_read_scope`), so a candidate read within a composite's population - is minted as that composite and labelled by the reading that read its rows.""" +) -> tuple[AnswerIdentity, dict[str, IdentityFacetBinding] | None]: + """The identity candidate `cand_index`'s Spec will be minted from and the facet bindings + it holds, read before its scope is decided or the rest of its Spec is assembled, so a + candidate whose identity an earlier one holds can be skipped at no further cost.""" facet_bindings: dict[str, IdentityFacetBinding] | None = ( # A bound outcome guarantees a unit request. The type checker cannot carry that # attribute invariant through this access. @@ -837,6 +832,22 @@ def _candidate_answer( pin_served_provenance=state.pin_served_provenance, named_groups=request_named_groups(state, cand.place.roles), ) + return identity, facet_bindings + + +def _candidate_answer( + state: ResolveState, + cand: _Candidate, + identity: AnswerIdentity, + facet_bindings: dict[str, IdentityFacetBinding] | None, +) -> _CandidateAnswer: + """The scope decided for candidate `cand`, whose Spec is minted from `identity` holding + `facet_bindings` (`_candidate_identity`), and the composite its Spec is minted as. + + The scope applies the provider reading the candidate's own read elected + (`_Candidate.reading`), as the definite answer's applies its read's + (`reading.decide_read_scope`), so a candidate read within a composite's population + is minted as that composite and labelled by the reading that read its rows.""" scoped = decide_read_scope( identity, cand.reading, @@ -1057,10 +1068,15 @@ def _listed_members( @dataclass(frozen=True) class _GroundedCandidates: """The kept candidates' records (`_GroundedCandidate`) by `spec_id`, one per Spec, in - candidate order. ``answer_t0`` excludes context preparation from answer timing.""" + candidate order. ``answer_t0`` excludes context preparation from answer timing. + `failed` is the elected candidate where it was read within a composite's population that + no scope minting the composite states (`reading.decide_read_scope`), which grounding + stops at with no record kept, so the part fails closed describing it + (`phase_answer.failed_closed`); None otherwise.""" entries: Mapping[str, _GroundedCandidate] answer_t0: int + failed: _Candidate | None def __getitem__(self, spec_id: str) -> _GroundedCandidate: return self.entries[spec_id] @@ -1154,6 +1170,12 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro alternate's places beside a role-bearing candidate's recipients, is no part of the comparison. A ``family_total_absent`` fact names one Spec's member, so each is recorded on the state by ``spec_id`` for the terminal that holds that Spec. + + A candidate read within a composite's population that no scope minting the composite + states has no label for its rows to stand under (`reading.decide_read_scope`). The + elected one stops grounding (`_GroundedCandidates.failed`), so the part fails closed as + the definite answer does, since leaving it out would let another reading answer in its + place. An alternate is left out, and the rest of the set stands. """ prep = _prepare_candidate_context(state, candidates) candidates = prep.candidates @@ -1171,12 +1193,19 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro groundworks = _ground_each_candidate(state, candidates, prep) entries: dict[str, _GroundedCandidate] = {} + # The identities of the candidates whose scopes were decided. + decided: set[str] = set() for cand_index, (cand, groundwork) in enumerate(zip(candidates, groundworks, strict=True)): - answer = _candidate_answer(state, cand_index, cand, groundwork, prep) - # A candidate whose Spec an earlier one holds adds no Spec or record, and its Spec is - # not assembled. It follows the candidate it repeats, so it is never the elected one. - # One read within a composite's population that cannot be minted as it is dropped, - # since the table's label would state the population's rows (`reading.decide_read_scope`). + identity, facet_bindings = _candidate_identity(state, cand_index, cand, groundwork, prep) + # A candidate whose identity or Spec an earlier one holds adds no Spec or record: its + # scope is not decided again, its composite not minted again, and its Spec not + # assembled. It follows the candidate it repeats, so it is never the elected one. + if identity.spec_id in decided: + continue + decided.add(identity.spec_id) + answer = _candidate_answer(state, cand, identity, facet_bindings) + if answer.decision is None and _is_elected(state, cand_index): + return _GroundedCandidates(entries={}, answer_t0=answer_t0, failed=cand) if answer.decision is None or answer.spec_id in entries: continue entry = _finish_candidate( @@ -1199,7 +1228,7 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro restate_scope_warning(state, list(entries.values())) _warn_for_candidates(state, prep, groundworks) - return _GroundedCandidates(entries=entries, answer_t0=answer_t0) + return _GroundedCandidates(entries=entries, answer_t0=answer_t0, failed=None) def _warn_for_candidates( diff --git a/src/search/engine/region/grounding.py b/src/search/engine/region/grounding.py index 5b2ea28d2..1b67d3c58 100644 --- a/src/search/engine/region/grounding.py +++ b/src/search/engine/region/grounding.py @@ -1571,9 +1571,9 @@ def _suggest_nearest_real( unsettled: UnsettledPlace | None = None, pin_served_provenance: bool = False, level: tuple[ShapeDraft, Relaxation] | None = None, - scope_selection: ScopeState | None = None, - operations: Sequence[Operation] = (), - request_catalogue: Catalogue | None = None, + scope_selection: ScopeState, + operations: Sequence[Operation], + request_catalogue: Catalogue | None, ) -> tuple[ list[NearestRealSpec], list[Warning], dict[str, SuggestionRead], ConsultedPresence | None ]: @@ -1652,7 +1652,7 @@ def _suggest_nearest_real( defaulted_properties=defaulted_properties, named_groups=named_groups, pin_served_provenance=pin_served_provenance, - scope_selection=ScopeState() if scope_selection is None else scope_selection, + scope_selection=scope_selection, operations=operations, request_catalogue=request_catalogue, ) diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index 82ba7abe0..48ce0821e 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -1283,13 +1283,15 @@ async def _assemble_carve_outs( mat_result: Materialised, spec: Spec, *, + decision: ScopeDecision | None, minted: MintedComposite | None = None, ) -> RegionResult | None: """Apply the remaining no-data gates after Spec construction. ``None`` means definite. Qualifier refusal precedes missing-slice and observation checks. Bound-unit availability is already verified by ``bound_unit_gate_reason``. `spec`, the flagged match each gate - names, is minted as `minted` where it is a composite. + names, is minted as `minted` where it is a composite, and carries the warning of + `decision`, the scope decided for it (`_show_decision`). """ matched = () if minted is None else (minted,) # Judge the final answer on every requirement (`requirements.settle_answer`). One it @@ -1300,6 +1302,7 @@ async def _assemble_carve_outs( state, state.shape_draft, state.bindings, served=served_prices(spec) ) if unbound_qualifiers: + _show_decision(state, decision) return state.no_data( "qualifier_not_bound", flagged_match=(spec,), @@ -1313,6 +1316,7 @@ async def _assemble_carve_outs( # place. Read the materialised value directly. Confidence masks it when no place is # named. A defaulted country scope can establish ``False``. if mat_result.available_at_place is False: + _show_decision(state, decision) return state.no_data( "no_observations", flagged_match=(spec,), @@ -1325,6 +1329,7 @@ async def _assemble_carve_outs( # Without a named place, ``has_data=False`` means no member asserted every bound # property. Report the matched Spec as ``flagged_match``. if state.destination_dcid is None and not mat_result.has_data: + _show_decision(state, decision) return state.no_data( "no_matching_slice", flagged_match=(spec,), @@ -1339,6 +1344,16 @@ async def _assemble_carve_outs( return None +def _show_decision(state: ResolveState, decision: ScopeDecision | None) -> None: + """Add the warning of `decision`, the scope decided for the answer a carve-out refusal + shows as its flagged match, to the part's once (`ScopeDecision.warning`): a reading the + bilateral default cannot reach says it reads total ODA, as a candidate a refusal shows + does (`candidate_collapse._shown`).""" + warning = None if decision is None else decision.warning + if warning is not None and warning not in state.warnings: + state.warnings.append(warning) + + def _drop_related_read(state: ResolveState) -> None: """Cancel the related-measures read started for the answer (`ResolveState.related_read`), whose result nothing will state.""" @@ -1593,7 +1608,7 @@ async def _assemble_definite_answer(state: ResolveState, mark: AnswerMark) -> Re state.state_reelected_reading(definite_spec.spec) early = await _assemble_carve_outs( - state, mat_result, definite_spec.spec, minted=coverage.minted + state, mat_result, definite_spec.spec, decision=coverage.decision, minted=coverage.minted ) if early is not None: return early diff --git a/src/search/engine/region/phase_place_materialise.py b/src/search/engine/region/phase_place_materialise.py index bbc2e67f7..495d3aae7 100644 --- a/src/search/engine/region/phase_place_materialise.py +++ b/src/search/engine/region/phase_place_materialise.py @@ -2324,6 +2324,8 @@ async def _declined_refusal(state: ResolveState) -> RegionResult: if declined.pinned_failure != "pinned_code": return await _denominator_refusal(state, None, members=members, open_roles=open_roles) flagged = await _elected_flagged_match(state, members=members, open_roles=open_roles) + if isinstance(flagged, RegionResult): + return flagged return state.no_data( "qualifier_not_bound", flagged_match=(flagged.spec,), @@ -2344,6 +2346,8 @@ async def refuse_unread_answer(state: ResolveState) -> RegionResult: names the terms its trace leaves unmet (`requirements.settle_answer`).""" members, open_roles = _unread_answer_members(state) flagged = await _elected_flagged_match(state, members=members, open_roles=open_roles) + if isinstance(flagged, RegionResult): + return flagged return state.no_data( "qualifier_not_bound", flagged_match=(flagged.spec,), @@ -2395,6 +2399,8 @@ async def _denominator_refusal( members = mat_result.sv_dcids open_roles = mat_result.open_axes flagged = await _elected_flagged_match(state, members=members, open_roles=open_roles) + if isinstance(flagged, RegionResult): + return flagged named = state.named_denominator level = _level_relaxation(state) return state.no_data( diff --git a/src/search/engine/region/phase_shared.py b/src/search/engine/region/phase_shared.py index be73b4c0a..26959d730 100644 --- a/src/search/engine/region/phase_shared.py +++ b/src/search/engine/region/phase_shared.py @@ -329,7 +329,7 @@ async def _elected_flagged_match( *, members: Sequence[str] | None = None, open_roles: Collection[RoleAxis] = (), -) -> FlaggedMatch: +) -> FlaggedMatch | RegionResult: """Build the elected shape's flagged identity without probing availability, for a refusal that judges the shape itself: role incompatibility, or a quantity form or denominator the shape does not serve. The identity names `members`, the StatVars @@ -346,9 +346,17 @@ async def _elected_flagged_match( before the read, on the shape's own names. Preserve an established entity mapping on materialise-stage refusals so the where slot, - entities, trace, and identity agree. Early refusals have no mapping to retain.""" + entities, trace, and identity agree. Early refusals have no mapping to retain. + + A match read within a composite's population that no scope minting the composite states + has no label to stand under, so it fails closed as the definite answer does + (`reading.ReadScope`): the result returned is no_data `facet_not_available` + (`phase_answer.failed_closed`), which the caller returns in place of its refusal, with + what the match raised dropped.""" await read_held_families((state.shape_draft,)) await judge_coverage(state) + # Everything the match raises from here is its own (`ResolveState.mark_answer`). + mark = state.mark_answer() sv_dcids = list(members) if members else list(state.shape_draft.sv_arc_facts or ()) ( five_tuple_refs, @@ -418,6 +426,11 @@ async def _elected_flagged_match( # The election and the scope read coverage and scan families, so they run off the # event loop. scoped = await asyncio.to_thread(_flagged_scope, state, answer) + if scoped.decision is None: + # Deferred: the answer phase imports this module. + from search.engine.region.phase_answer import failed_closed + + return failed_closed(state, mark) minted = scoped.minted spec = build_spec( composite=None if minted is None else minted.mint, @@ -548,6 +561,8 @@ async def settle_unjudged_refusal(state: ResolveState, result: RegionResult) -> mat_result = state.mat_result members = mat_result.sv_dcids if isinstance(mat_result, Materialised) else None flagged = await _elected_flagged_match(state, members=members) + if isinstance(flagged, RegionResult): + return flagged return state.no_data( "qualifier_not_bound", flagged_match=(flagged.spec,), @@ -563,6 +578,8 @@ async def settle_unjudged_refusal(state: ResolveState, result: RegionResult) -> async def _role_incompatible_no_data(state: ResolveState) -> RegionResult: """Return the common role-incompatibility outcome without a new probe.""" flagged = await _elected_flagged_match(state) + if isinstance(flagged, RegionResult): + return flagged return state.no_data( "role_incompatible", flagged_match=(flagged.spec,), diff --git a/src/search/engine/region/reading.py b/src/search/engine/region/reading.py index 558179ccb..f43dbff71 100644 --- a/src/search/engine/region/reading.py +++ b/src/search/engine/region/reading.py @@ -369,7 +369,11 @@ class ReadScope: over it (`decision`), and the composite the Spec is minted as where that scope compiles one (`minted`), as `decide_read_scope` decides them. `decision` is None where the reading was read within a composite's population and no scope minting that composite - could be decided, so its rows have no label to stand under.""" + could be decided, so its rows have no label to stand under, and every producer answers + no_data `facet_not_available` in its place (`phase_answer.failed_closed`): the definite + answer (`phase_answer._gather_definite_coverage`), each candidate + (`candidate_grounding.ground_candidates`) and a flagged match + (`phase_shared._elected_flagged_match`).""" answer: AnswerIdentity decision: ScopeDecision | None diff --git a/src/search/models/scope.py b/src/search/models/scope.py index 91676c8f0..8730a4297 100644 --- a/src/search/models/scope.py +++ b/src/search/models/scope.py @@ -186,7 +186,8 @@ class OptionAvailability(StrictModel): default_factory=dict, description=( "the code's parameters: `why` for provider_not_offered, with `slot`, the " - "property at fault, where why is slot_not_served, `dataset` for " + "property at fault, where why is slot_not_served, and `pointer`, the identity " + "member at fault, where why is composite_not_servable, `dataset` for " "basis_not_published and contributions_counted, `destination` for " "multilateral_destination, the multilateral agency the part names as recipient, " "whose flows are core contributions, `entry` and `cause` for conflicts_with" diff --git a/src/search/scope/provider.py b/src/search/scope/provider.py index d628b4633..38ef2234a 100644 --- a/src/search/scope/provider.py +++ b/src/search/scope/provider.py @@ -48,6 +48,7 @@ from __future__ import annotations import dataclasses +import logging from collections.abc import Collection, Mapping, Sequence from dataclasses import dataclass from datetime import date @@ -163,7 +164,12 @@ _DONOR_ORGANISATIONS = frozenset(DONOR_ORGANISATIONS) _NET = "net" _CONSTANT = "RealValue" +logger = logging.getLogger(__name__) _SLOT_NOT_SERVED = "slot_not_served" +_COMPOSITE_NOT_SERVED = "composite_not_servable" +"""The `why` of `provider_not_offered` where a provider composite refuses what the part binds +at a member that names no slot (`/entities/`, `/facets/provenance`, `/shape`), the +member in the detail's `pointer` (`_composite_fault`).""" """The `why` of `provider_not_offered` where a provider composite cannot serve a slot the part binds, whose property the detail's `slot` names (`_composite_fault`).""" @@ -748,9 +754,10 @@ def bilateral_default_withheld( def _withheld(availability: OptionAvailability) -> Withheld: """The reason an unavailable or partial bilateral option states (`_read_bilateral_option`): `destination_unclassified` or `slot_not_served`, with its slot, from a - `provider_not_offered` detail, and `option_not_covered` from that code. Raises - `ValueError` for any other: the option is unavailable with `multilateral_destination` - only on a part naming a multilateral destination, which the default does not reach.""" + `provider_not_offered` detail, and `option_not_covered` from that code. Any other state + is `scope_undecided`, logged as an error: the fallback for an unreachable state, + since the default does not reach a part naming a multilateral destination + (`multilateral_destination`), and `composite_not_servable` has no registered reason.""" detail = availability.detail match availability.code, detail.get("why"): case "provider_not_offered", "destination_unclassified": @@ -759,7 +766,12 @@ def _withheld(availability: OptionAvailability) -> Withheld: return Withheld(why="slot_not_served", slot=detail["slot"]) case "option_not_covered", None: return Withheld(why="option_not_covered") - raise ValueError(f"the bilateral option states no withheld reason: {availability!r}") + logger.error( + "bilateral option states no registered withheld reason: %s %s", + availability.code, + detail, + ) + return Withheld(why="scope_undecided") def bilateral_default_reading( @@ -1296,17 +1308,19 @@ def _composite_fault( unavailable population (`PopulationUnavailable`) and why `slot_not_served`, its `slot` the property at fault, for a slot the composite's components do not declare; unavailable with `uncovered`, the option's code for a composite serving no year, where the window - reaches none of the composite's component periods. Re-raises `exc` at any other member - (`/shape`, `/entities/`, a provenance set or a composite the index does not serve), - which no part reaches here: the options bind the composite as a value over a described - identity whose named destinations are on the composite's side.""" + reaches none of the composite's component periods. At any other member (`/shape`, + `/entities/`, a provenance set or a composite the index does not serve) it is not + offered with why `composite_not_servable` and the `pointer` at fault, so one option's + refusal never aborts the scope build.""" if isinstance(exc, PopulationUnavailable): return _unavailable("provider_not_offered", {"why": "destination_unclassified"}) if exc.pointer == "/requested_window": return _unavailable(uncovered) slot = _slot_at(identity, exc.pointer) if slot is None: - raise exc + return _unavailable( + "provider_not_offered", {"why": _COMPOSITE_NOT_SERVED, "pointer": exc.pointer} + ) return _unavailable("provider_not_offered", {"why": _SLOT_NOT_SERVED, "slot": slot}) diff --git a/tests/engine/test_candidate_collapse_rule_one.py b/tests/engine/test_candidate_collapse_rule_one.py index 376a80623..00dae970a 100644 --- a/tests/engine/test_candidate_collapse_rule_one.py +++ b/tests/engine/test_candidate_collapse_rule_one.py @@ -79,7 +79,9 @@ def _entry( def _grounded(*rows: tuple[Spec, str, ShapeDraft, CandidateBasis | None]) -> _GroundedCandidates: """Grounded candidates from (spec, StatVar, shape, basis) rows, in order.""" - return _GroundedCandidates(entries={row[0].spec_id: _entry(*row) for row in rows}, answer_t0=0) + return _GroundedCandidates( + entries={row[0].spec_id: _entry(*row) for row in rows}, answer_t0=0, failed=None + ) def _settle( diff --git a/tests/engine/test_dac2a_candidate_readings.py b/tests/engine/test_dac2a_candidate_readings.py index dd6ad534d..67fbe6025 100644 --- a/tests/engine/test_dac2a_candidate_readings.py +++ b/tests/engine/test_dac2a_candidate_readings.py @@ -32,6 +32,7 @@ ) from search.engine.region.scope_decision import _withheld_warning from search.engine.region.state import ResolveState +from search.engine.spec_id import identity_spec_id from search.models import ( BaseIdentity, CandidatesPart, @@ -280,6 +281,38 @@ def refusing(identity: BaseIdentity, **kwargs: Any) -> Any: monkeypatch.setattr(resubmit, "composite_mint", refusing) +def test_a_set_whose_elected_reading_cannot_be_minted_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The donor reading, elected first, is read within the bilateral population, and its + composite cannot be minted, so its rows have no label to stand under. The part fails + closed as the definite answer does (`phase_answer.failed_closed`), describing the + donor reading, and the recipient reading does not answer in its place.""" + mint = resubmit.composite_mint + + def refusing(identity: BaseIdentity, **kwargs: Any) -> Any: + if identity.entities.get("destination") != [_IND]: + raise RuntimeError("mint failed") + return mint(identity, **kwargs) + + monkeypatch.setattr(resubmit, "composite_mint", refusing) + closed: list[Any] = [] + close = candidate_collapse.failed_closed + + def closing(state: Any, mark: Any, *, candidate: Any = None) -> Any: + closed.append(candidate) + return close(state, mark, candidate=candidate) + + monkeypatch.setattr(candidate_collapse, "failed_closed", closing) + + part = _india(india_request(open_role="recipient")) + + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "facet_not_available" + (candidate,) = closed + assert candidate.reading is not None and candidate.reading.table is not None + + def test_a_set_left_with_one_composite_reading_answers_it_as_its_resubmit_does( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: @@ -573,6 +606,33 @@ async def capture(state: Any, specs: list[Any], grounded: Any, truncated: bool) assert scopes == [state.request_catalogue.scope] * 2 +def test_a_candidate_repeating_an_earlier_one_mints_no_composite( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A candidate whose identity an earlier one holds is skipped before its scope is + decided (`candidate_grounding.ground_candidates`), so its composite is not minted again + and the set keeps one Spec per reading.""" + minted: list[str] = [] + mint = resubmit.composite_mint + + def recording(identity: BaseIdentity, **kwargs: Any) -> Any: + minted.append(identity_spec_id(identity)) + return mint(identity, **kwargs) + + monkeypatch.setattr(resubmit, "composite_mint", recording) + ground = candidate_collapse.ground_candidates + + def repeated(state: Any, candidates: list[Any]) -> Any: + return ground(state, [*candidates, candidates[-1]]) + + monkeypatch.setattr(candidate_collapse, "ground_candidates", repeated) + + part = _candidates() + + assert len(part.candidates.specs) == 2 + assert len(minted) == len(set(minted)) == 2 + + def test_a_refusal_showing_a_composite_candidate_carries_its_mint_on_its_own( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/engine/test_dac2a_suggestion_readings.py b/tests/engine/test_dac2a_suggestion_readings.py index 31457e0f0..21ce8267b 100644 --- a/tests/engine/test_dac2a_suggestion_readings.py +++ b/tests/engine/test_dac2a_suggestion_readings.py @@ -17,9 +17,11 @@ from search.engine.errors import EngineInputError from search.engine.region import ( + RegionResult, phase_place_materialise, phase_shared, reading, + resubmit, ) from search.engine.region.state import ResolveState from search.engine.retrieval import role_resolve @@ -214,6 +216,30 @@ def refusing(*_args: object, **_kwargs: object) -> object: assert withheld == [{"why": "scope_undecided"}] +def test_a_flagged_match_whose_composite_cannot_be_minted_fails_closed( + fresh_state: Callable[[], ResolveState], monkeypatch: pytest.MonkeyPatch +) -> None: + """A flagged match read within the bilateral population whose composite cannot be + minted has no label for its rows to stand under, so the table's Spec, which would say + they are total ODA, is not built. The refusal fails closed as the definite answer does + (`phase_answer.failed_closed`): no_data `facet_not_available`, with nothing the match + raised left on the part.""" + state = fresh_state() + before = list(state.warnings) + + def raising(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("mint failed") + + monkeypatch.setattr(resubmit, "composite_mint", raising) + + refused = asyncio.run(phase_shared._elected_flagged_match(state)) + + assert isinstance(refused, RegionResult), refused + assert (refused.status, refused.no_data_reason) == ("no_data", "facet_not_available") + assert refused.flagged_match is None + assert state.warnings == before + + def test_a_resubmit_refusal_s_flagged_match_is_the_composite() -> None: """A composite posted with a destination that holds no row in its population refuses `no_observations` with the composite as its flagged match.""" diff --git a/tests/engine/test_nearest_real_index_sourced.py b/tests/engine/test_nearest_real_index_sourced.py index 4924b77f0..1ce9eb346 100644 --- a/tests/engine/test_nearest_real_index_sourced.py +++ b/tests/engine/test_nearest_real_index_sourced.py @@ -10,6 +10,7 @@ from search.engine.bind import SlotBindingDraft from search.engine.region.grounding import _suggest_nearest_real from search.engine.shape import ShapeDraft, SlotKeyDraft +from search.scope.state import ScopeState from tests.engine._builders import bound_place from tests.engine._fake_coverage import empty_coverage_snapshot from tests.engine._fake_sdmx import FakeSdmxClient @@ -95,6 +96,9 @@ def _shape() -> ShapeDraft: def _run(): graph = FakeGraph(nodes=_NODES, obs=_OBS, detect={}, resolve={}) return _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient(observations={}, availability={}), shape_draft=_shape(), diff --git a/tests/engine/test_nearest_real_relaxation.py b/tests/engine/test_nearest_real_relaxation.py index b8fd43eb2..884107441 100644 --- a/tests/engine/test_nearest_real_relaxation.py +++ b/tests/engine/test_nearest_real_relaxation.py @@ -34,6 +34,7 @@ from search.engine.shape import ShapeDraft, SlotKeyDraft, is_role_bearing from search.engine.spec_build import NamedGroup from search.models import CoverageBare, EntityGroup, EntityRoleKey, TimeWindow +from search.scope.state import ScopeState from series_coverage.schema import FacetMetadata, RoleNames from series_coverage.shared import CoverageSnapshot from tests.engine import _answer_invariants_world as fixture_world @@ -167,6 +168,9 @@ def _obs_graph(*, a_has_data: bool, b_has_data: bool) -> FakeGraph: def test_relaxation_names_the_dropped_binding_when_a_relaxes(): graph = _obs_graph(a_has_data=True, b_has_data=False) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -202,6 +206,9 @@ def test_relaxation_moves_to_the_other_binding_when_data_moves(): """ graph = _obs_graph(a_has_data=False, b_has_data=True) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -233,6 +240,9 @@ def test_untouched_binding_keeps_the_caller_supplied_provenance(): """ graph = _obs_graph(a_has_data=True, b_has_data=False) suggestions, _, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -260,6 +270,9 @@ def test_untouched_binding_keeps_the_caller_supplied_provenance(): def test_no_suggestions_when_neither_relaxation_finds_data(): graph = _obs_graph(a_has_data=False, b_has_data=False) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -321,6 +334,9 @@ def _region_suggestions(held_at: str): admitted=frozenset({("observationAbout", "Continent"), ("observationAbout", "Country")}), ) suggestions, _warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -376,6 +392,9 @@ def test_label_miss_warning_is_deduped_across_two_surviving_suggestions(): } graph = FakeGraph(nodes=label_miss_nodes, obs=obs, detect={}, resolve={}) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -406,6 +425,9 @@ def test_label_miss_warning_is_deduped_across_two_surviving_suggestions(): def test_suggestion_carries_confidence(): graph = _obs_graph(a_has_data=True, b_has_data=False) suggestions, _, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -443,6 +465,9 @@ def test_option_coverage_probe_failure_is_fail_open(monkeypatch): ) graph = _obs_graph(a_has_data=True, b_has_data=False) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -479,6 +504,9 @@ def observation_facets_batch_many( graph = _obs_graph(a_has_data=True, b_has_data=False) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -507,6 +535,9 @@ def test_option_coverage_off_by_default_is_a_no_op(monkeypatch): ) graph = _obs_graph(a_has_data=True, b_has_data=False) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -598,6 +629,9 @@ def test_relaxation_suggests_what_the_relaxed_first_answer_serves( } graph = FakeGraph(nodes=nodes, obs=obs, detect={}, resolve={}) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -678,6 +712,9 @@ def _record_entities(*, entities, slot_options, **_kwargs): ) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph(nodes=_NODES, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}), shape_draft=_shape_role_bearing(("creditor", "debtor")), @@ -716,6 +753,9 @@ def _record_entities(*, entities, slot_options, **_kwargs): role_names={_SV_RELAX_A: _CREDITOR_DEBTOR_ROLE_NAMES}, ) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph( nodes={ **_NODES, @@ -769,6 +809,9 @@ def _fail_if_called(**_kwargs): monkeypatch.setattr(grounding_module, "standard_option_coverage", _fail_if_called) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph(nodes=_NODES, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}), shape_draft=shape, @@ -810,6 +853,9 @@ def test_relaxation_probe_uses_every_named_destination_not_just_the_first(monkey role_names={_SV_RELAX_A: _CREDITOR_DEBTOR_ROLE_NAMES}, ) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph(nodes=_NODES, obs={}, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}), shape_draft=_shape_role_bearing(("creditor", "debtor")), @@ -849,6 +895,9 @@ def test_a_role_suggestion_cites_the_sources_its_plan_serves(monkeypatch): }, ) suggestions, warnings, reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph( nodes={**_NODES, ids: {"label": "IDS"}, alt: {"label": "Alternative"}}, obs={}, @@ -904,6 +953,9 @@ def test_a_relaxation_with_no_point_in_the_window_is_not_suggested(): ) since = DateRequest(window=TimeWindow(start_year=2021, end_year=None), latest=False) suggestions, _warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -986,6 +1038,9 @@ def _suggest_over_walked_shape(walks: RoleClassWalks | None): for sv in (_SV_RELAX_A, _SV_RELAX_B) } return _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph(nodes=nodes, obs=obs, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}, availability={}), shape_draft=_walked_shape(), @@ -1109,6 +1164,9 @@ def test_a_placeless_relaxation_reads_the_presence_of_the_members_its_walk_adds( with recording_degraded_reads() as lost: suggestions, _warnings, reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -1160,6 +1218,9 @@ def test_a_suggestion_whose_walk_settles_nothing_warns_membership_unverified( for sv in (_SV_RELAX_A, _SV_RELAX_B) } suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph(nodes=_walked_nodes(), obs=obs, detect={}, resolve={}), sdmx=FakeSdmxClient(observations={}, availability={}), shape_draft=_shape(), @@ -1195,6 +1256,9 @@ def counted(shape, query, **kwargs): def suggest(with_data: tuple[str, ...]): with fixture_world._recorded() as recorder: found = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=FakeGraph( nodes=_walked_nodes(), obs={ @@ -1264,6 +1328,9 @@ def test_a_place_less_suggestion_reads_its_own_members_default_scope(): def suggest(place: PlaceContext, presence: DefaultPresence | None): return _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -1314,6 +1381,9 @@ def test_the_relaxed_queries_read_no_scope_their_requests_presence_holds(): scope = presence.consulted_union([]) suggestions, _warnings, reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability @@ -1379,6 +1449,9 @@ def test_a_failed_presence_read_for_the_members_a_walk_adds_gives_no_suggestion( scope = presence.consulted_union([_SV_BOTH]) suggestions, warnings, reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=FakeSdmxClient( observations={}, availability={}, scope_availability=graph.scope_availability diff --git a/tests/engine/test_pinned_code_refusal.py b/tests/engine/test_pinned_code_refusal.py index 22055d350..4cc26e6a1 100644 --- a/tests/engine/test_pinned_code_refusal.py +++ b/tests/engine/test_pinned_code_refusal.py @@ -148,7 +148,7 @@ def _materialised() -> Materialised: def _carve_out(state: ResolveState, spec: Spec) -> RegionResult | None: """The definite door's carve-outs on `spec`, with the departures the part emits.""" - result = asyncio.run(_assemble_carve_outs(state, _materialised(), spec)) + result = asyncio.run(_assemble_carve_outs(state, _materialised(), spec, decision=None)) if result is None: return None return with_requirement_departures(result, state.requirements, state.judgement) diff --git a/tests/engine/test_qualifier_guard.py b/tests/engine/test_qualifier_guard.py index b95c2c127..1e007abcd 100644 --- a/tests/engine/test_qualifier_guard.py +++ b/tests/engine/test_qualifier_guard.py @@ -1069,7 +1069,7 @@ def test_a_word_naming_a_measure_value_the_answer_lacks_refuses_whatever_its_rat materialised = Materialised( sv_dcids=["psi_member"], has_data=True, coverage=CoverageBare(has_data=True) ) - result = asyncio.run(_assemble_carve_outs(state, materialised, spec)) + result = asyncio.run(_assemble_carve_outs(state, materialised, spec, decision=None)) terms = result.unbound_qualifiers if result is not None else () assert (result is not None and result.no_data_reason == "qualifier_not_bound") == bool(refused) diff --git a/tests/engine/test_reading_decision.py b/tests/engine/test_reading_decision.py index f25b6c15b..e6f067558 100644 --- a/tests/engine/test_reading_decision.py +++ b/tests/engine/test_reading_decision.py @@ -78,6 +78,7 @@ _sdmx, _trailing, _withheld, + _years_short, ) from tests.engine.test_whole_total_bind_e2e import _VARIABLE @@ -867,3 +868,40 @@ def raising(*_args: object, **_kwargs: object) -> object: assert isinstance(part, NoDataPart), part assert part.no_data.reason == "facet_not_available" assert _faults(caplog) + + +def test_a_carve_out_flagging_a_reading_that_reads_total_oda_says_so( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Over every donor with bilateral destinations covering fewer years than the answer, + the part reads total ODA. A carve-out refusal that flags that answer + (`phase_answer._assemble_carve_outs`), here no member holding data at the place, carries + the decision's `bilateral_reading_unavailable` keyed to the flagged match, as a definite + answer of it does.""" + _years_short(monkeypatch) + carve_outs = phase_answer._assemble_carve_outs + + async def unavailable(state: Any, mat_result: Any, *args: Any, **kwargs: Any) -> Any: + unheld = dataclasses.replace(mat_result, available_at_place=False) + return await carve_outs(state, unheld, *args, **kwargs) + + async def no_suggestion(*_args: object, **_kwargs: object) -> None: + return None + + monkeypatch.setattr(phase_answer, "_assemble_carve_outs", unavailable) + monkeypatch.setattr(phase_answer, "_nearest_real_for", no_suggestion) + + response = offline_resolve( + _SCENARIOS["H6"](), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=_sdmx() + ) + + assert isinstance(response, Resolution), response + (part,) = response.parts + assert isinstance(part, NoDataPart), part + assert part.no_data.reason == "no_observations" + flagged = part.no_data.flagged_match + assert flagged is not None and not isinstance(flagged, list) + withheld = [ + (w.spec_id, w.params) for w in part.warnings if w.code == "bilateral_reading_unavailable" + ] + assert withheld == [(flagged.spec_id, {"why": "option_not_covered"})] diff --git a/tests/engine/test_role_bearing_resolve.py b/tests/engine/test_role_bearing_resolve.py index 45b08dad7..197db5534 100644 --- a/tests/engine/test_role_bearing_resolve.py +++ b/tests/engine/test_role_bearing_resolve.py @@ -16,6 +16,7 @@ from search.engine.retrieval.role_resolve import graph_confirm_resolve from search.engine.shape import ShapeDraft, SlotKeyDraft from search.models import CoverageBare, CoverageExact, GraphRef, TimeWindow +from search.scope.state import ScopeState from series_coverage.schema import RoleNames from series_coverage.shared import CoverageSnapshot from tests.engine._builders import bound_place @@ -1138,6 +1139,9 @@ def test_the_nearest_real_relaxation_probe_carries_the_caller_own_deadline(monke graph = _DeadlineRecordingGraph(nodes=nodes) suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=graph, sdmx=sdmx, shape_draft=shape, diff --git a/tests/engine/test_slot_expansion_wire.py b/tests/engine/test_slot_expansion_wire.py index 20b4fd8f4..2b54a20b6 100644 --- a/tests/engine/test_slot_expansion_wire.py +++ b/tests/engine/test_slot_expansion_wire.py @@ -37,6 +37,7 @@ SlotKey, StatVar, ) +from search.scope.state import ScopeState from series_coverage.schema import RoleNames from tests.code_index.aid_identity import education_aid_identity from tests.engine._builders import bound_place, gref, shape_five_tuple, unbound_slot, value_slot @@ -372,6 +373,9 @@ def test_relaxing_a_different_binding_still_carries_the_untouched_crs_slots_expa """Relaxing prop_b leaves Health bound, so expansion and version metadata remain.""" bindings = [_nr_prop_b_binding([_NR_VAL_B1]), _nr_crs_purpose_binding([_HEALTH])] suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=_nr_graph(crs_purpose=_HEALTH, prop_b=_NR_VAL_B_OTHER), sdmx=FakeSdmxClient(observations={}, availability={}), shape_draft=_nr_shape(crs_purpose=_HEALTH, prop_b=_NR_VAL_B_OTHER), @@ -408,6 +412,9 @@ def test_relaxing_the_crs_binding_itself_carries_neither_field(self): _nr_prop_b_binding([_NR_VAL_B1]), ] suggestions, warnings, _reads, _presence = _suggest_nearest_real( + scope_selection=ScopeState(), + operations=(), + request_catalogue=None, graph=_nr_graph(crs_purpose=_NR_UNRELATED_PURPOSE, prop_b=_NR_VAL_B1), sdmx=FakeSdmxClient(observations={}, availability={}), shape_draft=_nr_shape(crs_purpose=_NR_UNRELATED_PURPOSE, prop_b=_NR_VAL_B1), diff --git a/tests/scope/test_provider.py b/tests/scope/test_provider.py index 1f8f93514..a42de3ecc 100644 --- a/tests/scope/test_provider.py +++ b/tests/scope/test_provider.py @@ -3399,6 +3399,66 @@ def refusing(document, *, facts): ) +@pytest.mark.parametrize("pointer", ["/entities/destination", "/facets/provenance", "/shape"]) +@pytest.mark.parametrize("failing", ["bilateral", "inclusive"]) +def test_a_composite_refusal_at_a_member_naming_no_slot_marks_only_its_option( + monkeypatch: pytest.MonkeyPatch, failing: str, pointer: str +): + """An option whose composite refuses the part at an entity, provenance or shape member is + not offered (`composite_not_servable`, naming the pointer); the scope builds and the + other option is read as before.""" + refused = _DAC2A_BILATERAL if failing == "bilateral" else _INCLUSIVE_SOURCE + served_components, component_identities_ = provider.served_components, component_identities + + def refuses(document) -> bool: + return document.get("facets", {}).get("provenance", {}).get("ref") == refused + + def refusing_served(document, *, facts): + if refuses(document): + raise CompositeNotServable("refused", ref=refused, pointer=pointer) + return served_components(document, facts=facts) + + def refusing_components(index, document): + if refuses(document): + raise CompositeNotServable("refused", ref=refused, pointer=pointer) + return component_identities_(index, document) + + monkeypatch.setattr(provider, "served_components", refusing_served) + monkeypatch.setattr(provider, "component_identities", refusing_components) + if failing == "bilateral": + identity, facts = _dac2a_identity("Net", destination=None), _dac2a_facts(_ALL_YEARS) + else: + identity, facts = _bilateral_identity(), _facts() + + table = provider_rule(identity, None, facts) + + assert table is not None + option = table.option(f"provider:{failing}") + assert option is not None + assert ( + option.availability.status, + option.availability.code, + option.availability.detail, + ) == ( + "unavailable", + "provider_not_offered", + {"why": "composite_not_servable", "pointer": pointer}, + ) + + +def test_a_bilateral_option_with_no_registered_reason_is_withheld_as_scope_undecided(caplog): + """`composite_not_servable` has no reason in the spec-kit registry: the warning states + `scope_undecided` and logs the state instead of raising.""" + availability = provider._unavailable( + "provider_not_offered", {"why": "composite_not_servable", "pointer": "/shape"} + ) + + withheld = provider._withheld(availability) + + assert withheld.params == {"why": "scope_undecided"} + assert "/shape" in caplog.text + + def test_a_fresh_dac2a_scope_reads_its_bilateral_option_once(monkeypatch: pytest.MonkeyPatch): """The default and the provider table of one fresh part read the same bilateral option and destination rule, once per scope build.""" From 950adf14a12504da38def0600e9268b8ebf5e671 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 20:42:50 +0100 Subject: [PATCH 08/12] fix(search): bind the population total inside the read alone A part naming no destination reads the population total through the read (reading.ReadIn), so state.place, the bindings and nearest-real relaxations keep the places the request named, and resolved.entities lists the total without the identity binding it. The election decides reach by provider.counted_source, the rule the scope after the read uses, over a canonical pre-read identity, and one ScopeFacts per elected dataset serves every election of a part. --- .../engine/region/candidate_grounding.py | 14 +- src/search/engine/region/candidates.py | 23 +- src/search/engine/region/grounding.py | 32 +- src/search/engine/region/phase_answer.py | 21 +- .../engine/region/phase_place_materialise.py | 20 +- src/search/engine/region/phase_shared.py | 4 +- src/search/engine/region/reading.py | 396 ++++++++++-------- src/search/engine/region/state.py | 4 +- src/search/scope/provider.py | 12 +- src/search/scope/tables.py | 7 +- .../engine/test_dac2a_suggestion_readings.py | 57 +++ tests/engine/test_degraded_read_sites.py | 6 +- tests/engine/test_reading_decision.py | 83 +++- 13 files changed, 455 insertions(+), 224 deletions(-) diff --git a/src/search/engine/region/candidate_grounding.py b/src/search/engine/region/candidate_grounding.py index 98c8248e6..19131938d 100644 --- a/src/search/engine/region/candidate_grounding.py +++ b/src/search/engine/region/candidate_grounding.py @@ -74,7 +74,7 @@ role_served_filter, ) from search.engine.region.place_roles import request_named_groups -from search.engine.region.reading import decide_read_scope +from search.engine.region.reading import decide_read_scope, read_totals, roles_read from search.engine.region.requirements import answer_terms from search.engine.region.result import FacetAxisFilter, answered_roles from search.engine.region.scope import ( @@ -509,7 +509,8 @@ def _ground_one_candidate( cand_shape, cand_bindings, cand_mat.sv_dcids, - cand.place.roles.roles, + # It lists the totals its read bound, as the answer does (`reading.roles_read`). + roles_read(cand.place.roles.roles, cand.reading), state.graph, # The candidate's filtered plan names its sources and dates its StatVars, as the # elected answer's does. @@ -577,8 +578,13 @@ def _ground_one_candidate( member_facts=cand_mat.member_facts or {}, ) cand_entities = [_build_entity(rd, er, etr, role_refs) for rd, er, etr in cand_entity_data] - # Hash the same role-dispatched entities presented in the response. - cand_identity_entities = build_identity_entities(cand_entity_data, role_refs) + # Hash the same role-dispatched entities presented in the response, but the totals the + # read bound, which the request did not name. + totals = read_totals(cand.reading) + cand_identity_entities = build_identity_entities( + [entity for entity in cand_entity_data if (entity[1].ref, entity[0]) not in totals], + role_refs, + ) cand_slots = bind_when_slot(cand_slots, window=_cand_coverage.window) cand_slots = apply_slot_expansions(cand_slots, slot_expansions(cand_bindings)) # Count the menus over the candidate's own probe entities, as its probes read them. Its diff --git a/src/search/engine/region/candidates.py b/src/search/engine/region/candidates.py index 33a5615ac..9512bd6f7 100644 --- a/src/search/engine/region/candidates.py +++ b/src/search/engine/region/candidates.py @@ -38,7 +38,7 @@ unread_context, with_where, ) -from search.engine.region.reading import ElectedReads, ReadingDecision, Within +from search.engine.region.reading import ElectedReads, ElectionFacts, ReadBinding, ReadingDecision from search.engine.region.result import RegionResult from search.engine.region.state import ( AlternateUnevaluatedReason, @@ -120,7 +120,8 @@ class _Candidate: `stale_departure` when that newest year is stale (`trailing.stale_departure`). `reading` is the provider reading its read elected at the place it settled on - (`reading.elect_at_read`), None where it binds no total with a bilateral source. A + (`reading.elect_at_read`), None where its identity counts no total with a bilateral + source. A reading that names a composite was read within the composite's population, and the candidate's Spec is minted as that composite (`candidate_grounding._candidate_answer`).""" @@ -429,6 +430,7 @@ def _materialise_alt_shapes( Each read elects its provider reading first, at the place it tries, under the part's provider entries `selection` and `operations`, and reads the composite the reading names within its population, as the elected shape's read does (`reading.ElectedReads`). The + alternates' elections share one set of scope facts (`reading.ElectionFacts`). The candidate carries the election of the place it settled on (`_Candidate.reading`), and a reading whose composite refuses its table read reads nothing, so the alternate is dropped as one without data. @@ -519,6 +521,7 @@ def _materialise_alt_shapes( results: list[_Candidate] = [] unevaluated: list[tuple[_Alternate, AlternateUnevaluatedReason]] = [] + facts = ElectionFacts(coverage=coverage, operations=operations) for alternate, unsettled, record in readable: with recording_degraded_reads(into=record): bindings = list(alternate.bindings) @@ -528,7 +531,7 @@ def _materialise_alt_shapes( def read( read_at: PlaceContext, facet_batch: FacetBatch | None, - within: Within, + read_binding: ReadBinding, alternate: _Alternate = alternate, bindings: list[SlotBindingDraft] = bindings, presences: list[ConsultedPresence | None] = presences, @@ -536,7 +539,7 @@ def read( return materialise( alternate.shape, with_where(bindings, read_at), - read_at.roles.resolved_entities, + read_binding.entities(alternate.shape, read_at), graph=graph, sdmx=sdmx, date_request=date_request, @@ -547,7 +550,7 @@ def read( open_axes=read_at.roles.open_axes, coverage=coverage, facet_batch=facet_batch, - within=within, + within=read_binding.within, ) reads = ElectedReads( @@ -555,8 +558,7 @@ def read( bindings, date_request=date_request, selection=selection, - coverage=coverage, - operations=operations, + facts=facts, read=read, ) try: @@ -614,16 +616,15 @@ def read( if isinstance(result, Materialised) and result.has_data: if alternate.unverified: result = dataclasses.replace(result, membership_unverified=True) - read_at, reading = reads.settled(place) results.append( _Candidate( settled_shape(alternate.shape, result), - with_where(bindings, read_at), + with_where(bindings, place), result, - read_at.settled(result), + place.settled(result), alternate.walked, date_request, - reading=reading, + reading=reads.settled(place), ) ) return _AlternatesRead(results, None, presence, tuple(unevaluated), consulted) diff --git a/src/search/engine/region/grounding.py b/src/search/engine/region/grounding.py index 1b67d3c58..267cd6bb3 100644 --- a/src/search/engine/region/grounding.py +++ b/src/search/engine/region/grounding.py @@ -122,7 +122,7 @@ from origin_catalogue import Catalogue from search.engine.place import ConsultedPresence from search.engine.region.place_context import PlaceContext, UnsettledPlace - from search.engine.region.reading import Within + from search.engine.region.reading import ElectionFacts, ReadBinding from search.engine.retrieval.read_plan import ReadPlan from search.engine.retrieval.retrieve import Materialised from search.engine.retrieval.role_scope import DiscoveredScope @@ -1064,6 +1064,9 @@ class _RelaxationProbeContext: defaulted_properties: frozenset[str] # The groups the request named, kept because a relaxation keeps their entities. named_groups: tuple[NamedGroup, ...] + # The scope facts every relaxation's provider election shares (`reading.ElectionFacts`), + # over `coverage` and `operations`. + election_facts: ElectionFacts # Whether the part posted source mentions, so a suggestion's identity binds the # provenances that serve it (`spec_build.build_spec`). pin_served_provenance: bool = False @@ -1241,25 +1244,28 @@ def _relaxed_binding_suggestion( ground_warnings: list[Warning] | None = None probe_warning: Warning | None = None - def read(read_at: PlaceContext, facet_batch: FacetBatch | None, within: Within) -> object: + def read( + read_at: PlaceContext, facet_batch: FacetBatch | None, read_binding: ReadBinding + ) -> object: """The relaxed query's own resolve at `read_at`, the context the provider reading the relaxed bindings elect reads in (`reading.ElectedReads`), as the answer's read does: - a composite reads within its population, where a standard shape resolves as the - standard route answers it, over the context's entities and places or its - default-scope presence read, and a role-bearing shape confirms against coverage.""" + a composite reads within its population, binding its totals on the axes the part + leaves out (`reading.ReadBinding`), where a standard shape resolves as the standard route + answers it, over the context's entities and places or its default-scope presence + read, and a role-bearing shape confirms against coverage.""" if is_role_bearing(shape): return graph_confirm_resolve( shape=shape, bindings=relaxed, - subject_dcids=_relaxation_places(read_at), - origin_dcids=read_at.roles.origin_dcids, + subject_dcids=read_binding.places("destination", _relaxation_places(read_at)), + origin_dcids=read_binding.places("origin", read_at.roles.origin_dcids), graph=ctx.graph, sdmx=ctx.sdmx, date_request=ctx.date_request, deadline=ctx.deadline, open_axes=read_at.roles.open_axes, coverage_snapshot=ctx.coverage, - within=within, + within=read_binding.within, ) return standard_resolve( shape=shape, @@ -1277,15 +1283,14 @@ def read(read_at: PlaceContext, facet_batch: FacetBatch | None, within: Within) relaxed, date_request=ctx.date_request, selection=ctx.scope_selection, - coverage=ctx.coverage, - operations=ctx.operations, + facts=ctx.election_facts, read=read, ) try: with fail_open_reads("nearest-real suggestion", budget_s=settings().nearest_real_budget_s): - tried_place, mat = settle_sets(_relaxation_place(ctx, shape), reads) - bound, reading = reads.settled(tried_place) + bound, mat = settle_sets(_relaxation_place(ctx, shape), reads) + reading = reads.settled(bound) if not isinstance(mat, Materialised): return None, None, None, None # A flagged-unavailable carrier represents the original binding. @@ -1632,6 +1637,8 @@ def _suggest_nearest_real( that shape under the same bindings: the level reading of a shape that divides by a denominator (`phase_place_materialise._level_relaxation`). """ + from search.engine.region.reading import ElectionFacts + ctx = _RelaxationProbeContext( graph=graph, sdmx=sdmx, @@ -1651,6 +1658,7 @@ def _suggest_nearest_real( measure_elected_properties=measure_elected_properties, defaulted_properties=defaulted_properties, named_groups=named_groups, + election_facts=ElectionFacts(coverage=coverage, operations=operations), pin_served_provenance=pin_served_provenance, scope_selection=scope_selection, operations=operations, diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index 48ce0821e..d3e813572 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -107,7 +107,13 @@ withdraw_typed_unit, ) from search.engine.region.place_roles import request_named_groups -from search.engine.region.reading import ReadingDecision, ReadScope, decide_read_scope +from search.engine.region.reading import ( + ReadingDecision, + ReadScope, + decide_read_scope, + read_totals, + roles_read, +) from search.engine.region.related_read import ( cancel_related_reads, start_answer_related_read, @@ -590,7 +596,9 @@ async def _ground_elected_answer(state: ResolveState) -> RegionResult | None: state.shape_draft, state.bindings, mat_result.sv_dcids, - state.place.roles.roles, + # The answer lists the totals its read bound on axes the part leaves out, as a + # resubmit of its composite lists them, and its identity leaves them out. + roles_read(state.place.roles.roles, state.reading), state.graph, # The filtered plan names the sources: under a unit request, the facets it serves # in the elected units. A conversion preview may read facets outside it. @@ -689,8 +697,13 @@ async def _ground_elected_answer(state: ResolveState) -> RegionResult | None: for role_draft, entity_ref, entity_type_ref in entity_objects_data ] state.entity_objects = entity_objects - # Hash the same role-dispatched entities presented in the response. - identity_entities = build_identity_entities(entity_objects_data, role_refs) + # Hash the same role-dispatched entities presented in the response, but the totals the + # read bound, which the request did not name. + totals = read_totals(state.reading) + identity_entities = build_identity_entities( + [entity for entity in entity_objects_data if (entity[1].ref, entity[0]) not in totals], + role_refs, + ) state.identity_entities = identity_entities state.timing["answer"] = now_ms() - t0 diff --git a/src/search/engine/region/phase_place_materialise.py b/src/search/engine/region/phase_place_materialise.py index 495d3aae7..7b96e97be 100644 --- a/src/search/engine/region/phase_place_materialise.py +++ b/src/search/engine/region/phase_place_materialise.py @@ -133,7 +133,7 @@ undirected_place_name, whole_total_candidates, ) -from search.engine.region.reading import ElectedReads, ReadingDecision, Within +from search.engine.region.reading import ElectedReads, ElectionFacts, ReadBinding, ReadingDecision from search.engine.region.refs import provenance_labels from search.engine.region.requirements import record_reading, settle_answer, stray_values from search.engine.region.result import FacetAxisFilter, RegionResult @@ -1641,7 +1641,7 @@ async def _phase_facet_requests(state: ResolveState) -> RegionResult | None: class _ElectedRead: """The elected shape's read (`_materialise_task`): the place context it was read in, its outcome, and the provider reading elected before it (`reading.elect_reading`), None - where the part binds no total with a bilateral source.""" + where the part's identity counts no total with a bilateral source.""" place: PlaceContext outcome: _MaterialiseOutcome @@ -1686,16 +1686,18 @@ def _elected_read( Each read elects its provider reading first, at the place it tries, and reads the composite the reading names within its population (`reading.ElectedReads`), so a part that settles on a composite is read once, as a resubmit of the composite reads it. The - read the sets settle on carries its own election. + read the sets settle on carries its own election, and the place it was read at stays as + the request named it: a total the composite binds on an axis the part leaves out enters + the read's entities alone (`reading.ReadBinding`). """ def read( - read_at: PlaceContext, facet_batch: FacetBatch | None, within: Within + read_at: PlaceContext, facet_batch: FacetBatch | None, read_binding: ReadBinding ) -> _MaterialiseOutcome: return run_materialise( shape, with_where(bindings, read_at), - read_at.roles.resolved_entities, + read_binding.entities(shape, read_at), graph=state.graph, sdmx=state.sdmx, date_request=state.date_request, @@ -1705,7 +1707,7 @@ def read( open_axes=read_at.roles.open_axes, coverage=state.coverage, facet_batch=facet_batch, - within=within, + within=read_binding.within, ) reads = ElectedReads( @@ -1713,13 +1715,11 @@ def read( bindings, date_request=state.date_request, selection=state.scope_selection, - coverage=state.coverage, - operations=state.operations, + facts=ElectionFacts(coverage=state.coverage, operations=state.operations), read=read, ) place, outcome = settle_sets(target, reads) - read_at, reading = reads.settled(place) - return _ElectedRead(place=read_at, outcome=outcome, reading=reading) + return _ElectedRead(place=place, outcome=outcome, reading=reads.settled(place)) class _RelaxedPartRefused(Exception): diff --git a/src/search/engine/region/phase_shared.py b/src/search/engine/region/phase_shared.py index 26959d730..3b7b58288 100644 --- a/src/search/engine/region/phase_shared.py +++ b/src/search/engine/region/phase_shared.py @@ -38,6 +38,7 @@ ) from search.engine.region.qualifier_readings import shape_carries_measure_match from search.engine.region.reading import ( + ElectionFacts, ReadingDecision, ReadScope, decide_read_scope, @@ -523,8 +524,7 @@ def _flagged_reading(state: ResolveState) -> ReadingDecision | None: state.place, date_request=state.date_request, selection=state.scope_selection, - coverage=state.coverage, - operations=state.operations, + facts=ElectionFacts(coverage=state.coverage, operations=state.operations), ) diff --git a/src/search/engine/region/reading.py b/src/search/engine/region/reading.py index f43dbff71..a51503ad0 100644 --- a/src/search/engine/region/reading.py +++ b/src/search/engine/region/reading.py @@ -8,7 +8,10 @@ tried at, the scope entries and records its binders hold, the coverage snapshot, and the years the read reads (`extract.read_window`). The read can then read the composite's table within its population (`ReadingDecision.table`), and the answer's scope applies the same -choice to the identity minted after the read (`scope_decision.build_part_scope`). +choice to the identity minted after the read (`scope_decision.build_part_scope`). What the +composite's table read binds beyond the part's own places, the population's total in place +of a destination the part leaves out, is bound inside the read alone (`ReadBinding`), so the +part's place, bindings and minted identity stay as the request named them. The window is the read's own. A trailing request settles on a narrower range after the read (`trailing.settle_trailing`), always inside the read window (`trailing._assert_inside`), and @@ -26,7 +29,12 @@ from typing import TYPE_CHECKING from code_index.composites import composite_ref -from code_index.members import SourceNotInFamily, identity_family_query, index_family +from code_index.members import ( + SourceNotInFamily, + canonical_identity, + identity_family_query, + index_family, +) from code_index.query import CodeIndex from code_index.scope_conventions import BILATERAL_SOURCES from code_index.shared import get_index @@ -54,8 +62,8 @@ from search.engine.retrieval.retrieve import NoDataDraft from search.engine.shape import entities_by_role, identity_shape_for from search.engine.spec_id import identity_spec_id -from search.models import BaseIdentity, Operation, Spec, TimeWindow, Warning -from search.scope.provider import Withheld +from search.models import BaseIdentity, EntityRoleKey, Operation, Spec, TimeWindow, Warning +from search.scope.provider import Withheld, counted_source from search.scope.state import ScopeState from search.scope.tables import PartFacts, ScopeFacts from series_coverage.shared import CoverageSnapshot @@ -80,7 +88,7 @@ """The reason of a reading the election withheld because the bilateral destinations do not cover every year the read reads (`provider.bilateral_default_withheld`).""" -_BILATERAL_CODES = frozenset((row.code_system, row.code) for row in BILATERAL_SOURCES) +_BILATERAL_CODE_SYSTEMS = frozenset(row.code_system for row in BILATERAL_SOURCES) _REFUSALS: ContextVar[set[str] | None] = ContextVar("search_refused_composites", default=None) """The spec_ids of the composites whose table read refused in the bound resolution @@ -131,11 +139,11 @@ class ReadingDecision: choice: ProviderChoice table: TableRead | None - # The `entities` keys of `table.identity` on a flow axis the part neither names nor - # opens: the population's total a component binds in place of a destination the part - # leaves out (`RolePopulation.total`), which the read names as the composite's table - # read does and the composite's own identity leaves out. - left_out: frozenset[str] = frozenset() + # The entities of `table.identity`, by role key, on each flow axis the part neither + # names nor opens: the population's total a component binds in place of a destination + # the part leaves out (`RolePopulation.total`). The read binds them as the composite's + # table read does (`ReadBinding`), and the part's place and minted identity leave them out. + totals: Mapping[str, tuple[str, ...]] = dataclasses.field(default_factory=dict) # Whether the composite the part's scope compiles to refuses its table read # (`identity_edit.composite_table_identity`), so the part reads nothing and answers # no_data, `facet_not_available`: a fresh search does not refuse its scope, and the @@ -145,6 +153,34 @@ class ReadingDecision: refused: bool = False +class ElectionFacts: + """The scope facts the elections of one part's reads read (`elect_reading`): the + request's `coverage`, the part's `operations` and the dataset the code index elects for + each election's document. One `ScopeFacts` is held per dataset, so the elections of + every place a read tries (`ElectedReads`), of a part's alternates and of its nearest-real + relaxations share the family scans and coverage reads it memoises, each keyed by what it + reads. The elections run one at a time, as a part's reads do.""" + + def __init__(self, *, coverage: CoverageSnapshot, operations: Sequence[Operation]) -> None: + self.coverage = coverage + self._operations = tuple(operations) + self._held: dict[str | None, ScopeFacts] = {} + + def facts(self, index: CodeIndex, dataset: str | None) -> ScopeFacts: + """The facts of a part served from `dataset`, read in `index`; a part serving no + dataset yet where it is None.""" + held = self._held.get(dataset) + if held is None or held.index is not index: + datasets = () if dataset is None else (dataset,) + held = ScopeFacts( + index=index, + coverage=self.coverage, + part=PartFacts(operations=self._operations, datasets=datasets), + ) + self._held[dataset] = held + return held + + def elect_reading( shape: ShapeDraft, bindings: Sequence[SlotBindingDraft], @@ -152,20 +188,21 @@ def elect_reading( *, date_request: DateRequest | None, selection: ScopeState, - coverage: CoverageSnapshot, - operations: Sequence[Operation], + facts: ElectionFacts, ) -> ReadingDecision | None: """The provider reading of the part read for `shape` under `bindings` at `place`, one settled place the read tries, with the scope entries and records of `selection`, judged - in `coverage` over the years the read reads for `date_request` (`extract.read_window`). - None where no binding counts a published total with a bilateral source - (`BILATERAL_SOURCES`), such as DAC2a's aid type 206. + in `facts`' coverage over the years the read reads for `date_request` + (`extract.read_window`). None where the document the part's pre-read state names + (`_pre_read_identity`) counts no published total with a bilateral source, such as + DAC2a's aid type 206, by the rule the answer's scope reads after the read + (`provider.counted_source`). - The provider rules read an identity, so the election reads the document the part's - pre-read state names (`_pre_read_identity`). Its choice + The provider rules read an identity, so the election reads that document. Its choice (`scope_decision.choose_provider`) is built into a scope over that document to name the composite the scope compiles to, whose table read is elected here - (`ReadingDecision.table`). + (`ReadingDecision.table`), with the totals it binds on axes the part leaves out + (`ReadingDecision.totals`). An election that raises anywhere, building the document included, is logged as a scope decision is (`scope_decision.note_undecided`), and the part reads the table as total @@ -176,31 +213,30 @@ def elect_reading( Raises `EngineInputError` (`request_invalid`) for a reversed window, as the answer's mint refuses it (`grounding.requested_window_from_date_request`).""" - if not binds_bilateral_total(bindings): + if not _declares_bilateral_code_system(shape): return None identity: BaseIdentity | None = None - facts: ScopeFacts | None = None + scope_facts: ScopeFacts | None = None try: index = get_index() - identity = _pre_read_identity(shape, bindings, place, date_request=date_request) + identity = _pre_read_identity( + shape, bindings, place, date_request=date_request, index=index + ) + if counted_source(identity, index) is None: + return None # The facts of a part serving no dataset yet, which an election that raises past here # reads the bilateral default's reach in (`_undecided_election`). - facts = ScopeFacts( - index=index, coverage=coverage, part=PartFacts(operations=tuple(operations)) - ) - if (dataset := _elected_dataset(identity, index)) is not None: - facts = dataclasses.replace( - facts, part=dataclasses.replace(facts.part, datasets=(dataset,)) - ) - choice = choose_provider(identity, _snapshot(selection), facts) - decision = build_part_scope(identity, choice, spec_id=_ELECTION, facts=facts) + scope_facts = facts.facts(index, None) + scope_facts = facts.facts(index, _elected_dataset(identity, index)) + choice = choose_provider(identity, _snapshot(selection), scope_facts) + decision = build_part_scope(identity, choice, spec_id=_ELECTION, facts=scope_facts) if not decision.composite: return ReadingDecision(choice=choice, table=None) assert decision.scope is not None composite = decision.scope.compiled.identity try: table = composite_table_identity( - composite, pointer="/input/identity", coverage=coverage + composite, pointer="/input/identity", coverage=facts.coverage ) except EngineInputError as exc: composite_id = identity_spec_id(composite) @@ -214,26 +250,37 @@ def elect_reading( note_degraded_read(_ELECTION) return ReadingDecision( choice=undecided_choice( - identity, selection=selection, facts=facts, subject=_ELECTION + identity, selection=selection, facts=scope_facts, subject=_ELECTION ), table=None, refused=True, ) - asked = {role_key_axis(key) for key in identity.entities} | set(identity.open_roles or ()) - left_out = frozenset( - key for key in table.identity.entities if role_key_axis(key) not in asked - ) - return ReadingDecision(choice=choice, table=table, left_out=left_out) + return ReadingDecision(choice=choice, table=table, totals=_totals(identity, table)) except Exception as exc: if identity is None and isinstance(exc, EngineInputError) and exc.code == "request_invalid": # A reversed window, refused building the document (`_read_window`). raise note_undecided(_ELECTION, exc) return ReadingDecision( - choice=_undecided_election(identity, selection=selection, facts=facts), table=None + choice=_undecided_election(identity, selection=selection, facts=scope_facts), + table=None, ) +def _totals(identity: BaseIdentity, table: TableRead) -> dict[str, tuple[str, ...]]: + """The entities `table`, the composite's table read for the part of `identity`, binds on + a flow axis `identity` neither names nor opens, by role key, left as bound once each + population role is open within its members (`identity_edit.open_within`): the + population's total a component binds in place of a destination the part leaves out.""" + asked = {role_key_axis(key) for key in identity.entities} | set(identity.open_roles or ()) + opened, _within = open_within(table.identity, table.populations) + return { + key: tuple(dcids) + for key, dcids in opened.entities.items() + if (axis := role_key_axis(key)) is not None and axis not in asked + } + + def _undecided_election( identity: BaseIdentity | None, *, selection: ScopeState, facts: ScopeFacts | None ) -> ProviderChoice: @@ -256,8 +303,7 @@ def elect_at_read( *, date_request: DateRequest | None, selection: ScopeState, - coverage: CoverageSnapshot, - operations: Sequence[Operation], + facts: ElectionFacts, ) -> ReadingDecision | None: """The provider reading a read of `shape` under `bindings` at `place` elects (`elect_reading`), as every read of a part elects it: the definite answer's, each @@ -265,13 +311,7 @@ def elect_at_read( refuses it with `request_invalid`, where a read that reaches the mint refuses it.""" try: return elect_reading( - shape, - bindings, - place, - date_request=date_request, - selection=selection, - coverage=coverage, - operations=operations, + shape, bindings, place, date_request=date_request, selection=selection, facts=facts ) except EngineInputError as exc: if exc.code != "request_invalid": @@ -279,46 +319,107 @@ def elect_at_read( return None -def read_within( - shape: ShapeDraft, place: PlaceContext, reading: ReadingDecision | None -) -> tuple[PlaceContext, Within]: - """The context a read of `shape` elected as `reading` at `place` reads in, and the - members each open axis is discovered within (`graph_confirm_resolve`): for a composite, - its table read (`ReadingDecision.table`) with each role bound to a population open within - its members (`identity_edit.open_within`), and the population's total a component binds - in place of a destination the part leaves out named on its axis - (`ReadingDecision.left_out`), as a resubmit of the composite reads it. `place` itself, - discovering no axis within members, for any other reading.""" +type Within = dict[RoleAxis, frozenset[str]] | None +"""The members each open axis of a read is discovered within (`ReadBinding.within`), None where it +discovers no axis within members.""" + + +@dataclasses.dataclass(frozen=True) +class ReadBinding: + """What a read of a part binds beyond the context it reads in, as the provider reading + it elected names it (`read_binding`): `within`, the members each open axis is discovered + within (`graph_confirm_resolve`), and `totals`, the population's total a component binds + on each flow axis the part leaves out (`ReadingDecision.totals`). A read of a composite + binds both, as a resubmit of the composite reads its table; a read of the table binds + neither. The totals enter the read's entities alone, so the part's place, bindings and + minted identity name what the request named.""" + + within: Within = None + totals: Mapping[RoleAxis, tuple[str, ...]] = dataclasses.field(default_factory=dict) + + def places(self, axis: RoleAxis, dcids: Sequence[str]) -> list[str]: + """`dcids`, the places a read binds on `axis`, followed by the totals it binds there.""" + return [*dcids, *(dcid for dcid in self.totals.get(axis, ()) if dcid not in dcids)] + + def entities(self, shape: ShapeDraft, place: PlaceContext) -> dict[str, list[str]]: + """The entities by role name a read of `shape` at `place` binds: the context's own + (`PlaceRoles.resolved_entities`) and the totals, each on its axis.""" + entities = place.roles.resolved_entities + if not self.totals: + return entities + bound = {name: list(dcids) for name, dcids in entities.items()} + for name, dcids in entities_by_role(shape=shape, by_axis=self.totals).items(): + held = bound.setdefault(name, []) + held.extend(dcid for dcid in dcids if dcid not in held) + return bound + + +def read_binding(reading: ReadingDecision | None) -> ReadBinding: + """What a read elected as `reading` binds beyond its context (`ReadBinding`): for a + composite, each role its table read (`ReadingDecision.table`) binds to a population open + within the population's members (`identity_edit.open_within`), and the totals it binds + on the axes the part leaves out (`ReadingDecision.totals`). Nothing for any other + reading.""" if reading is None or reading.table is None: - return place, None - opened, within = open_within(reading.table.identity, reading.table.populations) - if not reading.left_out: - return place, within - return _naming(shape, place, {key: opened.entities[key] for key in reading.left_out}), within + return ReadBinding() + _opened, within = open_within(reading.table.identity, reading.table.populations) + totals: dict[RoleAxis, tuple[str, ...]] = {} + for key, dcids in reading.totals.items(): + axis = role_key_axis(key) + assert axis is not None, f"{key!r} names no flow axis" + totals[axis] = (*totals.get(axis, ()), *dcids) + return ReadBinding(within=within, totals=totals) -type Within = dict[RoleAxis, frozenset[str]] | None -"""The members each open axis of a read is discovered within (`read_within`), None where it -discovers no axis within members.""" +def read_totals(reading: ReadingDecision | None) -> frozenset[tuple[str, str]]: + """Each total the read elected as `reading` binds on an axis the part leaves out + (`ReadingDecision.totals`), as a dcid and its role key. The reading's Spec lists them + among its entities (`roles_read`), and its identity leaves them out.""" + if reading is None or reading.table is None: + return frozenset() + return frozenset((dcid, key) for key, dcids in reading.totals.items() for dcid in dcids) + + +def roles_read( + roles: Sequence[tuple[str, EntityRoleKey]], reading: ReadingDecision | None +) -> list[tuple[str, EntityRoleKey]]: + """`roles`, a context's wire roles, with the totals the read elected as `reading` binds + (`read_totals`), destinations first, as a resubmit of the composite binds the places of + its table identity (`promote_places._posted_place_roles`): the entities the reading's + Spec lists. `roles` itself where the read binds no total.""" + totals = read_totals(reading) + if not totals: + return list(roles) + by_axis: dict[RoleAxis, list[str]] = {"destination": [], "origin": []} + for dcid, key in [*roles, *sorted(totals)]: + axis = role_key_axis(key) + assert axis is not None, f"{key!r} names no flow axis" + if dcid not in by_axis[axis]: + by_axis[axis].append(dcid) + return [(dcid, "destination") for dcid in by_axis["destination"]] + [ + (dcid, "origin") for dcid in by_axis["origin"] + ] + -type WithinRead[T] = Callable[[PlaceContext, FacetBatch | None, Within], T] -"""A read of a part at the context it reads in, with its facet batch and the members each -open axis is discovered within.""" +type WithinRead[T] = Callable[[PlaceContext, FacetBatch | None, ReadBinding], T] +"""A read of a part at the context it reads in, with its facet batch and what it binds +beyond that context (`ReadBinding`).""" class ElectedReads[T]: """The read of `shape` under `bindings` at each place a place-set settlement tries (`place_context.settle_sets`), the definite answer's, each alternate's and each nearest-real suggestion's. Each read elects its provider reading first, at the place it - tries (`elect_at_read`), and reads within the composite's population that reading names - (`read_within`) through `read`, so a part that settles on a composite is read once, as a - resubmit of the composite reads it. A composite that refuses its table read reads nothing - and answers no_data, `facet_not_available`. + tries (`elect_at_read`), with the scope facts the part's elections share (`facts`), and + reads within the composite's population that reading names (`read_binding`) through `read`, + so a part that settles on a composite is read once, as a resubmit of the composite reads + it. A composite that refuses its table read reads nothing and answers no_data, + `facet_not_available`. - Each place tried is held by identity with the context it was read in and its election, - so its identity is not reused while the settlement runs, and the place the sets settle on - gives both (`settled`). `bindings` is read at each call, so a caller that edits the list - between rounds reads the edited bindings.""" + Each place tried is held by identity with its election, so its identity is not reused + while the settlement runs, and the place the sets settle on gives its election + (`settled`). `bindings` is read at each call, so a caller that edits the list between + rounds reads the edited bindings.""" def __init__( self, @@ -327,18 +428,16 @@ def __init__( *, date_request: DateRequest | None, selection: ScopeState, - coverage: CoverageSnapshot, - operations: Sequence[Operation], + facts: ElectionFacts, read: WithinRead[T], ) -> None: self._shape = shape self._bindings = bindings self._date_request = date_request self._selection = selection - self._coverage = coverage - self._operations = operations + self._facts = facts self._read = read - self._tried: dict[int, tuple[PlaceContext, PlaceContext, ReadingDecision | None]] = {} + self._tried: dict[int, tuple[PlaceContext, ReadingDecision | None]] = {} def __call__(self, place: PlaceContext, facet_batch: FacetBatch | None) -> T | NoDataDraft: reading = elect_at_read( @@ -347,20 +446,16 @@ def __call__(self, place: PlaceContext, facet_batch: FacetBatch | None) -> T | N place, date_request=self._date_request, selection=self._selection, - coverage=self._coverage, - operations=self._operations, + facts=self._facts, ) - read_at, within = read_within(self._shape, place, reading) - self._tried[id(place)] = (place, read_at, reading) + self._tried[id(place)] = (place, reading) if reading is not None and reading.refused: return NoDataDraft(reason="facet_not_available") - return self._read(read_at, facet_batch, within) + return self._read(place, facet_batch, read_binding(reading)) - def settled(self, place: PlaceContext) -> tuple[PlaceContext, ReadingDecision | None]: - """The context the read at `place`, a place this settlement tried, read in, and the - reading it elected.""" - _place, read_at, reading = self._tried[id(place)] - return read_at, reading + def settled(self, place: PlaceContext) -> ReadingDecision | None: + """The reading the read at `place`, a place this settlement tried, elected.""" + return self._tried[id(place)][1] @dataclasses.dataclass(frozen=True) @@ -416,12 +511,12 @@ def decide_read_scope( The scope applies the provider choice `reading` elected (`scope_decision.build_part_scope`) and chooses nothing, so the reading that labels the Spec is the one that read its rows. A reading that elected none applies the choice its own identity makes - (`_unelected_choice`). The scope is built over the identity the request names, without - the population's total the read named in place of a destination the part leaves out - (`request_identity`), and checked against the scope decided over it (`_checked`). A + (`_unelected_choice`). The scope is built over the reading's own identity, which names + what the request named, and checked against the scope decided over it (`_checked`). A scope that compiles a composite mints the Spec as it (`MintedComposite`), with the - composite's components listed on its compiled scope. A decision that raises leaves the - reading as a single table that states no scope (`scope_decision.guarded_scope_decision`). + composite's components listed on its compiled scope and the identity its rows were read + for (`_read_identity`). A decision that raises leaves the reading as a single table that + states no scope (`scope_decision.guarded_scope_decision`). A read made within a composite's population (`ReadingDecision.table`) holds that population's rows alone, so it fails closed: where its scope cannot be decided, the mint @@ -431,7 +526,7 @@ def decide_read_scope( # Deferred: resubmit imports the answer phase, which imports this module. from search.engine.region.resubmit import composite_mint - identity = request_identity(answer.identity, reading) + identity = answer.identity minted: list[MintedComposite] = [] def decide() -> ScopeDecision: @@ -459,7 +554,7 @@ def decide() -> ScopeDecision: minted.append( MintedComposite( mint=mint, - read=answer.identity, + read=_read_identity(identity, reading), populations=reading.table.populations if reading is not None and reading.table is not None else {}, @@ -507,16 +602,14 @@ def _unminted(answer: AnswerIdentity, exc: Exception | None) -> ReadScope: return ReadScope(answer=answer, decision=None, minted=None) -def request_identity(identity: BaseIdentity, reading: ReadingDecision | None) -> BaseIdentity: - """`identity`, the identity minted from the read, without the entities the read named in - place of a destination the part leaves out (`ReadingDecision.left_out`): the identity the - request names, whose composite reads that destination at the population's total.""" - if reading is None or not reading.left_out: +def _read_identity(identity: BaseIdentity, reading: ReadingDecision | None) -> BaseIdentity: + """`identity`, the identity minted from the read elected as `reading`, with the totals + the read bound on the axes the part leaves out (`ReadingDecision.totals`): the identity + its rows were read for, as a resubmit of the composite reads its table identity.""" + if reading is None or not reading.totals: return identity document = identity.model_dump(exclude_none=True) - document["entities"] = { - key: dcids for key, dcids in identity.entities.items() if key not in reading.left_out - } + document["entities"] = {**identity.entities, **{k: list(v) for k, v in reading.totals.items()}} return BaseIdentity.model_validate(document) @@ -621,55 +714,17 @@ def _composite_of(decision: ScopeDecision) -> str | None: ) -def _naming( - shape: ShapeDraft, place: PlaceContext, named: Mapping[str, Sequence[str]] -) -> PlaceContext: - """`place`, the context of role-bearing `shape`, with the places `named` maps by flow role - key bound on that role's axis after those it binds, its destinations as its where places, - as a posted identity's places bind (`promote_places._posted_place_roles`).""" - roles = place.roles - by_axis: dict[RoleAxis, list[str]] = { - "destination": list(roles.destination_dcids), - "origin": list(roles.origin_dcids), - } - for key, dcids in named.items(): - axis = role_key_axis(key) - assert axis is not None, f"{key!r} names no flow axis" - by_axis[axis].extend(dcid for dcid in dcids if dcid not in by_axis[axis]) - destinations, origins = by_axis["destination"], by_axis["origin"] - bound = dataclasses.replace( - roles, - destination_dcids=destinations, - origin_dcids=origins, - resolved_entities=entities_by_role(shape=shape, by_axis=by_axis), - roles=[(dcid, "destination") for dcid in destinations] - + [(dcid, "origin") for dcid in origins], - where_dcids=destinations, - obs_destination_dcid=destinations[0] if destinations else None, - ) - probe = None if len(destinations) <= 1 else tuple(destinations) - return dataclasses.replace(place, roles=bound, probe_dcids=probe) - - def _snapshot(selection: ScopeState) -> ScopeState: """`selection` as it stands, so a record a binder writes after the election leaves the choice's selection as it was elected.""" return dataclasses.replace(selection, records=dict(selection.records)) -def binds_bilateral_total(bindings: Sequence[SlotBindingDraft]) -> bool: - """Whether some binding of `bindings` binds a bilateral source's total as a value - (`BILATERAL_SOURCES`), the parts `elect_reading` elects a reading for.""" - return any(_binds_bilateral_code(binding) for binding in bindings) - - -def _binds_bilateral_code(binding: SlotBindingDraft) -> bool: - """Whether `binding` binds a bilateral source's total as a value (`BILATERAL_SOURCES`).""" - return ( - binding.kind == "value" - and len(binding.value_dcids) == 1 - and (binding.property_dcid, binding.value_dcids[0]) in _BILATERAL_CODES - ) +def _declares_bilateral_code_system(shape: ShapeDraft) -> bool: + """Whether `shape` declares a slot on a bilateral source's code system + (`BILATERAL_SOURCES`), as every part whose identity can count such a source's total + does, bound in its draft or by its dataset's invariants once canonical.""" + return any(slot.property_dcid in _BILATERAL_CODE_SYSTEMS for slot in shape.slot_keys) def _pre_read_identity( @@ -678,27 +733,32 @@ def _pre_read_identity( place: PlaceContext, *, date_request: DateRequest | None, + index: CodeIndex, ) -> BaseIdentity: - """The identity document the provider rules read for a part before its read: the - shape's five-tuple, each value, set or unbound binding on the what and how axes as a - slot, the entities `place` binds by role key and the axes it leaves open, the read window - (`_read_window`), and a provenance pin to the dataset the bindings or the shape name. - - It is no answer identity: the mint after the read binds invariant slots and settles the - window, and neither moves the provider choice.""" + """The identity the read will mint for a part, as far as its pre-read state names it: + the shape's five-tuple, a slot for each what and how slot key of `shape` bound as its + draft in `bindings` binds a value or a set of two or more and unbound otherwise, as the + answer's identity states its slots (`spec_build.build_slot`), the entities `place` binds + by role key and the axes it leaves open, the read window (`_read_window`), and a + provenance pin to the dataset the bindings or the shape name. It is in canonical form + (`code_index.members.canonical_identity`), so each slot an invariant of that dataset + fixes binds the invariant's value, as in the minted identity. + + It is no answer identity: the mint after the read settles the window, which does not + move the provider choice.""" + drafts = {binding.property_dcid: binding for binding in bindings if binding.property_dcid} slots = [] - for binding in bindings: - if binding.axis not in ("what", "how") or not binding.property_dcid: + for key in shape.slot_keys: + if key.axis not in ("what", "how") or not key.property_dcid: continue - if binding.kind == "value" and len(binding.value_dcids) == 1: - member: dict[str, object] = {"state": "value", "ref": binding.value_dcids[0]} - elif binding.kind == "set" and binding.value_dcids: - member = {"state": "set", "refs": sorted(binding.value_dcids)} - elif binding.kind == "unbound": - member = {"state": "unbound"} - else: - continue - slots.append({"property": binding.property_dcid, "binding": member}) + binding = drafts.get(key.property_dcid) + values = [] if binding is None else binding.value_dcids + member: dict[str, object] = {"state": "unbound"} + if binding is not None and binding.kind == "value" and values: + member = {"state": "value", "ref": values[0]} + elif binding is not None and binding.kind == "set" and len(values) >= 2: + member = {"state": "set", "refs": sorted(values)} + slots.append({"property": key.property_dcid, "binding": member}) entities: dict[str, list[str]] = {} for dcid, key in place.roles.roles: held = entities.setdefault(key, []) @@ -723,7 +783,7 @@ def _pre_read_identity( document["open_roles"] = sorted(place.roles.open_axes) if dataset := bound_source(bindings) or shape.named_dataset: document["facets"] = {"provenance": {"state": "value", "ref": dataset}} - return BaseIdentity.model_validate(document) + return BaseIdentity.model_validate(canonical_identity(index, document)) def _read_window(date_request: DateRequest | None) -> dict[str, object]: diff --git a/src/search/engine/region/state.py b/src/search/engine/region/state.py index faed6e581..855c872d7 100644 --- a/src/search/engine/region/state.py +++ b/src/search/engine/region/state.py @@ -943,8 +943,8 @@ class ResolveState: # (`reading.elect_reading`), set beside `mat_result` by # `phase_place_materialise._run_materialise_task`: the choice the definite answer's # scope is built from (`phase_answer._decide_answer_scope`) and the composite's table read - # the read read within. None where the part binds no total with a bilateral source, and - # where a stand-in's read answers in place of the elected shape's. + # the read read within. None where the part's identity counts no total with a bilateral + # source, and where a stand-in's read answers in place of the elected shape's. reading: ReadingDecision | None = None # The composite the definite answer's Spec is minted as (`spec_build.build_spec`), with # the identity its rows were read for and the populations its roles were read within, diff --git a/src/search/scope/provider.py b/src/search/scope/provider.py index 38ef2234a..f19217464 100644 --- a/src/search/scope/provider.py +++ b/src/search/scope/provider.py @@ -703,6 +703,16 @@ def _default_bilateral_option( return reached.source, _bilateral_option(identity, reached, facts) +def counted_source(identity: BaseIdentity, index: CodeIndex) -> BilateralSource | None: + """The bilateral source whose published total `identity`'s bound slots count + (`edits.counted_bilateral_source`), declared for the dataset `index` elects for + `identity`'s family: the one rule deciding whether the bilateral default can reach a + reading, read by the default itself (`_default_rule`) and by the election that settles + a part's provider reading before its read (`reading.elect_reading`). None where no + source is counted.""" + return counted_bilateral_source(identity, dataset_of=lambda held: _dataset(held, index)) + + def bilateral_default_reaches(identity: BaseIdentity, facts: ScopeFacts) -> bool: """Whether the bilateral default reaches `identity` (`with_bilateral_default`), whether or not its option is available. It reads the code index alone, never coverage.""" @@ -716,7 +726,7 @@ def _default_rule(identity: BaseIdentity, facts: ScopeFacts) -> DestinationRule otherwise.""" if not isinstance(identity.shape, IdentityShapeSchema): return None - source = counted_bilateral_source(identity, dataset_of=lambda held: _dataset(held, facts.index)) + source = counted_source(identity, facts.index) if source is None: return None pin = (identity.facets or {}).get("provenance") diff --git a/src/search/scope/tables.py b/src/search/scope/tables.py index 83e931ad4..cf8d41576 100644 --- a/src/search/scope/tables.py +++ b/src/search/scope/tables.py @@ -120,9 +120,10 @@ class FamilyValues: class ScopeFacts: """The engine state the axis rules read: the release's code index, the request's coverage snapshot and the part's own facts (`PartFacts`), all in memory. Passed - explicitly, so a rule reads no global. One `ScopeFacts` serves one build of one part's - scope, and memoises the family scans its rules share (`family_values`) and the years a - composite's components hold (`remembered`).""" + explicitly, so a rule reads no global. One `ScopeFacts` serves the scope builds of one + part over these facts, the elections of its reads included (`reading.ElectionFacts`), + and memoises the family scans its rules share (`family_values`) and the years a + composite's components hold (`remembered`), each keyed by what it reads.""" index: CodeIndex coverage: CoverageSnapshot diff --git a/tests/engine/test_dac2a_suggestion_readings.py b/tests/engine/test_dac2a_suggestion_readings.py index 21ce8267b..e2e341e1b 100644 --- a/tests/engine/test_dac2a_suggestion_readings.py +++ b/tests/engine/test_dac2a_suggestion_readings.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio +import dataclasses import logging from collections.abc import Callable from typing import Any @@ -36,7 +37,9 @@ ) from tests.engine._harness import offline_resolve from tests.engine.test_dac2a_destination_populations import ( + _AT_TOTAL, _BILATERAL, + _DEVELOPING, _FRA, _KEN, _VARIABLE, @@ -45,6 +48,7 @@ _ended_state, _graph_with_parties, _identity, + _patched_coverage, _resolve_resubmit, _resubmit, _ScriptedLLM, @@ -360,3 +364,56 @@ def confirmed(**kwargs: Any) -> Any: assert any(within for within in opened_within) monitor = [r for r in caplog.records if r.name == reading.__name__ and r.levelno >= 30] assert monitor == [] + + +@pytest.mark.parametrize("scenario", ["D2", "D2-ranking"]) +def test_a_part_naming_no_destination_reads_the_total_inside_its_read_alone( + scenario: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """A part naming no destination reads the bilateral composite at the population's total + (`_DEVELOPING`) inside its read alone (`reading.ReadBinding`): its place after the read and + the where slot of its bindings name no destination, as before the read. A nearest-real + suggestion read at that place elects there, reads at the total in its own read and mints + an identity binding no destination, as the composite the answer is minted as binds + none.""" + _patched_coverage(monkeypatch, _AT_TOTAL) + before: list[ResolveState] = [] + task = phase_place_materialise._materialise_task + + def reading_from(state: ResolveState) -> Any: + before.append(dataclasses.replace(state.place)) + return task(state) + + monkeypatch.setattr(phase_place_materialise, "_materialise_task", reading_from) + state = _ended_state(monkeypatch, scenario, sdmx=_sdmx(_AT_TOTAL)) + + assert state.reading is not None and state.reading.table is not None + (unread,) = before + for place in (unread, state.place): + assert place.roles.destination_dcids == [] + assert place.roles.where_dcids == [] + assert place.roles.obs_destination_dcid != _DEVELOPING + assert not [b for b in state.bindings if b.axis == "where" and b.value_dcids] + + # The relaxation reads at the part's own place, as for a shape installed with its + # settled context alone (`grounding._relaxation_place`). + state.unsettled_place = None + read = state.mat_result + subjects: list[list[str]] = [] + + def confirmed(**kwargs: Any) -> Any: + subjects.append(list(kwargs["subject_dcids"])) + return read + + monkeypatch.setattr(role_resolve, "graph_confirm_resolve", confirmed) + level = ( + state.shape_draft, + Relaxation(axis="what", property_dcid="measurementDenominator", dropped_value_dcids=["x"]), + ) + near = asyncio.run(phase_place_materialise._nearest_real_for(state, level=level)) + + assert near is not None + (suggestion,) = near + assert _provenance(suggestion) == _BILATERAL + assert "destination" not in suggestion.definition.identity.entities + assert subjects == [[_DEVELOPING]] diff --git a/tests/engine/test_degraded_read_sites.py b/tests/engine/test_degraded_read_sites.py index c0ae10c2f..1dd38211b 100644 --- a/tests/engine/test_degraded_read_sites.py +++ b/tests/engine/test_degraded_read_sites.py @@ -15,7 +15,7 @@ from concurrent.futures import Future, ThreadPoolExecutor from contextlib import nullcontext from types import SimpleNamespace -from typing import cast +from typing import Any, cast import pydantic import pytest @@ -51,6 +51,7 @@ from search.engine.region.availability import read_coverage from search.engine.region.codes import OBSERVATIONS_TIMED_OUT from search.engine.region.phase_rank_bind import _stray_requirements +from search.engine.region.reading import ElectionFacts from search.engine.region.related_read import PendingRelated from search.engine.region.state import ResolveState from search.engine.retrieval import bound_values, derive @@ -130,7 +131,7 @@ def _relaxation_ctx(**changes: object) -> SimpleNamespace: **{ "graph": None, "sdmx": None, - "shape_draft": SimpleNamespace(sv_arc_facts={"Count_Person": {}}), + "shape_draft": SimpleNamespace(sv_arc_facts={"Count_Person": {}}, slot_keys=[]), "place": SimpleNamespace( roles=roles, default_scope="countries", @@ -145,6 +146,7 @@ def _relaxation_ctx(**changes: object) -> SimpleNamespace: "coverage": None, "scope_selection": ScopeState(), "operations": (), + "election_facts": ElectionFacts(coverage=cast(Any, None), operations=()), **changes, } ) diff --git a/tests/engine/test_reading_decision.py b/tests/engine/test_reading_decision.py index e6f067558..7a5548c04 100644 --- a/tests/engine/test_reading_decision.py +++ b/tests/engine/test_reading_decision.py @@ -20,6 +20,7 @@ import pytest +from code_index import members from code_index.composites import destination_population from code_index.declarations import DAC2A_INCLUSIVE from code_index.shared import get_index @@ -36,7 +37,7 @@ ) from search.engine.region.candidates import _Candidate from search.engine.region.place_context import empty_place_context -from search.engine.region.reading import ReadingDecision, elect_reading +from search.engine.region.reading import ElectionFacts, ReadingDecision, elect_reading from search.engine.region.scope_decision import ( READ_WINDOW_NOT_COVERED, ProviderChoice, @@ -69,6 +70,7 @@ _HELD, _SCENARIOS, _TOTAL_YEARS, + _ended_state, _fresh_request, _graph_with_parties, _identity, @@ -120,8 +122,7 @@ def electing_task(state: Any) -> Any: state.place, date_request=state.date_request, selection=copy.deepcopy(state.scope_selection), - coverage=state.coverage, - operations=state.operations, + facts=ElectionFacts(coverage=state.coverage, operations=state.operations), ) ) return task(state) @@ -789,8 +790,7 @@ def elect() -> ReadingDecision | None: empty_place_context(), date_request=None, selection=ScopeState(), - coverage=cast(Any, None), - operations=(), + facts=ElectionFacts(coverage=cast(Any, None), operations=()), ) if raises: @@ -905,3 +905,76 @@ async def no_suggestion(*_args: object, **_kwargs: object) -> None: (w.spec_id, w.params) for w in part.warnings if w.code == "bilateral_reading_unavailable" ] assert withheld == [(flagged.spec_id, {"why": "option_not_covered"})] + + +def test_a_part_whose_dataset_fixes_the_aid_type_elects_the_bilateral_composite( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A part whose draft leaves DAC2a's aid type unbound while the dataset it names fixes + 206 as an invariant mints an identity binding 206 (`code_index.members.canonical_identity`), + which the bilateral default reaches after the read. The election reads that identity + before the read by the same rule (`provider.counted_source`), so it elects the bilateral + composite and reads within its population, as the part's answer is minted.""" + state = _ended_state(monkeypatch, "H1") + elected_invariants = members.elected_invariants + + def fixing_206(index: Any, query: Any) -> dict[str, str]: + if query.source == _DAC2A: + return {"dac2aAidType": _AID_TYPE_206.value_dcids[0]} + return elected_invariants(index, query) + + monkeypatch.setattr(members, "elected_invariants", fixing_206) + bindings = [ + binding.model_copy(update={"kind": "unbound", "value_dcids": []}) + if binding.property_dcid == "dac2aAidType" + else binding + for binding in state.bindings + ] + assert not any(b.property_dcid == "dac2aAidType" and b.value_dcids for b in bindings) + + elected = elect_reading( + dataclasses.replace(state.shape_draft, named_dataset=_DAC2A), + bindings, + state.place, + date_request=state.date_request, + selection=ScopeState(), + facts=ElectionFacts(coverage=state.coverage, operations=state.operations), + ) + + assert elected is not None and elected.table is not None + assert elected.choice.bilateral == _BILATERAL + + +def test_the_elections_of_one_part_s_reads_share_one_set_of_scope_facts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Each read of a part elects at the place it tries, a set's node and then its members, + and each nearest-real relaxation elects at its own read. All of them read one + `ScopeFacts` per dataset (`reading.ElectionFacts`), so its family scans and coverage + reads run once for the part.""" + state = _ended_state(monkeypatch, "H1") + read: list[ScopeFacts] = [] + choose = reading.choose_provider + + def choosing(identity: Any, selection: Any, facts: ScopeFacts) -> Any: + read.append(facts) + return choose(identity, selection, facts) + + monkeypatch.setattr(reading, "choose_provider", choosing) + facts = ElectionFacts(coverage=state.coverage, operations=state.operations) + node, members = state.place, dataclasses.replace(state.place) + + for bindings in (list(state.bindings), list(state.bindings)): + reads = reading.ElectedReads( + state.shape_draft, + bindings, + date_request=state.date_request, + selection=ScopeState(), + facts=facts, + read=lambda *_args: "read", + ) + assert [reads(place, None) for place in (node, members)] == ["read", "read"] + assert reads.settled(members) is not None + + assert len(read) == 4 + assert all(facts is read[0] for facts in read) From f873a21ca452452111c49a9c5034da7c3792a4f1 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 20:46:00 +0100 Subject: [PATCH 09/12] fix(search): state a composite's defaulted role scope as its Spec counts it CompositeMint.role_scope_warnings takes the count from the Spec's own role axis facts, so the warning and the facts agree when a component's parties cannot be listed, and a composite candidate whose scope lists no member drops the warning as any other candidate does. --- .../engine/region/candidate_grounding.py | 10 +-- src/search/scope/composite.py | 23 +++---- tests/engine/test_composite_mint.py | 63 ++++++++++++++++--- 3 files changed, 73 insertions(+), 23 deletions(-) diff --git a/src/search/engine/region/candidate_grounding.py b/src/search/engine/region/candidate_grounding.py index 19131938d..4b78bf048 100644 --- a/src/search/engine/region/candidate_grounding.py +++ b/src/search/engine/region/candidate_grounding.py @@ -1131,7 +1131,8 @@ def restate_scope_warning(state: ResolveState, entries: Sequence[_GroundedCandid over every kept candidate, and the collapse over the candidates the unit filter keeps, so a lone survivor's warning counts its own scope, as its resubmit does. Where the elected candidate's Spec is minted as a composite, the count is the composite's parties - (`CompositeMint.role_scope_warnings`), as the definite answer's is.""" + (`CompositeMint.role_scope_warnings`), as the definite answer's is, and a scope that + lists no member removes the warning, as it does for any candidate.""" if state.opposite_labels is None or not isinstance(state.mat_result, Materialised): return axis = scope_axis(state.mat_result) @@ -1148,15 +1149,14 @@ def restate_scope_warning(state: ResolveState, entries: Sequence[_GroundedCandid axis=axis, ) own = next((entry for entry in entries if entry.candidate.mat is state.mat_result), None) - if own is not None and own.minted is not None and len(distinct) <= 1: + members = own.scope_members if own is not None and own.scope_members is not None else on_axis[0] + if own is not None and own.minted is not None and len(distinct) <= 1 and members: state.warnings[:] = own.minted.mint.role_scope_warnings(state.warnings, own.spec) return restate_role_scope_warnings( state.warnings, state.mat_result, - members=own.scope_members - if own is not None and own.scope_members is not None - else on_axis[0], + members=members, boundary_roles=state.place.roles.roles, label_of=state.opposite_labels, unnamed_wholes=state.place.roles.unnamed_wholes, diff --git a/src/search/scope/composite.py b/src/search/scope/composite.py index ef79ca7da..3e3a71add 100644 --- a/src/search/scope/composite.py +++ b/src/search/scope/composite.py @@ -478,9 +478,9 @@ def parties(self, axis: RoleAxis) -> list[str] | None: def member_count(self, axis: RoleAxis) -> int | None: """How many parties `parties(axis)` lists, as `enrichment.party_count` counts them - in the components' datasets. It is the count of each `RoleAxisFact` `open_axes` states - and of the `role_scope_defaulted` warning (`role_scope_warnings`). None where they - cannot be listed.""" + in the components' datasets. It is the count of each `RoleAxisFact` `open_axes` states, + which the `role_scope_defaulted` warning states (`role_scope_warnings`). None where + they cannot be listed.""" held = self._held_parties if held is None: return None @@ -604,18 +604,19 @@ def role_scope_warnings(self, warnings: Sequence[Warning], spec: Spec) -> tuple[ """`warnings`, with each `role_scope_defaulted` warning about `spec`, the Spec minted from this composite, worded to count the `parties` of its `params["axis"]`, as a fresh answer's warning states its count (`grounding.role_scope_defaulted_message`). - A warning on an axis whose parties cannot be listed states no count. A warning is - about `spec` where it is keyed to `spec` or to no Spec. One keyed to another reading, - such as a flagged match or a suggestion beside the answer, counts that reading's - parties and keeps its words. An axis on which `spec` has no counted role axis fact - keeps its warning, as does every other warning.""" + The count is the one `spec`'s role axis fact states (`open_axes`), so the warning + and the Spec agree where the snapshot cannot list the parties and the fact keeps the + table's count. A warning is about `spec` where it is keyed to `spec` or to no Spec. + One keyed to another reading, such as a flagged match or a suggestion beside the + answer, counts that reading's parties and keeps its words. An axis on which `spec` + has no counted role axis fact keeps its warning, as does every other warning.""" # Deferred: grounding imports this module while it loads. from search.engine.region.codes import ROLE_SCOPE_DEFAULTED from search.engine.region.grounding import role_scope_defaulted_message counted = { - fact.role - for fact in spec.resolved.role_axis_facts or () + fact.role: fact.member_count + for fact in reversed(spec.resolved.role_axis_facts or ()) if fact.member_count is not None } labels = {entity.ref: entity.label or entity.ref for entity in spec.resolved.entities} @@ -632,7 +633,7 @@ def stated(warning: Warning) -> Warning: opposite: EntityRoleKey = "origin" if role == "destination" else "destination" message = role_scope_defaulted_message( role, - member_count=self.member_count(role), + member_count=counted[role], bound_dcids=list(dict.fromkeys(self.identity.entities.get(opposite, ()))), label_of=lambda dcids: [labels.get(dcid, dcid) for dcid in dcids], ) diff --git a/tests/engine/test_composite_mint.py b/tests/engine/test_composite_mint.py index 345cbca55..2fe8681ad 100644 --- a/tests/engine/test_composite_mint.py +++ b/tests/engine/test_composite_mint.py @@ -8,7 +8,8 @@ from collections.abc import Callable from dataclasses import dataclass -from typing import Any +from types import SimpleNamespace +from typing import Any, cast import pytest @@ -24,8 +25,11 @@ ) from search.engine.region.codes import ROLE_SCOPE_DEFAULTED from search.engine.retrieval.bound_values import served_members +from search.engine.retrieval.retrieve import Materialised +from search.engine.retrieval.role_scope import DiscoveredScope from search.engine.spec_id import identity_spec_id -from search.models import BaseIdentity, Spec, Warning +from search.models import BaseIdentity, CoverageBare, Spec, Warning +from search.scope import composite from series_coverage.schema import RoleNames from tests.engine._fake_coverage import FakeCoverageService, build_coverage_index from tests.engine._harness import offline_resolve @@ -128,14 +132,19 @@ def test_a_minted_dac2a_composite_states_the_composite_s_identity_and_components assert len(mint.components()) == len(mint.served) +@pytest.mark.parametrize("listed", [True, False], ids=["listed", "unlisted"]) @pytest.mark.usefixtures("_dac2a_coverage") def test_the_mint_states_the_defaulted_scope_count_its_role_axis_fact_counts( - monkeypatch: pytest.MonkeyPatch, + listed: bool, monkeypatch: pytest.MonkeyPatch ) -> None: """A `role_scope_defaulted` warning on the composite's open axis, keyed to the minted - Spec or to none, states the count of the parties the mint lists there, found by its code - and axis whatever its text says. A warning on an axis the composite counts nothing on - stands, as does one keyed to another reading beside the answer.""" + Spec or to none, states the count the Spec's role axis fact states there, found by its + code and axis whatever its text says, also where the snapshot cannot list the + composite's parties and the fact keeps the table's count. A warning on an axis the + composite counts nothing on stands, as does one keyed to another reading beside the + answer.""" + if not listed: + monkeypatch.setattr(composite, "_lists_parties", lambda *_args: False) (first, *_rest) = _minted(monkeypatch, _fresh("H2")) minted = first.spec mint = first.kwargs["composite"] @@ -152,6 +161,46 @@ def test_the_mint_states_the_defaulted_scope_count_its_role_axis_fact_counts( stated = mint.role_scope_warnings((on_axis, keyed, off_axis, other, elsewhere), minted) - assert stated[0].message != on_axis.message + assert stated[0].message == grounding.stated_role_scope_message(minted, "destination") assert stated[1].message == stated[0].message assert stated[2:] == (off_axis, other, elsewhere) + + +@pytest.mark.usefixtures("_dac2a_coverage") +def test_a_composite_candidate_listing_no_member_drops_the_defaulted_scope_warning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Grounding a candidate minted as a composite whose Spec lists no member on the open + axis removes the `role_scope_defaulted` warning, as it does for any candidate + (`grounding.restate_role_scope_warnings`).""" + (first, *_rest) = _minted(monkeypatch, _fresh("H2")) + mat = Materialised( + sv_dcids=[], + has_data=True, + coverage=CoverageBare(has_data=True), + discovered_scope=DiscoveredScope( + axis="destination", role_name="recipient", member_dcids=[], filters={} + ), + ) + warning = Warning( + code=ROLE_SCOPE_DEFAULTED, + severity="info", + message="anything", + params={"axis": "destination"}, + ) + state = SimpleNamespace( + opposite_labels=lambda dcids: list(dcids), + mat_result=mat, + place=SimpleNamespace(roles=SimpleNamespace(roles={}, unnamed_wholes=frozenset())), + warnings=[warning], + ) + entry = SimpleNamespace( + candidate=SimpleNamespace(mat=mat), + scope_members=(), + minted=SimpleNamespace(mint=first.kwargs["composite"]), + spec=first.spec, + ) + + candidate_grounding.restate_scope_warning(cast(Any, state), [cast(Any, entry)]) + + assert state.warnings == [] From 7ffa92dae5a9b9116ca340fa19fd748103f812d0 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 20:47:30 +0100 Subject: [PATCH 10/12] build(deps): pin dfo-spec-kit to 67a0fed for the warning-reason registry The registry binding test reads the installed WARNING_REASONS, so it checks Search's bilateral_reading_unavailable reasons against the registry on every run. --- packages/dfo-analysis-engine/pyproject.toml | 4 +-- packages/dfo-data-tools/pyproject.toml | 10 +++---- pyproject.toml | 7 ++--- tests/platform/test_registry_binding.py | 21 ++------------- uv.lock | 30 ++++++++++----------- 5 files changed, 28 insertions(+), 44 deletions(-) diff --git a/packages/dfo-analysis-engine/pyproject.toml b/packages/dfo-analysis-engine/pyproject.toml index c4ac74dc3..6b0822e21 100644 --- a/packages/dfo-analysis-engine/pyproject.toml +++ b/packages/dfo-analysis-engine/pyproject.toml @@ -20,8 +20,8 @@ dependencies = [ # See the root pyproject.toml's dfo-spec-kit source comment for the # pin-consistency invariant these two revs must satisfy: every spec-kit source in the # workspace moves together as one rev. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/spec-kit" } -dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/frame-grammar" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/spec-kit" } +dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/frame-grammar" } [build-system] requires = ["uv_build>=0.11,<0.12"] diff --git a/packages/dfo-data-tools/pyproject.toml b/packages/dfo-data-tools/pyproject.toml index 91e68a43c..fa6fb2f60 100644 --- a/packages/dfo-data-tools/pyproject.toml +++ b/packages/dfo-data-tools/pyproject.toml @@ -94,11 +94,11 @@ graph-release = ["google-cloud-spanner>=3.57", "dfo-data-tools[coverage]"] dfo-analysis-engine = { workspace = true } # See the root pyproject.toml's dfo-spec-kit source comment for the # pin-consistency invariant this rev must satisfy. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/spec-kit" } -dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/frame-grammar" } -dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/dfo-spec-kit-components" } -dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/provenance-docs" } -dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/descriptions" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/spec-kit" } +dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/frame-grammar" } +dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/dfo-spec-kit-components" } +dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/provenance-docs" } +dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/descriptions" } [build-system] requires = ["uv_build>=0.11,<0.12"] diff --git a/pyproject.toml b/pyproject.toml index 8ddb02d96..6ef3cb330 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,7 +254,8 @@ dfo-analysis-engine = { workspace = true } # The registry refuses a part's `source_code` on search.resolve with # `source_code_not_held`, `source_code_ambiguous`, and # `source_code_shares_variable`. -# The pin is spec-kit 9092fec, which registers `bilateral_reading_unavailable` on +# The pin is spec-kit 67a0fed, which lists the `params.why` reasons of +# `bilateral_reading_unavailable` (`WARNING_REASONS`), registers that warning on # search.resolve and the organisation reading codes, # defining the tie between them by the role the query's reading states, # and the Presentation limit names @@ -326,8 +327,8 @@ dfo-analysis-engine = { workspace = true } # role-keys rule names the frame party columns of each role key. # Follow-up: move all nine to a real spec-kit tag once one is cut, so a future # force-push or GC on spec-kit's dev can't silently break this lock. -dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/spec-kit" } -dfo-spec-kit-client-contract = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "9092fec999331a8dc2b58cdf7baaccc20b608175", subdirectory = "python/client-contract" } +dfo-spec-kit = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/spec-kit" } +dfo-spec-kit-client-contract = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f", subdirectory = "python/client-contract" } [tool.pytest.ini_options] testpaths = ["tests", "packages/dfo-analysis-engine/tests"] diff --git a/tests/platform/test_registry_binding.py b/tests/platform/test_registry_binding.py index fd51accb7..4db2ac387 100644 --- a/tests/platform/test_registry_binding.py +++ b/tests/platform/test_registry_binding.py @@ -13,13 +13,11 @@ from __future__ import annotations -import json -import os from collections.abc import Callable from typing import get_args import pytest -from dfo_spec_kit.registry import load_registry +from dfo_spec_kit.registry import WARNING_REASONS, load_registry import datasets_api.codes as datasets_codes import datasets_api.warnings as datasets_warnings @@ -51,7 +49,6 @@ from search.models import NoDataReason from search.scope.provider import WITHHELD_WHYS, Withheld, WithheldWhy from tests.code_registry import registered_endpoints, registered_status_codes -from tests.spec_kit_bundle import spec_kit_dir _REGISTRY = load_registry() @@ -370,25 +367,11 @@ def test_no_served_code_resolves_to_a_retired_registry_entry(): assert not retired, f"served code(s) resolve to a retired registry entry: {retired}" -def _registered_warning_reasons(code: str) -> frozenset[str]: - """The reasons dfo-spec-kit's warning-reason registry lists for `code`, read from the kit - checkout `DFO_SPEC_KIT_DIR` names, else the sibling checkout. A missing registry skips, - and fails when `DFO_SPEC_KIT_REQUIRED` is set.""" - path = spec_kit_dir() / "registries" / "warning-reasons.json" - if not path.is_file(): - if os.environ.get("DFO_SPEC_KIT_REQUIRED"): - raise RuntimeError(f"DFO_SPEC_KIT_REQUIRED is set but {path} does not exist") - pytest.skip("warning-reason registry not found -- set DFO_SPEC_KIT_DIR") - registry = json.loads(path.read_text()) - assert registry["format_version"] == 1 - return frozenset(entry["reason"] for entry in registry["entries"] if entry["code"] == code) - - def test_the_bilateral_reading_warning_states_the_registered_reasons_and_no_other(): """Every `why` Search can put on `bilateral_reading_unavailable` (`WITHHELD_WHYS`) is a reason the registry lists for it, every listed reason is one Search can state, and each has a message, `slot_not_served` alone naming the slot in `params`.""" - assert _registered_warning_reasons(BILATERAL_READING_UNAVAILABLE) == WITHHELD_WHYS + assert frozenset(WARNING_REASONS[BILATERAL_READING_UNAVAILABLE]) == WITHHELD_WHYS for why in get_args(WithheldWhy): slot = "crsTypeOfFinance" if why == "slot_not_served" else None warning = _withheld_warning("spec", Withheld(why=why, slot=slot)) diff --git a/uv.lock b/uv.lock index 82e7d2768..e74a1bdc6 100644 --- a/uv.lock +++ b/uv.lock @@ -726,8 +726,8 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, - { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, + { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, { name = "pyarrow", specifier = ">=24" }, { name = "pydantic", specifier = ">=2.13" }, { name = "rfc8785", specifier = ">=0.1.4" }, @@ -804,11 +804,11 @@ requires-dist = [ { name = "dfo-data-tools", extras = ["generation"], marker = "extra == 'resolve'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'coverage'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'formats'", editable = "packages/dfo-data-tools" }, - { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, - { name = "dfo-spec-kit-components", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, - { name = "dfo-spec-kit-descriptions", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, - { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, - { name = "dfo-spec-kit-provenance-docs", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, + { name = "dfo-spec-kit-components", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, + { name = "dfo-spec-kit-descriptions", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, + { name = "dfo-spec-kit-frame-grammar", extras = ["arrow"], git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, + { name = "dfo-spec-kit-provenance-docs", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, { name = "fastapi", marker = "extra == 'surface'", specifier = ">=0.138" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'generation'", specifier = ">=2.55" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'sdmx'", specifier = ">=2.55" }, @@ -834,7 +834,7 @@ provides-extras = ["sdmx", "formats", "surface", "resolve", "generation", "cover [[package]] name = "dfo-spec-kit" version = "0.2.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f#67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" } dependencies = [ { name = "rfc8785" }, ] @@ -842,12 +842,12 @@ dependencies = [ [[package]] name = "dfo-spec-kit-client-contract" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f#67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" } [[package]] name = "dfo-spec-kit-components" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdfo-spec-kit-components&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f#67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" } dependencies = [ { name = "dfo-spec-kit-descriptions" }, ] @@ -855,7 +855,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-descriptions" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fdescriptions&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f#67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" } dependencies = [ { name = "rfc8785" }, ] @@ -863,7 +863,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-frame-grammar" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fframe-grammar&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f#67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" } dependencies = [ { name = "dfo-spec-kit" }, { name = "pyarrow" }, @@ -873,7 +873,7 @@ dependencies = [ [[package]] name = "dfo-spec-kit-provenance-docs" version = "0.1.0" -source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=9092fec999331a8dc2b58cdf7baaccc20b608175#9092fec999331a8dc2b58cdf7baaccc20b608175" } +source = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fprovenance-docs&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f#67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" } dependencies = [ { name = "markdown-it-py" }, ] @@ -3094,7 +3094,7 @@ requires-dist = [ { name = "dfo-data-tools", extras = ["resolve", "sdmx", "coverage"], marker = "extra == 'graph'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["sdmx"], marker = "extra == 'sdmx'", editable = "packages/dfo-data-tools" }, { name = "dfo-data-tools", extras = ["surface", "resolve", "coverage", "sdmx"], marker = "extra == 'engine'", editable = "packages/dfo-data-tools" }, - { name = "dfo-spec-kit", marker = "extra == 'engine'", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit", marker = "extra == 'engine'", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fspec-kit&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, { name = "fastapi", marker = "extra == 'api'", specifier = ">=0.138" }, { name = "google-auth", marker = "extra == 'datasets'", specifier = ">=2.34" }, { name = "google-auth", extras = ["requests"], marker = "extra == 'eval'", specifier = ">=2.34" }, @@ -3127,7 +3127,7 @@ lint = [ ] test = [ { name = "dfo-analysis-engine", editable = "packages/dfo-analysis-engine" }, - { name = "dfo-spec-kit-client-contract", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=9092fec999331a8dc2b58cdf7baaccc20b608175" }, + { name = "dfo-spec-kit-client-contract", git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git?subdirectory=python%2Fclient-contract&rev=67a0fed31b150ef3c1f3d905d525eb3fd2ddfa6f" }, { name = "frictionless", specifier = ">=5.19.0" }, { name = "pytest", specifier = ">=8" }, { name = "pytest-split", specifier = ">=0.10.0" }, From 45006e09828917f3e8e9e6e12762bfc9b76a7969 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 21:15:33 +0100 Subject: [PATCH 11/12] fix(search): keep every reading truthful when the election and the read disagree A composite elected without its population read answers as the table with bilateral_reading_unavailable scope_undecided, an election that raises answers undecided over the answer identity, and a window mismatch the election window explains answers the table with no degraded note. A flagged refusal that read no rows keeps its own refusal, nearest-real suggestions read and list the population total as answers do, and roles_read keeps every role key it is given. --- src/search/engine/region/grounding.py | 25 +- src/search/engine/region/phase_shared.py | 50 ++-- src/search/engine/region/reading.py | 231 ++++++++++-------- src/search/engine/region/state.py | 6 +- src/search/scope/provider.py | 4 +- .../engine/test_dac2a_suggestion_readings.py | 96 +++++++- tests/engine/test_reading_decision.py | 113 ++++++++- 7 files changed, 388 insertions(+), 137 deletions(-) diff --git a/src/search/engine/region/grounding.py b/src/search/engine/region/grounding.py index 267cd6bb3..775003f1f 100644 --- a/src/search/engine/region/grounding.py +++ b/src/search/engine/region/grounding.py @@ -1237,7 +1237,12 @@ def _relaxed_binding_suggestion( resolved response intact. """ from search.engine.region.place_context import settle_sets, settled_shape, with_where - from search.engine.region.reading import ElectedReads, decide_read_scope + from search.engine.region.reading import ( + ElectedReads, + decide_read_scope, + read_totals, + roles_read, + ) from search.engine.retrieval.retrieve import Materialised from search.engine.retrieval.role_resolve import graph_confirm_resolve @@ -1251,8 +1256,8 @@ def read( relaxed bindings elect reads in (`reading.ElectedReads`), as the answer's read does: a composite reads within its population, binding its totals on the axes the part leaves out (`reading.ReadBinding`), where a standard shape resolves as the standard route - answers it, over the context's entities and places or its default-scope presence - read, and a role-bearing shape confirms against coverage.""" + answers it, over the context's entities with those totals and its places or its + default-scope presence read, and a role-bearing shape confirms against coverage.""" if is_role_bearing(shape): return graph_confirm_resolve( shape=shape, @@ -1270,7 +1275,7 @@ def read( return standard_resolve( shape=shape, bindings=relaxed, - entities=read_at.roles.resolved_entities, + entities=read_binding.entities(shape, read_at), graph=ctx.graph, date_request=ctx.date_request, place_dcids=read_at.probe_places, @@ -1306,7 +1311,8 @@ def read( shape, relaxed, mat.sv_dcids, - roles.roles, + # The suggestion lists the totals its read bound, as the answer does. + roles_read(roles.roles, reading), ctx.graph, mat.read_plan, ctx.query_entity_text_by_dcid, @@ -1362,6 +1368,13 @@ def read( member_facts=mat.member_facts or {}, ) entities_list = [_build_entity(rd, er, etr, role_refs) for rd, er, etr in entity_data] + # Hash the entities listed, but the totals the read bound, which the relaxed + # query did not name. + totals = read_totals(reading) + identity_entities = build_identity_entities( + [entity for entity in entity_data if (entity[1].ref, entity[0]) not in totals], + role_refs, + ) slots = bind_when_slot(slots, window=mat.coverage.window) slots = apply_slot_expansions(slots, slot_expansions(relaxed)) @@ -1453,7 +1466,7 @@ def read( slots, stat_vars_list, entities_list, - identity_entities=None, + identity_entities=identity_entities, open_roles=mat.open_axes, requested_window=requested_window, facet_bindings=None, diff --git a/src/search/engine/region/phase_shared.py b/src/search/engine/region/phase_shared.py index 3b7b58288..c3dc365a5 100644 --- a/src/search/engine/region/phase_shared.py +++ b/src/search/engine/region/phase_shared.py @@ -17,6 +17,7 @@ cause_from_recipe_error, ) from search.engine.bind import SlotBindingDraft +from search.engine.degraded_reads import carry, recording_degraded_reads from search.engine.departures import measure_not_applied, price_basis_not_applied from search.engine.qualifier_match import match_qualifiers from search.engine.region.codes import QUERY_QUALIFIER_DROPPED, ROLE_SCOPE_DEFAULTED @@ -349,11 +350,13 @@ async def _elected_flagged_match( Preserve an established entity mapping on materialise-stage refusals so the where slot, entities, trace, and identity agree. Early refusals have no mapping to retain. - A match read within a composite's population that no scope minting the composite states - has no label to stand under, so it fails closed as the definite answer does - (`reading.ReadScope`): the result returned is no_data `facet_not_available` - (`phase_answer.failed_closed`), which the caller returns in place of its refusal, with - what the match raised dropped.""" + A match whose part read rows within a composite's population (`ResolveState.reading`) + that no scope minting the composite states has no label for those rows to stand under, + so it fails closed as the definite answer does (`reading.ReadScope`): the result + returned is no_data `facet_not_available` (`phase_answer.failed_closed`), which the + caller returns in place of its refusal, with what the match raised dropped. A match + whose part read nothing labels no rows, so where its composite cannot be minted it is + minted as none and the caller's refusal stands (`_flagged_scope`).""" await read_held_families((state.shape_draft,)) await judge_coverage(state) # Everything the match raises from here is its own (`ResolveState.mark_answer`). @@ -427,7 +430,7 @@ async def _elected_flagged_match( # The election and the scope read coverage and scan families, so they run off the # event loop. scoped = await asyncio.to_thread(_flagged_scope, state, answer) - if scoped.decision is None: + if scoped.decision is None and state.reading is not None: # Deferred: the answer phase imports this module. from search.engine.region.phase_answer import failed_closed @@ -481,15 +484,32 @@ def _flagged_scope(state: ResolveState, answer: AnswerIdentity) -> ReadScope: (`reading.decide_read_scope`), under the reading its read elected (`_flagged_reading`). A composite that refused its table read elected an undecided reading's choice (`ReadingDecision.refused`), so the match is the table's Spec and says it reads total - ODA.""" - return decide_read_scope( - answer, - _flagged_reading(state), - selection=state.scope_selection, - operations=state.operations, - coverage=state.coverage, - catalogue=state.request_catalogue, - ) + ODA. + + Where the part read nothing (`ResolveState.reading` is None), the reading is elected for + the match alone and labels no rows. A composite it names that cannot be minted leaves + the scope without a decision, as for rows read within the population, but the match is + then minted as none and its scope's degraded-read note is dropped: no rows went + unlabelled and no answer stands on the lost mint.""" + reading = _flagged_reading(state) + + def decide() -> ReadScope: + return decide_read_scope( + answer, + reading, + selection=state.scope_selection, + operations=state.operations, + coverage=state.coverage, + catalogue=state.request_catalogue, + ) + + if state.reading is not None: + return decide() + with recording_degraded_reads(detached=True) as record: + scoped = decide() + if scoped.decision is not None: + carry(record) + return scoped def _key_reading_warnings( diff --git a/src/search/engine/region/reading.py b/src/search/engine/region/reading.py index a51503ad0..e53fbe1a2 100644 --- a/src/search/engine/region/reading.py +++ b/src/search/engine/region/reading.py @@ -61,8 +61,15 @@ from search.engine.retrieval.bound_values import bound_source from search.engine.retrieval.retrieve import NoDataDraft from search.engine.shape import entities_by_role, identity_shape_for -from search.engine.spec_id import identity_spec_id -from search.models import BaseIdentity, EntityRoleKey, Operation, Spec, TimeWindow, Warning +from search.engine.spec_id import _requested_window_member, identity_spec_id +from search.models import ( + BaseIdentity, + EntityRoleKey, + IdentityWindow, + Operation, + Spec, + Warning, +) from search.scope.provider import Withheld, counted_source from search.scope.state import ScopeState from search.scope.tables import PartFacts, ScopeFacts @@ -151,6 +158,14 @@ class ReadingDecision: # reading's (`scope_decision.undecided_choice`), so a flagged match built over the part, # which reads nothing, is the table's Spec and says it reads total ODA. refused: bool = False + # Whether the election raised (`elect_reading`), so it chose nothing and the part reads + # the table. The answer's scope then judges over the answer's own identity whether the + # bilateral default reaches it (`decide_read_scope`, `scope_decision.undecided_choice`). + undecided: bool = False + # The window of the document the election read (`_pre_read_identity`): the years the + # read reads, which a trailing request's answer narrows (`_window_explains`). None where + # the election refused or raised. + window: IdentityWindow | None = None class ElectionFacts: @@ -205,9 +220,10 @@ def elect_reading( (`ReadingDecision.totals`). An election that raises anywhere, building the document included, is logged as a scope - decision is (`scope_decision.note_undecided`), and the part reads the table as total - ODA, saying so where the bilateral default reaches it (`_undecided_election`). A - composite that refuses its table read is elected as refused, with an undecided reading's + decision is (`scope_decision.note_undecided`) and elected as undecided + (`ReadingDecision.undecided`): the part reads the table as total ODA, saying so where the + bilateral default reaches the answer's identity (`decide_read_scope`). A composite that + refuses its table read is elected as refused, with an undecided reading's choice (`ReadingDecision.refused`), and logged at ERROR and noted as a degraded read, so no cache stores the answer, once per region's resolution (`noting_refusals_once`). @@ -216,7 +232,6 @@ def elect_reading( if not _declares_bilateral_code_system(shape): return None identity: BaseIdentity | None = None - scope_facts: ScopeFacts | None = None try: index = get_index() identity = _pre_read_identity( @@ -224,14 +239,12 @@ def elect_reading( ) if counted_source(identity, index) is None: return None - # The facts of a part serving no dataset yet, which an election that raises past here - # reads the bilateral default's reach in (`_undecided_election`). - scope_facts = facts.facts(index, None) scope_facts = facts.facts(index, _elected_dataset(identity, index)) choice = choose_provider(identity, _snapshot(selection), scope_facts) decision = build_part_scope(identity, choice, spec_id=_ELECTION, facts=scope_facts) + window = identity.requested_window if not decision.composite: - return ReadingDecision(choice=choice, table=None) + return ReadingDecision(choice=choice, table=None, window=window) assert decision.scope is not None composite = decision.scope.compiled.identity try: @@ -255,16 +268,18 @@ def elect_reading( table=None, refused=True, ) - return ReadingDecision(choice=choice, table=table, totals=_totals(identity, table)) + return ReadingDecision( + choice=choice, table=table, totals=_totals(identity, table), window=window + ) except Exception as exc: if identity is None and isinstance(exc, EngineInputError) and exc.code == "request_invalid": # A reversed window, refused building the document (`_read_window`). raise note_undecided(_ELECTION, exc) - return ReadingDecision( - choice=_undecided_election(identity, selection=selection, facts=scope_facts), - table=None, + unchosen = ProviderChoice( + states_scope=False, bilateral=None, withheld=None, selection=selection, provider=None ) + return ReadingDecision(choice=unchosen, table=None, undecided=True) def _totals(identity: BaseIdentity, table: TableRead) -> dict[str, tuple[str, ...]]: @@ -281,21 +296,6 @@ def _totals(identity: BaseIdentity, table: TableRead) -> dict[str, tuple[str, .. } -def _undecided_election( - identity: BaseIdentity | None, *, selection: ScopeState, facts: ScopeFacts | None -) -> ProviderChoice: - """The provider choice of a part whose election raised (`elect_reading`): the table as - total ODA, withheld with `why` `scope_undecided` where the bilateral default reaches - `identity` (`scope_decision.undecided_choice`). Where the election raised before its - document was built, the reach cannot be read, so it withholds nothing, as a reach check - that raises withholds nothing.""" - if identity is None or facts is None: - return ProviderChoice( - states_scope=False, bilateral=None, withheld=None, selection=selection, provider=None - ) - return undecided_choice(identity, selection=selection, facts=facts, subject=_ELECTION) - - def elect_at_read( shape: ShapeDraft, bindings: Sequence[SlotBindingDraft], @@ -383,22 +383,24 @@ def read_totals(reading: ReadingDecision | None) -> frozenset[tuple[str, str]]: def roles_read( roles: Sequence[tuple[str, EntityRoleKey]], reading: ReadingDecision | None ) -> list[tuple[str, EntityRoleKey]]: - """`roles`, a context's wire roles, with the totals the read elected as `reading` binds - (`read_totals`), destinations first, as a resubmit of the composite binds the places of - its table identity (`promote_places._posted_place_roles`): the entities the reading's - Spec lists. `roles` itself where the read binds no total.""" - totals = read_totals(reading) - if not totals: - return list(roles) - by_axis: dict[RoleAxis, list[str]] = {"destination": [], "origin": []} - for dcid, key in [*roles, *sorted(totals)]: + """`roles`, a context's wire roles, each kept in order, with each total the read elected + as `reading` binds (`read_totals`) that `roles` does not hold: the entities the + reading's Spec lists. A destination total goes before the first origin, as a resubmit of + the composite lists the places of its table identity destinations first + (`promote_places._posted_place_roles`), and an origin total goes last. A total binds on + an axis the part leaves out, so this lists destinations first wherever `roles` holds + flow roles alone. Each total binds on a flow axis (`ReadingDecision.totals`).""" + listed = list(roles) + for dcid, key in sorted(read_totals(reading)): axis = role_key_axis(key) - assert axis is not None, f"{key!r} names no flow axis" - if dcid not in by_axis[axis]: - by_axis[axis].append(dcid) - return [(dcid, "destination") for dcid in by_axis["destination"]] + [ - (dcid, "origin") for dcid in by_axis["origin"] - ] + assert axis is not None, f"total {dcid!r} binds {key!r}, no flow axis" + if (dcid, axis) in listed: + continue + at = len(listed) + if axis == "destination": + at = next((i for i, (_dcid, role) in enumerate(listed) if role == "origin"), at) + listed.insert(at, (dcid, axis)) + return listed type WithinRead[T] = Callable[[PlaceContext, FacetBatch | None, ReadBinding], T] @@ -511,7 +513,9 @@ def decide_read_scope( The scope applies the provider choice `reading` elected (`scope_decision.build_part_scope`) and chooses nothing, so the reading that labels the Spec is the one that read its rows. A reading that elected none applies the choice its own identity makes - (`_unelected_choice`). The scope is built over the reading's own identity, which names + (`_unelected_choice`). A reading whose election raised (`ReadingDecision.undecided`) + reads the table as total ODA, saying so where the bilateral default reaches `answer`'s + identity (`scope_decision.undecided_choice`). The scope is built over the reading's own identity, which names what the request named, and checked against the scope decided over it (`_checked`). A scope that compiles a composite mints the Spec as it (`MintedComposite`), with the composite's components listed on its compiled scope and the identity its rows were read @@ -537,11 +541,14 @@ def decide() -> ScopeDecision: operations=operations, coverage=coverage, ) - choice = ( - _unelected_choice(answer.spec_id, identity, selection=selection, facts=facts) - if reading is None - else reading.choice - ) + if reading is None: + choice = _unelected_choice(answer.spec_id, identity, selection=selection, facts=facts) + elif reading.undecided: + choice = undecided_choice( + identity, selection=selection, facts=facts, subject=answer.spec_id + ) + else: + choice = reading.choice decision = build_part_scope(identity, choice, spec_id=answer.spec_id, facts=facts) if reading is not None: decision = _checked(decision, identity, reading, selection=selection, facts=facts) @@ -588,12 +595,13 @@ def decide() -> ScopeDecision: def _unminted(answer: AnswerIdentity, exc: Exception | None) -> ReadScope: - """The scope of the reading `answer` names, read within a composite's population, whose - scope minting that composite raised `exc` or, with `exc` None, compiled no composite: no - decision (`decide_read_scope`).""" + """The scope of the reading `answer` names, elected to read within a composite's + population, whose scope minting that composite raised `exc` or, with `exc` None, + compiled no composite: no decision (`decide_read_scope`). A flagged match whose part read + nothing reaches here too and drops the degraded-read note + (`phase_shared._flagged_scope`), so the log states the election alone.""" logger.error( - "reading %s was read within a composite's population, but %s; it has no label and " - "is not cached", + "reading %s was elected within a composite's population, but %s, so it has no label", answer.spec_id, "its scope compiles no composite" if exc is None else "the composite could not be minted", exc_info=exc, @@ -646,21 +654,25 @@ def _checked( """`decision`, built from the choice `reading` elected before the read, checked against the scope decided over `identity`, the reading's own identity, itself under `selection` (`scope_decision.decide_scope`). The check decides nothing the reading states but one - thing: the reading was elected over the years the read reads (`extract.read_window`), and - where the election withheld the composite because it does not cover every year read - (`option_not_covered`) and the range the answer settled on reads it, its - `bilateral_reading_unavailable` warning names `read_window_not_covered`. A part with a - provider axis states that on its bilateral option. Any other difference, or a check that - raises, is a fault in the election: it is logged at ERROR and noted as a degraded read, - so no cache stores the answer, and the reading keeps `decision`, whose label matches the - rows it read. - - A refused reading (`ReadingDecision.refused`) is not checked: the election logged its - refusal and chose an undecided reading on purpose, so its choice differs from the - identity's by design and is no fault of the election.""" - if reading.refused: + thing: the reading was elected over the years the read reads (`extract.read_window`), so + where the election kept the table and the range the answer settled on reads a composite, + the read window explains it when `identity` over those years chooses as the election + chose (`_window_explains`). The answer then reads the table it read. Where the election + withheld the composite because it does not cover every year read (`option_not_covered`), + its `bilateral_reading_unavailable` warning names `read_window_not_covered`; a part with + a provider axis states the composite's years on its bilateral option. + + Any other difference, or a check that raises, is a fault in the election: it is logged at + ERROR and noted as a degraded read, so no cache stores the answer, and the reading keeps + the label that matches the rows it read (`_as_read`). + + A refused reading (`ReadingDecision.refused`) or an undecided one + (`ReadingDecision.undecided`) is not checked: the election logged its refusal or fault + and chose no reading, so its choice differs from the identity's by design.""" + if reading.refused or reading.undecided: return decision spec_id = decision.spec_id + read_within = reading.table is not None try: choice = choose_provider(identity, selection, facts) # `decision` is `build_part_scope` over this identity, spec_id and facts, so where the @@ -671,38 +683,72 @@ def _checked( if choice == reading.choice else build_part_scope(identity, choice, spec_id=spec_id, facts=facts) ) + elected, settled = _composite_of(decision), _composite_of(checked) + if elected == settled and decision.composite == read_within: + return decision + explained = ( + elected is None + and settled is not None + and not read_within + and _window_explains(identity, reading, selection=selection, facts=facts) + ) except Exception: logger.exception("reading check of %s raised; the answer is not cached", spec_id) note_degraded_read("reading check") - return decision - elected, settled = _composite_of(decision), _composite_of(checked) - read_within = reading.table is not None - if elected == settled and decision.composite == read_within: - return decision - if ( - elected is None - and settled is not None - and not read_within - and reading.choice.withheld == _OPTION_NOT_COVERED - ): + return _as_read(decision, identity, read_within=read_within, selection=selection) + if explained: logger.warning( "answer %s reads total ODA: the bilateral reading %s covers the range it settled " "on, not every year it read", spec_id, settled, ) + if reading.choice.withheld != _OPTION_NOT_COVERED: + return decision unread = dataclasses.replace(reading.choice, withheld=Withheld(why=READ_WINDOW_NOT_COVERED)) return build_part_scope(identity, unread, spec_id=spec_id, facts=facts) logger.error( "answer %s elected %s before its read%s, but its identity decides %s; it keeps the " - "reading it read and is not cached", + "label of the rows it read and is not cached", spec_id, elected or "the table", " within the composite's population" if read_within else "", settled or "the table", ) note_degraded_read(_ELECTION) - return decision + return _as_read(decision, identity, read_within=read_within, selection=selection) + + +def _window_explains( + identity: BaseIdentity, reading: ReadingDecision, *, selection: ScopeState, facts: ScopeFacts +) -> bool: + """Whether the years the read reads explain why `reading` kept the table where + `identity`, the answer's own identity over the narrower range it settled on, reads a + composite: `identity` over the election's window (`ReadingDecision.window`) withholds + what the election withheld and compiles no composite.""" + if reading.window is None: + return False + read = identity.model_copy(update={"requested_window": reading.window}) + choice = choose_provider(read, selection, facts) + if choice.withheld != reading.choice.withheld: + return False + return _composite_of(build_part_scope(read, choice, spec_id=_ELECTION, facts=facts)) is None + + +def _as_read( + decision: ScopeDecision, identity: BaseIdentity, *, read_within: bool, selection: ScopeState +) -> ScopeDecision: + """`decision`, the scope of the reading of `identity`, where its label matches the rows + read. A decision compiling a composite over rows read without the composite's population + (`read_within` false) would state the whole table's rows, total ODA, as the composite's, + so the reading reads as the table it read, saying it reads total ODA where the bilateral + default reaches it (`scope_decision.undecided_choice`). A read within a population that + compiles no composite has no label, which `decide_read_scope` refuses (`_unminted`).""" + if not decision.composite or read_within: + return decision + spec_id, facts = decision.spec_id, decision.facts + choice = undecided_choice(identity, selection=selection, facts=facts, subject=spec_id) + return build_part_scope(identity, choice, spec_id=spec_id, facts=facts) def _composite_of(decision: ScopeDecision) -> str | None: @@ -786,31 +832,14 @@ def _pre_read_identity( return BaseIdentity.model_validate(canonical_identity(index, document)) -def _read_window(date_request: DateRequest | None) -> dict[str, object]: +def _read_window(date_request: DateRequest | None) -> dict: """The identity's `requested_window` member for the years the part's read reads (`extract.read_window`): for a trailing span, every range it can settle on; otherwise the window the answer's identity requests (`grounding.requested_window_from_date_request`, which refuses a reversed window), the latest year, or every year.""" if date_request is not None and date_request.trailing is not None: - window = read_window(date_request) - assert window is not None - return _range(window) - requested = requested_window_from_date_request(date_request) - if requested is None: - return {"kind": "all"} - if requested == "latest": - return {"kind": "latest"} - return _range(requested) - - -def _range(window: TimeWindow) -> dict[str, object]: - """The `range` member of `window`, with the bounds it sets.""" - member: dict[str, object] = {"kind": "range"} - if window.start_year is not None: - member["start_year"] = window.start_year - if window.end_year is not None: - member["end_year"] = window.end_year - return member + return _requested_window_member(read_window(date_request)) + return _requested_window_member(requested_window_from_date_request(date_request)) def _elected_dataset(identity: BaseIdentity, index: CodeIndex) -> str | None: diff --git a/src/search/engine/region/state.py b/src/search/engine/region/state.py index 855c872d7..65848fe48 100644 --- a/src/search/engine/region/state.py +++ b/src/search/engine/region/state.py @@ -943,8 +943,10 @@ class ResolveState: # (`reading.elect_reading`), set beside `mat_result` by # `phase_place_materialise._run_materialise_task`: the choice the definite answer's # scope is built from (`phase_answer._decide_answer_scope`) and the composite's table read - # the read read within. None where the part's identity counts no total with a bilateral - # source, and where a stand-in's read answers in place of the elected shape's. + # the read read within. A stand-in whose read answers in place of the elected shape's + # sets the reading its own read elected + # (`phase_place_materialise._reelect_zero_observation_sibling`). None where the part's + # identity counts no total with a bilateral source, and before the part reads. reading: ReadingDecision | None = None # The composite the definite answer's Spec is minted as (`spec_build.build_spec`), with # the identity its rows were read for and the populations its roles were read within, diff --git a/src/search/scope/provider.py b/src/search/scope/provider.py index f19217464..3dad4895d 100644 --- a/src/search/scope/provider.py +++ b/src/search/scope/provider.py @@ -166,12 +166,12 @@ _CONSTANT = "RealValue" logger = logging.getLogger(__name__) _SLOT_NOT_SERVED = "slot_not_served" +"""The `why` of `provider_not_offered` where a provider composite cannot serve a slot the part +binds, whose property the detail's `slot` names (`_composite_fault`).""" _COMPOSITE_NOT_SERVED = "composite_not_servable" """The `why` of `provider_not_offered` where a provider composite refuses what the part binds at a member that names no slot (`/entities/`, `/facets/provenance`, `/shape`), the member in the detail's `pointer` (`_composite_fault`).""" -"""The `why` of `provider_not_offered` where a provider composite cannot serve a slot the part -binds, whose property the detail's `slot` names (`_composite_fault`).""" WithheldWhy = Literal[ "destination_unclassified", diff --git a/tests/engine/test_dac2a_suggestion_readings.py b/tests/engine/test_dac2a_suggestion_readings.py index e2e341e1b..27f1f467e 100644 --- a/tests/engine/test_dac2a_suggestion_readings.py +++ b/tests/engine/test_dac2a_suggestion_readings.py @@ -16,9 +16,12 @@ import pytest +from search.engine.degraded_reads import recording_degraded_reads from search.engine.errors import EngineInputError +from search.engine.interpretation import Interpretation from search.engine.region import ( RegionResult, + grounding, phase_place_materialise, phase_shared, reading, @@ -244,6 +247,60 @@ def raising(*_args: object, **_kwargs: object) -> object: assert state.warnings == before +@pytest.mark.parametrize( + ("reason", "refuse"), + [ + pytest.param( + "role_incompatible", phase_shared._role_incompatible_no_data, id="role-incompatible" + ), + pytest.param( + "denominator_not_available", + lambda state: phase_place_materialise._denominator_refusal(state, None), + id="denominator", + ), + pytest.param( + "qualifier_not_bound", + phase_place_materialise._declined_refusal, + id="pinned-code", + ), + ], +) +def test_a_refusal_that_read_nothing_keeps_its_own_reason_when_no_composite_can_be_minted( + reason: str, + refuse: Callable[[ResolveState], Any], + fresh_state: Callable[[], ResolveState], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A refusal raised before the part reads (`ResolveState.reading` is None) labels no rows, + so where the composite its flagged match elects cannot be minted, the match is minted as + none and the refusal keeps its own reason and match, noting no degraded read.""" + state = fresh_state() + state.reading = None + state.declined = Interpretation( + shape=state.shape_draft, reason="qualifier_not_bound", pinned_failure="pinned_code" + ) + + async def unsuggested(*_args: object, **_kwargs: object) -> None: + """No nearest-real suggestion, whose own read within the population would note its + lost mint.""" + + monkeypatch.setattr(phase_place_materialise, "_nearest_real_for", unsuggested) + + def raising(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("mint failed") + + monkeypatch.setattr(resubmit, "composite_mint", raising) + + with recording_degraded_reads() as record: + refused = asyncio.run(refuse(state)) + + assert isinstance(refused, RegionResult) + assert (refused.status, refused.no_data_reason) == ("no_data", reason) + (flagged,) = refused.flagged_match or () + assert _provenance(flagged) != _BILATERAL + assert record.reads == [] + + def test_a_resubmit_refusal_s_flagged_match_is_the_composite() -> None: """A composite posted with a destination that holds no row in its population refuses `no_observations` with the composite as its flagged match.""" @@ -375,7 +432,8 @@ def test_a_part_naming_no_destination_reads_the_total_inside_its_read_alone( the where slot of its bindings name no destination, as before the read. A nearest-real suggestion read at that place elects there, reads at the total in its own read and mints an identity binding no destination, as the composite the answer is minted as binds - none.""" + none, while it lists the total among its entities as the answer does + (`reading.roles_read`).""" _patched_coverage(monkeypatch, _AT_TOTAL) before: list[ResolveState] = [] task = phase_place_materialise._materialise_task @@ -417,3 +475,39 @@ def confirmed(**kwargs: Any) -> Any: assert _provenance(suggestion) == _BILATERAL assert "destination" not in suggestion.definition.identity.entities assert subjects == [[_DEVELOPING]] + listed = [(entity.ref, entity.role) for entity in suggestion.resolved.entities] + answered = [(entity.ref.ref, entity.role.axis) for entity in state.entity_objects] + assert (_DEVELOPING, "destination") in listed + assert listed == answered + + +def test_a_standard_relaxation_reads_with_the_totals_its_reading_binds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A relaxation read through the standard route reads the totals the reading it elected + binds on the axes the part leaves out (`reading.ReadBinding.entities`), as the answer's + own read does (`phase_place_materialise._materialise_task`), so a suggestion minted as + the composite reads the rows it is labelled with.""" + _patched_coverage(monkeypatch, _AT_TOTAL) + state = _ended_state(monkeypatch, "D2", sdmx=_sdmx(_AT_TOTAL)) + assert state.reading is not None and state.reading.totals + state.unsettled_place = None + read = state.mat_result + entities: list[dict[str, list[str]]] = [] + + def standard(**kwargs: Any) -> Any: + entities.append(kwargs["entities"]) + return read + + monkeypatch.setattr(grounding, "is_role_bearing", lambda _shape: False) + monkeypatch.setattr(grounding, "standard_resolve", standard) + level = ( + state.shape_draft, + Relaxation(axis="what", property_dcid="measurementDenominator", dropped_value_dcids=["x"]), + ) + asyncio.run(phase_place_materialise._nearest_real_for(state, level=level)) + + assert entities + assert all( + _DEVELOPING in [dcid for dcids in bound.values() for dcid in dcids] for bound in entities + ) diff --git a/tests/engine/test_reading_decision.py b/tests/engine/test_reading_decision.py index 7a5548c04..52dae865b 100644 --- a/tests/engine/test_reading_decision.py +++ b/tests/engine/test_reading_decision.py @@ -65,7 +65,9 @@ _COMPOSITE_AHEAD, _COMPOSITE_BEHIND, _DAC2A, + _DEVELOPING, _ETH, + _FRA, _FRANCE, _HELD, _SCENARIOS, @@ -290,14 +292,20 @@ def _degraded(monkeypatch: pytest.MonkeyPatch, module: Any) -> list[str]: @pytest.mark.parametrize( - ("withheld", "stated", "faulted"), + ("scenario", "withheld", "stated", "faulted"), [ - pytest.param(True, [{"why": READ_WINDOW_NOT_COVERED}], False, id="option-not-covered"), + pytest.param( + "H6", True, [{"why": READ_WINDOW_NOT_COVERED}], False, id="option-not-covered" + ), # Withheld for no reason, the table election is no window gap, so it is a fault. - pytest.param(False, [], True, id="withheld-nothing"), + pytest.param("H6", False, [], True, id="withheld-nothing"), + # A part with a provider axis withholds nothing: its bilateral option states the + # composite's years, so the window gap reads the table with no warning and no fault. + pytest.param("H1", True, [], False, id="provider-axis"), ], ) def test_a_reading_the_read_window_withholds_and_the_settled_range_allows_says_so( + scenario: str, withheld: bool, stated: list[dict[str, str]], faulted: bool, @@ -317,7 +325,10 @@ def test_a_reading_the_read_window_withholds_and_the_settled_range_allows_says_s with caplog.at_level(logging.WARNING, logger=reading.__name__): part = _part( offline_resolve( - _trailing("H6"), graph=_graph_with_parties(), llm=_ScriptedLLM(_VARIABLE), sdmx=sdmx + _trailing(scenario), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=sdmx, ) ) @@ -329,17 +340,25 @@ def test_a_reading_the_read_window_withholds_and_the_settled_range_allows_says_s assert degraded == (["reading election"] if faulted else []) +@pytest.mark.parametrize( + "step", + [ + pytest.param("_elected_dataset", id="electing-the-dataset"), + # Before the document exists, so the bilateral default's reach is read after the read. + pytest.param("_pre_read_identity", id="building-the-document"), + ], +) def test_an_election_that_raises_building_its_document_reads_total_oda_saying_so( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + step: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - """An election that raises before its choice, here electing the dataset its document - names, is logged at ERROR and noted as a degraded read, and the part reads the table as - total ODA saying its scope could not be decided.""" + """An election that raises before its choice, building its document or electing the + dataset that document names, is logged at ERROR and noted as a degraded read, and the + part reads the table as total ODA saying its scope could not be decided.""" - def raising(*_args: object) -> object: + def raising(*_args: object, **_kwargs: object) -> object: raise RuntimeError("index fault") - monkeypatch.setattr(reading, "_elected_dataset", raising) + monkeypatch.setattr(reading, step, raising) degraded = _degraded(monkeypatch, scope_decision) with caplog.at_level(logging.ERROR, logger=scope_decision.logger.name): @@ -361,6 +380,40 @@ def raising(*_args: object) -> object: assert "scope decision" in degraded +def test_a_composite_elected_without_its_read_within_reads_as_the_table( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A reading whose choice names the bilateral composite but whose read was not made within + the composite's population (`ReadingDecision.table` None) read the whole table, so the + answer is the table's Spec, never minted as the composite. It says it reads total ODA, + and the election's fault is logged at ERROR and noted as a degraded read.""" + elect = reading.elect_reading + + def unread(*args: Any, **kwargs: Any) -> ReadingDecision | None: + elected = elect(*args, **kwargs) + assert elected is not None and elected.table is not None + return dataclasses.replace(elected, table=None, totals={}) + + monkeypatch.setattr(reading, "elect_reading", unread) + degraded = _degraded(monkeypatch, reading) + + with caplog.at_level(logging.ERROR, logger=reading.__name__): + part = _part( + offline_resolve( + _SCENARIOS["H1"](), + graph=_graph_with_parties(), + llm=_ScriptedLLM(_VARIABLE), + sdmx=_sdmx(), + ) + ) + + assert _pin(part) is None + assert part.spec.resolved.scope_choices is None + assert [params for _spec_id, params in _withheld(part)] == [{"why": "scope_undecided"}] + assert _faults(caplog) + assert degraded == ["reading election"] + + def _decided_as_the_table(monkeypatch: pytest.MonkeyPatch, *, spec_id: str | None) -> None: """The answer phase's scope from here on compiles no composite and mints none, so the finished answer is the table's Spec. The decision is made for another answer, or, with @@ -978,3 +1031,43 @@ def choosing(identity: Any, selection: Any, facts: ScopeFacts) -> Any: assert len(read) == 4 assert all(facts is read[0] for facts in read) + + +@pytest.mark.parametrize( + ("roles", "listed"), + [ + pytest.param( + [(_FRA, "origin")], + [(_DEVELOPING, "destination"), (_FRA, "origin")], + id="resubmit-order", + ), + pytest.param( + [(_ETH, "observationAbout"), (_FRA, "origin")], + [(_ETH, "observationAbout"), (_DEVELOPING, "destination"), (_FRA, "origin")], + id="subject-first", + ), + pytest.param( + [(_FRA, "origin"), (_ETH, "observationAbout")], + [(_DEVELOPING, "destination"), (_FRA, "origin"), (_ETH, "observationAbout")], + id="subject-last", + ), + pytest.param( + [(_ETH, "observationAbout")], + [(_ETH, "observationAbout"), (_DEVELOPING, "destination")], + id="no-flow-role", + ), + ], +) +def test_the_roles_a_read_lists_keep_every_context_role_and_add_its_totals( + roles: list[tuple[str, Any]], listed: list[tuple[str, str]] +) -> None: + """A read that binds a total on an axis the part leaves out lists the context's roles, + every key in order, with the total before the first origin, as a resubmit of the + composite lists destinations first (`reading.roles_read`).""" + elected = ReadingDecision( + choice=cast(Any, None), + table=cast(Any, SimpleNamespace()), + totals={"destination": (_DEVELOPING,)}, + ) + + assert reading.roles_read(roles, elected) == listed From 6f60eac413cad36a46025670e1fb79b4040424d7 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Sun, 15 Nov 2026 22:13:18 +0100 Subject: [PATCH 12/12] fix(search): warn, note and probe only for the readings a part answers Candidates warn only for readings that ground, a refused composite is noted once per region, a composite candidate probes its slot options as a definite answer does, and the role-scope restatement rewrites only the answer's own warning. A discarded flagged match leaves no degraded-read note, the spec_id kind points only its off-side composite refusal at /input/spec_id, and reading.named_entity_data keeps read totals out of identity entities at every door. --- .../src/materialise/bundle_sources.py | 3 +- src/datasets_api/kinds/spec_id.py | 8 ++- .../engine/region/candidate_grounding.py | 59 +++++++++++++------ src/search/engine/region/candidate_options.py | 4 +- src/search/engine/region/grounding.py | 48 ++++----------- src/search/engine/region/phase_answer.py | 6 +- src/search/engine/region/phase_shared.py | 7 +-- src/search/engine/region/reading.py | 37 ++++++++---- .../test_resolve_definition_members_index.py | 2 + tests/engine/test_candidate_place_roles.py | 1 + .../test_composite_decided_before_preview.py | 49 ++++++++++++++- tests/engine/test_composite_mint.py | 17 +++--- tests/engine/test_composite_resubmit_mint.py | 14 +---- tests/engine/test_dac2a_candidate_readings.py | 41 +++++++++++++ .../engine/test_dac2a_suggestion_readings.py | 29 +++++++++ .../test_grounding_identity_entities.py | 7 ++- tests/engine/test_reading_decision.py | 22 ++++++- 17 files changed, 253 insertions(+), 101 deletions(-) diff --git a/packages/dfo-data-tools/src/materialise/bundle_sources.py b/packages/dfo-data-tools/src/materialise/bundle_sources.py index 69559a4de..3ebc8ae35 100644 --- a/packages/dfo-data-tools/src/materialise/bundle_sources.py +++ b/packages/dfo-data-tools/src/materialise/bundle_sources.py @@ -34,7 +34,8 @@ def composite_bundle_label(index: CodeIndex, composite: CompositeDeclaration) -> """The name a bundle gives `composite`: its label, followed by its `measure_noun` where the label is one of its component tables' labels in `index`, so the composite and the table read apart in one bundle ("OECD DAC2a (bilateral flows)" beside "OECD DAC2a"). - Citations keep the composite's label.""" + The Search provenance facet (`search.observations.composite_facet_ref`) and a bundle's + provenance cells carry this name.""" tables = { declared.label for recipe in composite.components diff --git a/src/datasets_api/kinds/spec_id.py b/src/datasets_api/kinds/spec_id.py index 49c06215d..f47147430 100644 --- a/src/datasets_api/kinds/spec_id.py +++ b/src/datasets_api/kinds/spec_id.py @@ -175,9 +175,15 @@ def _resolve_members( except MemberNotResolvable as exc: exc.issue_pointer = _POINTER raise - except (CompositeUnservable, CompositeWindowUnsupported, FacetNotAvailableError) as exc: + except (CompositeUnservable, CompositeWindowUnsupported) as exc: exc.pointer = _POINTER raise + except FacetNotAvailableError as exc: + # Only the composite's off-side refusal carries issues; `adapter._resolve_index_members`'s + # refusal of a provenance outside the five-tuple's publishers keeps its facet pointer. + if exc.issues is not None: + exc.pointer = _POINTER + raise return resolved.shapes diff --git a/src/search/engine/region/candidate_grounding.py b/src/search/engine/region/candidate_grounding.py index 4b78bf048..a6ab88234 100644 --- a/src/search/engine/region/candidate_grounding.py +++ b/src/search/engine/region/candidate_grounding.py @@ -74,7 +74,7 @@ role_served_filter, ) from search.engine.region.place_roles import request_named_groups -from search.engine.region.reading import decide_read_scope, read_totals, roles_read +from search.engine.region.reading import decide_read_scope, named_entity_data, roles_read from search.engine.region.requirements import answer_terms from search.engine.region.result import FacetAxisFilter, answered_roles from search.engine.region.scope import ( @@ -580,10 +580,8 @@ def _ground_one_candidate( cand_entities = [_build_entity(rd, er, etr, role_refs) for rd, er, etr in cand_entity_data] # Hash the same role-dispatched entities presented in the response, but the totals the # read bound, which the request did not name. - totals = read_totals(cand.reading) cand_identity_entities = build_identity_entities( - [entity for entity in cand_entity_data if (entity[1].ref, entity[0]) not in totals], - role_refs, + named_entity_data(cand_entity_data, cand.reading), role_refs ) cand_slots = bind_when_slot(cand_slots, window=_cand_coverage.window) cand_slots = apply_slot_expansions(cand_slots, slot_expansions(cand_bindings)) @@ -754,6 +752,15 @@ def probe_inputs( _flat_obs_entities(self.probe_entities), ) + @property + def streamed_probe_inputs( + self, + ) -> tuple[ShapeDraft, list[SlotBindingDraft], list[SlotOptions], list[str]] | None: + """The probe inputs (`probe_inputs`) the streamed slot-option events read, None for a + Spec minted as a composite, which streams no option of the table it read, as the + definite answer's does (`phase_answer._finalize_definite_result`).""" + return None if self.minted is not None else self.probe_inputs + class _CarrierFlags(NamedTuple): """What a candidate's own reads left unsettled, which the response warns of once @@ -1036,10 +1043,11 @@ def _finish_candidate( confirmed_facets=tuple(groundwork.confirmed_facets), obs_entities=prep.obs_entities[cand_index][0], probe_entities=groundwork.probe_entities, - # Retain inputs for both buffered and SSE probes. + # Retain inputs for both buffered and SSE probes. A Spec minted as a composite + # streams none (`streamed_probe_inputs`), as the definite answer's does. probe_slot_options=( tuple(groundwork.slot_options_raw) - if minted is None and (state.stream_slot_options or state.include_option_coverage) + if state.stream_slot_options or state.include_option_coverage else None ), role_axis_facts=tuple(cand_role_axis_facts), @@ -1199,19 +1207,23 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro groundworks = _ground_each_candidate(state, candidates, prep) entries: dict[str, _GroundedCandidate] = {} - # The identities of the candidates whose scopes were decided. - decided: set[str] = set() + # Whether each candidate whose scope was decided grounded, by its identity's spec_id. + decided: dict[str, bool] = {} + # Whether each candidate grounded, in candidate order, which decides whose reads warn. + grounded: list[bool] = [] for cand_index, (cand, groundwork) in enumerate(zip(candidates, groundworks, strict=True)): identity, facet_bindings = _candidate_identity(state, cand_index, cand, groundwork, prep) # A candidate whose identity or Spec an earlier one holds adds no Spec or record: its # scope is not decided again, its composite not minted again, and its Spec not # assembled. It follows the candidate it repeats, so it is never the elected one. if identity.spec_id in decided: + grounded.append(decided[identity.spec_id]) continue - decided.add(identity.spec_id) answer = _candidate_answer(state, cand, identity, facet_bindings) if answer.decision is None and _is_elected(state, cand_index): return _GroundedCandidates(entries={}, answer_t0=answer_t0, failed=cand) + decided[identity.spec_id] = answer.decision is not None + grounded.append(answer.decision is not None) if answer.decision is None or answer.spec_id in entries: continue entry = _finish_candidate( @@ -1233,26 +1245,35 @@ def ground_candidates(state: ResolveState, candidates: list[_Candidate]) -> _Gro state.family_total_absent[entry.spec_id] = cand.mat.family_total_absent restate_scope_warning(state, list(entries.values())) - _warn_for_candidates(state, prep, groundworks) + _warn_for_candidates(state, prep, groundworks, grounded) return _GroundedCandidates(entries=entries, answer_t0=answer_t0, failed=None) def _warn_for_candidates( - state: ResolveState, prep: _CandidatePrep, groundworks: Sequence[_CandidateGroundwork] + state: ResolveState, + prep: _CandidatePrep, + groundworks: Sequence[_CandidateGroundwork], + grounded: Sequence[bool], ) -> None: """Add the warnings the candidates' own reads raise, once each, over every candidate - that survives its unit election, a duplicate of a kept Spec included, since its reads - speak for the answer too: each observation warning in candidate order, the first - candidate's entity warnings, and one warning per carrier flag any candidate raises - (`_carrier_flags`).""" + that survives its unit election and grounds (`grounded`, in candidate order), a + duplicate of a kept Spec included, since its reads speak for the answer too: each + observation warning in candidate order, the first grounded candidate's entity warnings, + and one warning per carrier flag any grounded candidate raises (`_carrier_flags`). A + candidate left out because no scope could be decided for it (`_candidate_answer`) + answers nothing, so its reads warn nothing.""" state.warnings.extend( - warning for _entities, warning in prep.obs_entities if warning is not None + warning + for (_entities, warning), kept in zip(prep.obs_entities, grounded, strict=True) + if kept and warning is not None ) - if groundworks: - state.warnings.extend(groundworks[0].entity_warnings) + first = next((gw for gw, kept in zip(groundworks, grounded, strict=True) if kept), None) + if first is not None: + state.warnings.extend(first.entity_warnings) flags = [ _carrier_flags(cand, elected=_is_elected(state, cand_index)) - for cand_index, cand in enumerate(prep.candidates) + for cand_index, (cand, kept) in enumerate(zip(prep.candidates, grounded, strict=True)) + if kept ] state.warnings.extend( carrier_flag_warnings( diff --git a/src/search/engine/region/candidate_options.py b/src/search/engine/region/candidate_options.py index 3ea6a3409..083382254 100644 --- a/src/search/engine/region/candidate_options.py +++ b/src/search/engine/region/candidate_options.py @@ -61,7 +61,7 @@ def _make_slot_options_task( probes: list[_SlotOptionsProbe] = [] for s in surviving: entry = grounded[s.spec_id] - if (inputs := entry.probe_inputs) is None: + if (inputs := entry.streamed_probe_inputs) is None: continue _cand_shape, _cand_bindings, _cand_slot_options, _cand_place_dcids = inputs probes.append( @@ -108,7 +108,7 @@ def _make_slot_options_relaunch( Expose the same optional facet probe override as the single-shape path. """ entry = grounded[original_spec.spec_id] - if not state.stream_slot_options or (inputs := entry.probe_inputs) is None: + if not state.stream_slot_options or (inputs := entry.streamed_probe_inputs) is None: return None _shape, _bindings, _, _place_dcids = inputs return functools.partial( diff --git a/src/search/engine/region/grounding.py b/src/search/engine/region/grounding.py index 775003f1f..8453a742f 100644 --- a/src/search/engine/region/grounding.py +++ b/src/search/engine/region/grounding.py @@ -363,31 +363,6 @@ def role_scope_defaulted_message_for_axis(axis: EntityRoleKey) -> str: return f"{_role_scope_defaulted_base(axis)}." -def stated_role_scope_message(spec: Spec, axis: RoleAxis) -> str | None: - """The `role_scope_defaulted` message `spec` states for its open `axis`, built from its - own facts by the builder a fresh answer's warning is built by - (`role_scope_defaulted_message`): the parties its role axis fact counts there - (`RoleAxisFact.member_count`), and the entities its identity binds on the opposite axis, - in binding order, labelled as `resolved.entities` labels them. None where `spec` counts - no party on `axis`.""" - counted = [ - fact.member_count - for fact in spec.resolved.role_axis_facts or () - if fact.role == axis and fact.member_count is not None - ] - if not counted: - return None - opposite: EntityRoleKey = "origin" if axis == "destination" else "destination" - bound = list(dict.fromkeys(spec.definition.identity.entities.get(opposite, ()))) - labels = {entity.ref: entity.label or entity.ref for entity in spec.resolved.entities} - return role_scope_defaulted_message( - axis, - member_count=counted[0], - bound_dcids=bound, - label_of=lambda dcids: [labels.get(dcid, dcid) for dcid in dcids], - ) - - def memoised_labels( label_of: Callable[[Sequence[str]], Sequence[str]], ) -> Callable[[Sequence[str]], Sequence[str]]: @@ -493,11 +468,12 @@ def restate_role_scope_warnings( `mat` so it counts `members`, the scope the answer lists under the request's facet filter (`scope.stated_scope_members`), or with `state_count` false names every member with data, for a response whose answers list different scopes. The warning is the - `role_scope_defaulted` one on the same axis, found by its code and `params` axis - (`role_scope_defaulted_warning`), whatever count it states, so a response restates it - once its candidates are grounded and again once they are filtered. It keeps its - position, and `label_of`, the memoised lookup that built it (`memoised_labels`), makes - no new request. + `role_scope_defaulted` one on the same axis that no Spec is keyed to, found by its code + and `params` axis (`role_scope_defaulted_warning`), whatever count it states, so a + response restates it once its candidates are grounded and again once they are + filtered. One keyed to a flagged match or a suggestion beside the answer speaks for that + reading and stands. The restated warning keeps its position, and `label_of`, the + memoised lookup that built it (`memoised_labels`), makes no new request. An empty `members` is an answer that lists no member on the axis: it covers none, so the warning is removed. An axis in `unnamed_wholes` built no warning @@ -521,7 +497,11 @@ def restate_role_scope_warnings( restated: Warning | None = fresh[0] if members or not state_count else None axis = mat.discovered_scope.axis for index, old in enumerate(warnings): - if old.code == ROLE_SCOPE_DEFAULTED and (old.params or {}).get("axis") == axis: + if ( + old.code == ROLE_SCOPE_DEFAULTED + and old.spec_id is None + and (old.params or {}).get("axis") == axis + ): if restated is None: del warnings[index] else: @@ -1240,7 +1220,7 @@ def _relaxed_binding_suggestion( from search.engine.region.reading import ( ElectedReads, decide_read_scope, - read_totals, + named_entity_data, roles_read, ) from search.engine.retrieval.retrieve import Materialised @@ -1370,10 +1350,8 @@ def read( entities_list = [_build_entity(rd, er, etr, role_refs) for rd, er, etr in entity_data] # Hash the entities listed, but the totals the read bound, which the relaxed # query did not name. - totals = read_totals(reading) identity_entities = build_identity_entities( - [entity for entity in entity_data if (entity[1].ref, entity[0]) not in totals], - role_refs, + named_entity_data(entity_data, reading), role_refs ) slots = bind_when_slot(slots, window=mat.coverage.window) slots = apply_slot_expansions(slots, slot_expansions(relaxed)) diff --git a/src/search/engine/region/phase_answer.py b/src/search/engine/region/phase_answer.py index d3e813572..eb3a4dbd8 100644 --- a/src/search/engine/region/phase_answer.py +++ b/src/search/engine/region/phase_answer.py @@ -111,7 +111,7 @@ ReadingDecision, ReadScope, decide_read_scope, - read_totals, + named_entity_data, roles_read, ) from search.engine.region.related_read import ( @@ -699,10 +699,8 @@ async def _ground_elected_answer(state: ResolveState) -> RegionResult | None: state.entity_objects = entity_objects # Hash the same role-dispatched entities presented in the response, but the totals the # read bound, which the request did not name. - totals = read_totals(state.reading) identity_entities = build_identity_entities( - [entity for entity in entity_objects_data if (entity[1].ref, entity[0]) not in totals], - role_refs, + named_entity_data(entity_objects_data, state.reading), role_refs ) state.identity_entities = identity_entities diff --git a/src/search/engine/region/phase_shared.py b/src/search/engine/region/phase_shared.py index c3dc365a5..3411cb2d1 100644 --- a/src/search/engine/region/phase_shared.py +++ b/src/search/engine/region/phase_shared.py @@ -491,9 +491,8 @@ def _flagged_scope(state: ResolveState, answer: AnswerIdentity) -> ReadScope: the scope without a decision, as for rows read within the population, but the match is then minted as none and its scope's degraded-read note is dropped: no rows went unlabelled and no answer stands on the lost mint.""" - reading = _flagged_reading(state) - def decide() -> ReadScope: + def decide(reading: ReadingDecision | None) -> ReadScope: return decide_read_scope( answer, reading, @@ -504,9 +503,9 @@ def decide() -> ReadScope: ) if state.reading is not None: - return decide() + return decide(state.reading) with recording_degraded_reads(detached=True) as record: - scoped = decide() + scoped = decide(_flagged_reading(state)) if scoped.decision is not None: carry(record) return scoped diff --git a/src/search/engine/region/reading.py b/src/search/engine/region/reading.py index e53fbe1a2..08841bda1 100644 --- a/src/search/engine/region/reading.py +++ b/src/search/engine/region/reading.py @@ -65,6 +65,7 @@ from search.models import ( BaseIdentity, EntityRoleKey, + GraphRef, IdentityWindow, Operation, Spec, @@ -97,10 +98,14 @@ _BILATERAL_CODE_SYSTEMS = frozenset(row.code_system for row in BILATERAL_SOURCES) -_REFUSALS: ContextVar[set[str] | None] = ContextVar("search_refused_composites", default=None) -"""The spec_ids of the composites whose table read refused in the bound resolution -(`noting_refusals_once`), each logged and noted once. The set is shared by the threads the -resolution starts, so `_refusals_lock` guards it.""" +_REFUSALS: ContextVar[set[tuple[str | None, str | None, str | None]] | None] = ContextVar( + "search_refused_composites", default=None +) +"""The refusals of composites' table reads met in the bound resolution +(`noting_refusals_once`), each the composite's ref with the refusal's code and pointer, so +a refusal is logged and noted once however many places, windows and shapes elect the +composite. The set is shared by the threads the resolution starts, so `_refusals_lock` +guards it.""" _refusals_lock = threading.Lock() @@ -120,16 +125,17 @@ def noting_refusals_once() -> Iterator[None]: _REFUSALS.reset(token) -def _first_refusal(composite: str) -> bool: - """Whether `composite`'s refusal is the first the bound resolution meets, recording it. - True outside a binding (`noting_refusals_once`).""" +def _first_refusal(composite: str | None, refusal: EngineInputError) -> bool: + """Whether `refusal` of composite `composite`'s table read is the first such the bound + resolution meets, recording it. True outside a binding (`noting_refusals_once`).""" refused = _REFUSALS.get() if refused is None: return True with _refusals_lock: - if composite in refused: + key = (composite, refusal.code, refusal.pointer) + if key in refused: return False - refused.add(composite) + refused.add(key) return True @@ -253,7 +259,7 @@ def elect_reading( ) except EngineInputError as exc: composite_id = identity_spec_id(composite) - if _first_refusal(composite_id): + if _first_refusal(_composite_of(decision), exc): logger.error( "reading election of %s: the composite refuses its table read (%s): %s", composite_id, @@ -380,6 +386,17 @@ def read_totals(reading: ReadingDecision | None) -> frozenset[tuple[str, str]]: return frozenset((dcid, key) for key, dcids in reading.totals.items() for dcid in dcids) +def named_entity_data( + entity_data: Sequence[tuple[EntityRoleKey, GraphRef, GraphRef | None]], + reading: ReadingDecision | None, +) -> list[tuple[EntityRoleKey, GraphRef, GraphRef | None]]: + """`entity_data`, a read's grounded entities by role, without the totals the read + elected as `reading` binds (`read_totals`): the entities its identity is built from + (`grounding.build_identity_entities`), which name what the request named.""" + totals = read_totals(reading) + return [entity for entity in entity_data if (entity[1].ref, entity[0]) not in totals] + + def roles_read( roles: Sequence[tuple[str, EntityRoleKey]], reading: ReadingDecision | None ) -> list[tuple[str, EntityRoleKey]]: diff --git a/tests/datasets_api/test_resolve_definition_members_index.py b/tests/datasets_api/test_resolve_definition_members_index.py index 67c2fb797..c78fd723c 100644 --- a/tests/datasets_api/test_resolve_definition_members_index.py +++ b/tests/datasets_api/test_resolve_definition_members_index.py @@ -687,6 +687,8 @@ def test_a_scope_naming_another_datasets_provenance_refuses(): **resolve, ) assert refusal.value.facet_pointer == "/facets/provenance" + # The refusal's own pointer is the facet's, not the `spec_id` input's. + assert refusal.value.pointer is None def _debt_service_identity(**facets: object) -> dict: diff --git a/tests/engine/test_candidate_place_roles.py b/tests/engine/test_candidate_place_roles.py index 9b31cfaf5..b99b17de9 100644 --- a/tests/engine/test_candidate_place_roles.py +++ b/tests/engine/test_candidate_place_roles.py @@ -3523,6 +3523,7 @@ def test_each_option_probe_of_a_lone_candidate_binds_its_own_entities(monkeypatc "spec_ppp": SimpleNamespace( candidate=SimpleNamespace(shape=object(), place=place, date_request=None), probe_inputs=(object(), [], [], [_GHA, _NOR]), + streamed_probe_inputs=(object(), [], [], [_GHA, _NOR]), role_axis_facts=(object(),), sv_dcids=[_GNI_PPP], ) diff --git a/tests/engine/test_composite_decided_before_preview.py b/tests/engine/test_composite_decided_before_preview.py index 9485695a7..d60cfaed9 100644 --- a/tests/engine/test_composite_decided_before_preview.py +++ b/tests/engine/test_composite_decided_before_preview.py @@ -17,9 +17,10 @@ from search.engine import spec_build from search.engine.concurrency import scoped_concurrency from search.engine.errors import GraphInfraError -from search.engine.events import DoneEvent, ResponseEvent +from search.engine.events import DoneEvent, ResponseEvent, SlotOptionsEvent from search.engine.region import ( candidate_collapse, + candidate_options, phase_answer, reading, scope_decision, @@ -489,3 +490,49 @@ async def capture(*args: Any, **kwargs: Any) -> RegionResult: assert result.observations_task is None assert isinstance(result.observations_relaunch, CompositeRelaunch) assert reads == [] + + +@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streamed"]) +@pytest.mark.parametrize("door", ["definite", "candidates"]) +def test_a_composite_answer_probes_its_slot_options_on_every_door(monkeypatch, door, stream): + """An answer minted as a composite with option coverage asked for probes its uncounted + slot menus once (`_maybe_enrich_slot_options`) when buffered, and streams no option + event of the table it read, whether the answer phase mints it (`phase_answer._finalize_definite_result`) or + a candidate set's survivor is minted as it (`candidate_grounding._finish_candidate`). + The set is the one `test_a_candidates_survivor_minted_as_a_composite_reads_no_table_preview` + reads.""" + probes: list[str] = [] + for module in (phase_answer, candidate_options): + enrich = module._maybe_enrich_slot_options + + def recording(*args: Any, _enrich: Any = enrich, **kwargs: Any) -> Any: + probes.append("slot_options") + return _enrich(*args, **kwargs) + + monkeypatch.setattr(module, "_maybe_enrich_slot_options", recording) + coverage = ResolveOptions(include_option_coverage=True) + if door == "definite": + _components(monkeypatch) + request = _fresh(_QUESTION).model_copy(update={"response_options": coverage}) + clients = _clients() + provider = None + else: + patch_india_coverage(monkeypatch) + refuse_the_recipient_reading_s_mint(monkeypatch) + request = india_request(open_role="recipient").model_copy( + update={"response_options": coverage} + ) + clients = india_clients() + provider = _DAC2A_BILATERAL + + events = offline_events(request, stream_slot_options=stream, **clients) + + spec = _answered(events) + assert spec.resolved.scope_choices is not None + assert spec.resolved.scope_choices.compiled.spec_id == spec.spec_id + if provider is not None: + pin = (spec.definition.identity.facets or {}).get("provenance") + assert pin is not None and pin.ref == provider + # A streamed request reads its option coverage from the trailing events alone. + assert probes == ([] if stream else ["slot_options"]) + assert not [event for event in events if isinstance(event, SlotOptionsEvent)] diff --git a/tests/engine/test_composite_mint.py b/tests/engine/test_composite_mint.py index 2fe8681ad..86a92f65e 100644 --- a/tests/engine/test_composite_mint.py +++ b/tests/engine/test_composite_mint.py @@ -138,14 +138,17 @@ def test_the_mint_states_the_defaulted_scope_count_its_role_axis_fact_counts( listed: bool, monkeypatch: pytest.MonkeyPatch ) -> None: """A `role_scope_defaulted` warning on the composite's open axis, keyed to the minted - Spec or to none, states the count the Spec's role axis fact states there, found by its - code and axis whatever its text says, also where the snapshot cannot list the - composite's parties and the fact keeps the table's count. A warning on an axis the - composite counts nothing on stands, as does one keyed to another reading beside the - answer.""" + Spec or to none, states what the fresh answer's response states there, the count the + Spec's role axis fact states, found by its code and axis whatever its text says, also + where the snapshot cannot list the composite's parties and the fact keeps the table's + count. A warning on an axis the composite counts nothing on stands, as does one keyed + to another reading beside the answer.""" if not listed: monkeypatch.setattr(composite, "_lists_parties", lambda *_args: False) - (first, *_rest) = _minted(monkeypatch, _fresh("H2")) + responses: list[Any] = [] + (first, *_rest) = _minted(monkeypatch, lambda: responses.append(_fresh("H2")())) + (part,) = responses[0].parts + (answered,) = [w for w in part.warnings if w.code == ROLE_SCOPE_DEFAULTED] minted = first.spec mint = first.kwargs["composite"] on_axis = Warning( @@ -161,7 +164,7 @@ def test_the_mint_states_the_defaulted_scope_count_its_role_axis_fact_counts( stated = mint.role_scope_warnings((on_axis, keyed, off_axis, other, elsewhere), minted) - assert stated[0].message == grounding.stated_role_scope_message(minted, "destination") + assert stated[0].message == answered.message assert stated[1].message == stated[0].message assert stated[2:] == (off_axis, other, elsewhere) diff --git a/tests/engine/test_composite_resubmit_mint.py b/tests/engine/test_composite_resubmit_mint.py index bb79ba036..b2ed629fb 100644 --- a/tests/engine/test_composite_resubmit_mint.py +++ b/tests/engine/test_composite_resubmit_mint.py @@ -21,7 +21,7 @@ from search.engine import core as search_core from search.engine.entry import dataset_resubmit from search.engine.events import ResponseEvent -from search.engine.region import RegionResult, composite_answer, result +from search.engine.region import RegionResult from search.engine.region.composite_answer import CompositeRelaunch from search.engine.spec_id import identity_spec_id from search.models import ( @@ -153,14 +153,6 @@ def _headline( ) -def _assert_nothing_restates() -> None: - """No function restates a table answer as the composite. The rekeying, the recount and the - restated reading do not exist, so no part or resubmit can run them.""" - assert not hasattr(result, "with_restated_keys") - assert not hasattr(result, "recounted_scope") - assert not hasattr(composite_answer, "restated_reading") - - @pytest.mark.parametrize( "case", [ @@ -180,11 +172,10 @@ def _assert_nothing_restates() -> None: pytest.param(_headline, id="oecd-headline"), ], ) -def test_a_composite_resubmit_answers_as_the_composite_with_no_restatement_layer( +def test_a_composite_resubmit_answers_as_the_composite( case: Callable[[pytest.MonkeyPatch], Callable[[], DefinitePart]], monkeypatch: pytest.MonkeyPatch, ) -> None: - _assert_nothing_restates() answered = case(monkeypatch)() assert answered.spec.definition.identity.facets is not None @@ -249,7 +240,6 @@ def test_a_fresh_answer_equals_a_resubmit_of_its_definition( composite, so it states the scope, headline, coverage, entities and warnings a resubmit of its definition states.""" _patched_coverage(monkeypatch, held) - _assert_nothing_restates() fresh = _resolve_fresh(scenario, held=held) again = _resolve_resubmit(fresh.spec.definition.identity, held=held) diff --git a/tests/engine/test_dac2a_candidate_readings.py b/tests/engine/test_dac2a_candidate_readings.py index 67fbe6025..0788d1568 100644 --- a/tests/engine/test_dac2a_candidate_readings.py +++ b/tests/engine/test_dac2a_candidate_readings.py @@ -30,6 +30,7 @@ reading, resubmit, ) +from search.engine.region.grounding import carrier_flag_warnings from search.engine.region.scope_decision import _withheld_warning from search.engine.region.state import ResolveState from search.engine.spec_id import identity_spec_id @@ -44,6 +45,7 @@ Resolution, ResolveOptions, ResolveRequest, + Warning, ) from search.scope.provider import Withheld from series_coverage.schema import FacetMetadata @@ -367,6 +369,45 @@ async def assembled(*args: Any, **kwargs: Any) -> Any: assert component.identity.entities["destination"] == list(population.members) +def test_a_reading_dropped_unminted_raises_no_warning_of_its_reads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The recipient reading's composite cannot be minted, so it is dropped and the donor + reading answers. Each reading's reads are made to raise an observation warning and + every carrier flag (`candidate_grounding._carrier_flags`); the answer states the donor + reading's observation warning and none of the dropped reading's, and no carrier flag, + since the elected donor reading's flags warned at binding.""" + refuse_the_recipient_reading_s_mint(monkeypatch) + observed = candidate_grounding._candidate_observation_entities + + def warning_of(cand: Any) -> Any: + entities, _warning = observed(cand) + role = "recipient" if cand.place.roles.obs_destination_dcid == _IND else "donor" + return entities, Warning(code=f"observed_{role}", severity="info", message=role) + + monkeypatch.setattr(candidate_grounding, "_candidate_observation_entities", warning_of) + flags = candidate_grounding._CarrierFlags(True, True, True) + monkeypatch.setattr( + candidate_grounding, + "_carrier_flags", + lambda _cand, *, elected: ( + candidate_grounding._CarrierFlags(False, False, False) if elected else flags + ), + ) + + part = _india(india_request(open_role="recipient")) + + assert isinstance(part, DefinitePart), part + codes = [warning.code for warning in part.warnings] + assert "observed_donor" in codes + assert "observed_recipient" not in codes + flagged = carrier_flag_warnings( + role_declaration_unverified=True, open_origin_defaulted=True, membership_unverified=True + ) + stated = {(warning.code, str(warning.params)) for warning in part.warnings} + assert stated.isdisjoint((warning.code, str(warning.params)) for warning in flagged) + + def test_a_survivor_no_scope_states_fails_closed_launching_no_task( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/engine/test_dac2a_suggestion_readings.py b/tests/engine/test_dac2a_suggestion_readings.py index 27f1f467e..a0864096b 100644 --- a/tests/engine/test_dac2a_suggestion_readings.py +++ b/tests/engine/test_dac2a_suggestion_readings.py @@ -223,6 +223,35 @@ def refusing(*_args: object, **_kwargs: object) -> object: assert withheld == [{"why": "scope_undecided"}] +def test_a_discarded_flagged_match_leaves_its_election_s_degraded_read_behind( + fresh_state: Callable[[], ResolveState], monkeypatch: pytest.MonkeyPatch +) -> None: + """A flagged match whose part read nothing and whose composite cannot be minted is + minted as none, so no answer stands on its election: a degraded read the election noted + is dropped with the match, as its scope's own are.""" + state = fresh_state() + state.reading = None + elect = phase_shared.elect_at_read + + def noting(*args: Any, **kwargs: Any) -> Any: + decided = elect(*args, **kwargs) + reading.note_degraded_read("reading election") + return decided + + def raising(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("mint failed") + + monkeypatch.setattr(phase_shared, "elect_at_read", noting) + monkeypatch.setattr(resubmit, "composite_mint", raising) + + with recording_degraded_reads() as record: + match = asyncio.run(phase_shared._elected_flagged_match(state)) + + assert isinstance(match, phase_shared.FlaggedMatch) + assert match.minted is None + assert record.reads == [] + + def test_a_flagged_match_whose_composite_cannot_be_minted_fails_closed( fresh_state: Callable[[], ResolveState], monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/engine/test_grounding_identity_entities.py b/tests/engine/test_grounding_identity_entities.py index 2d85f7954..42a7b38a6 100644 --- a/tests/engine/test_grounding_identity_entities.py +++ b/tests/engine/test_grounding_identity_entities.py @@ -409,7 +409,8 @@ def test_a_scope_warning_naming_every_member_is_restated_to_count_the_answers_sc def test_a_scope_warning_is_found_by_its_code_and_axis_whatever_its_text(): """The warning on the discovered axis is restated whatever its text says, and the companion axis's warning, whose text another reader could have worded like the - discovered axis's, stays as it was.""" + discovered axis's, stays as it was, as does one on the same axis keyed to a flagged + match beside the answer, which speaks for that reading.""" discovered = _discovered_scope("destination", 3) mat = _materialised( discovered_scope=discovered, @@ -421,7 +422,8 @@ def test_a_scope_warning_is_found_by_its_code_and_axis_whatever_its_text(): ) reworded = on_axis.model_copy(update={"message": "Recipients were counted elsewhere."}) lookalike = companion.model_copy(update={"message": on_axis.message}) - warnings = [lookalike, reworded] + flagged = reworded.model_copy(update={"spec_id": "spec_flagged_match"}) + warnings = [flagged, lookalike, reworded] restate_role_scope_warnings( warnings, @@ -432,6 +434,7 @@ def test_a_scope_warning_is_found_by_its_code_and_axis_whatever_its_text(): ) assert [w.message for w in warnings] == [ + flagged.message, on_axis.message, "No recipient was named, so the answer covers the one recipient with data.", ] diff --git a/tests/engine/test_reading_decision.py b/tests/engine/test_reading_decision.py index 52dae865b..8d47adc38 100644 --- a/tests/engine/test_reading_decision.py +++ b/tests/engine/test_reading_decision.py @@ -781,19 +781,35 @@ async def read() -> Any: assert settled.outcome == read_within[-1] -@pytest.mark.parametrize("elections", [1, 2], ids=["read-once", "read-twice"]) +@pytest.mark.parametrize( + ("elections", "identities"), + [ + pytest.param(1, 1, id="read-once"), + pytest.param(2, 1, id="read-twice"), + # The members of a place set, or other windows or shapes, each elect the composite + # under an identity of their own. + pytest.param(2, 2, id="read-twice-under-two-identities"), + ], +) def test_a_composite_that_refuses_its_table_read_answers_facet_not_available( - elections: int, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + elections: int, + identities: int, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: """A composite the part's scope compiles to that refuses its table read is logged at ERROR and noted as a degraded read, so no cache stores the answer, and the part reads nothing and answers no_data, `facet_not_available`. A part whose reads elect the refused - composite twice logs and notes it once (`reading.noting_refusals_once`).""" + composite twice logs and notes it once (`reading.noting_refusals_once`), also where + each read elects it under its own identity.""" def refusing(*_args: object, **_kwargs: object) -> object: raise EngineInputError("refused", code="requested_option_unavailable", status=422) monkeypatch.setattr(reading, "composite_table_identity", refusing) + if identities > 1: + minted = iter(range(elections)) + monkeypatch.setattr(reading, "identity_spec_id", lambda _identity: f"spec_{next(minted)}") elect = reading.elect_reading def electing(*args: Any, **kwargs: Any) -> Any: