From c8d3dcdfb5ffb06198988625479674fea729e4a6 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Fri, 13 Nov 2026 17:34:04 +0100 Subject: [PATCH 01/11] fix: close the 0.1.0 follow-ups and the CodeQL alerts - typescript: trimEnd in core/text.ts trims trailing characters in one scan and replaces every trailing-trim regex, which ran in quadratic time on long runs (CodeQL js/polynomial-redos). Two tests escape their input in full (js/incomplete-sanitization). - python: the transport builds its own httpcore2 pools with the attempt-deadline backend through public names only, reads proxies from urllib, and drops the <2.16 bounds on httpx2 and httpcore2. - Both clients set answer.timed_out when any part's own reason is timed_out, and expose the party totals a rendering left out as Dataset.party_totals_left_out (partyTotalsLeftOut), saved under dfo.party_totals_left_out. - typescript: Answer.facts and partFacts give the checked facts of each part. - The dashboard journeys check the local-stack seed once and stop with one failure that names the missing readings. - CI installs the lowest permitted versions on Python 3.12 and the newest on 3.14, weekly as well, and tests the apache-arrow peer at 21.0.0 and the newest 21.x. Struct columns that differ between members now stack on pyarrow 15 to 20, and float16 columns compare on pyarrow 15 to 25. polars needs 1.3. Closes #11 Closes #24 --- .github/workflows/ci.yml | 109 +++++++ .github/workflows/python-newest.yml | 42 +++ CONTRIBUTING.md | 2 +- python/CHANGELOG.md | 15 + python/docs/datasets.md | 2 + python/pyproject.toml | 11 +- python/src/dfo/_datasets/dataset.py | 214 +++++++++++++- python/src/dfo/_http/proxies.py | 139 +++++++++ python/src/dfo/_http/transport.py | 158 +++++++--- python/src/dfo/_search/answer.py | 10 +- python/src/dfo/_sync/_http/transport.py | 158 +++++++--- python/src/dfo/_sync/_search/answer.py | 10 +- python/src/dfo/types.py | 3 +- python/tests/_async/http/test_transport.py | 85 ++++-- python/tests/_async/search/test_answer.py | 16 +- python/tests/_sync/http/test_transport.py | 85 ++++-- python/tests/_sync/search/test_answer.py | 16 +- python/tests/datasets/test_dataset.py | 271 +++++++++++++++++- python/tests/e2e/test_local_stack.py | 33 ++- python/tests/http/test_environment_proxies.py | 51 ++++ python/tests/test_codegen.py | 2 + python/tests/test_dfo.py | 1 + python/uv.lock | 26 +- scripts/gen_sync.py | 19 +- typescript/CHANGELOG.md | 12 + typescript/README.md | 18 +- typescript/src/client.ts | 3 +- typescript/src/config/credentials.ts | 7 +- typescript/src/core/parse.ts | 4 +- typescript/src/core/text.ts | 16 ++ typescript/src/datasets/cards.ts | 4 +- typescript/src/datasets/dataset.ts | 114 ++++++++ typescript/src/datasets/fetch.ts | 3 +- typescript/src/datasets/pin.ts | 3 +- typescript/src/datasets/publication.ts | 4 +- typescript/src/http/transport.ts | 3 +- typescript/src/index.ts | 11 +- typescript/src/search/answer.ts | 25 +- typescript/src/search/choices.ts | 4 +- typescript/test/answer.test.ts | 22 ++ typescript/test/dataset.test.ts | 108 ++++++- typescript/test/e2e/local-stack.local.ts | 22 ++ typescript/test/fetch.test.ts | 2 +- typescript/test/package.test.ts | 1 + typescript/test/published-types.test.ts | 18 +- typescript/test/reprs.test.ts | 2 +- typescript/test/text.test.ts | 55 ++++ 47 files changed, 1762 insertions(+), 177 deletions(-) create mode 100644 .github/workflows/python-newest.yml create mode 100644 python/src/dfo/_http/proxies.py create mode 100644 python/tests/http/test_environment_proxies.py create mode 100644 typescript/test/text.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 643f5df..3a4490a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: - 'scripts/**' - 'ruff.toml' - '.github/workflows/ci.yml' + - '.github/workflows/python-newest.yml' contract: - 'contract/SOURCE' - 'scripts/**' @@ -125,6 +126,65 @@ jobs: with: files: ./python/coverage.xml + # The extras' floors (pandas, polars) are lower than the dev group's, so the + # project and its extras are installed without the dev group. + python-lowest: + needs: changes + if: needs.changes.outputs.python == 'true' + runs-on: ubuntu-latest + env: + VIRTUAL_ENV: ${{ github.workspace }}/python/.venv-lowest + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.0 + + - name: Cache uv + uses: actions/cache@v6 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-uv-lowest-${{ hashFiles('python/uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv-lowest- + + # Old pyarrow has no wheels for 3.14, so the floors run on the oldest Python. + - name: Set up Python 3.12 + run: uv python install 3.12 + + # One install call, so `lowest-direct` lowers only the package's own direct + # dependencies and its extras. The test tools are pinned to their uv.lock + # versions on the command line, and their other dependencies resolve + # together with the package's, so a shared one such as typing-extensions + # lands on the package's floor. + - name: Install the package and its extras at their lowest versions + run: | + uv venv --python 3.12 "$VIRTUAL_ENV" + uv export --frozen --only-group dev --no-emit-project --no-hashes \ + --output-file "$RUNNER_TEMP/dev-lock.txt" > /dev/null + grep -iE '^(pytest|pytest-cov|pyright|duckdb)==' "$RUNNER_TEMP/dev-lock.txt" \ + | sed 's/ .*//; s/;.*//' > "$RUNNER_TEMP/test-tools.txt" + test "$(wc -l < "$RUNNER_TEMP/test-tools.txt")" -eq 4 + uv pip install --resolution lowest-direct \ + -r "$RUNNER_TEMP/test-tools.txt" -e ".[pandas,polars,xlsx]" + + - name: Show the resolved versions + run: uv pip list + + - name: Fetch the contract at contract/SOURCE + run: uv run --no-project --python 3.12 ../scripts/sync_contract.py + + - name: Run tests + run: uv run --active --no-project pytest --no-cov + + python-newest: + needs: changes + if: needs.changes.outputs.python == 'true' + uses: ./.github/workflows/python-newest.yml + typescript: needs: changes if: needs.changes.outputs.typescript == 'true' @@ -176,6 +236,55 @@ jobs: - name: Build run: pnpm build + # `apache-arrow` is an optional peer dependency, `^21.0.0`. The lockfile pins + # the dev copy, so these two jobs swap it for the oldest and the newest 21.x. + typescript-arrow: + needs: changes + if: needs.changes.outputs.typescript == 'true' + runs-on: ubuntu-latest + strategy: + matrix: + arrow: ["21.0.0", newest] + defaults: + run: + working-directory: typescript + steps: + - uses: actions/checkout@v7 + + - name: Install pnpm + uses: pnpm/action-setup@v6 + with: + package_json_file: typescript/package.json + + - name: Set up Node 22 + uses: actions/setup-node@v7 + with: + node-version: "22" + cache: pnpm + cache-dependency-path: typescript/pnpm-lock.yaml + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.0 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install apache-arrow ${{ matrix.arrow }} + env: + ARROW: ${{ matrix.arrow }} + run: | + if [ "$ARROW" = newest ]; then + ARROW=$(npm view 'apache-arrow@^21' version --json | jq -r 'if type == "array" then .[-1] else . end') + fi + pnpm add --save-dev "apache-arrow@$ARROW" + pnpm ls apache-arrow --depth 0 + + - name: Typecheck + run: pnpm typecheck + + - name: Run tests + run: pnpm test + contract: needs: changes if: needs.changes.outputs.contract == 'true' diff --git a/.github/workflows/python-newest.yml b/.github/workflows/python-newest.yml new file mode 100644 index 0000000..ee0f877 --- /dev/null +++ b/.github/workflows/python-newest.yml @@ -0,0 +1,42 @@ +name: Python newest + +# The package sets no upper bound on its dependencies, so a release of httpx2, +# pyarrow, pandas or polars can break users with no change in this repository. +# This runs the suite against the newest of every dependency, on every pull +# request through ci.yml and once a week on the default branch. +on: + schedule: + - cron: "17 5 * * 1" + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + python-newest: + runs-on: ubuntu-latest + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.0 + + - name: Set up Python 3.14 + run: uv python install 3.14 + + # Without --frozen, the resolution ignores uv.lock and takes the newest of all. + - name: Install the newest versions of every dependency + run: uv sync --upgrade --all-extras --python 3.14 + + - name: Show the resolved versions + run: uv pip list + + - name: Fetch the contract at contract/SOURCE + run: uv run --no-sync python ../scripts/sync_contract.py + + - name: Run tests + run: uv run --no-sync pytest --no-cov diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7484e69..ea07c40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -172,4 +172,4 @@ Publishing uses OIDC trusted publishing from `.github/workflows/release.yml`, so A runtime dependency in `python/pyproject.toml` takes a lower bound and no upper bound, so the package installs beside the newest releases of what it uses. `uv.lock` pins the versions the tests run against. -`httpx2` and `httpcore2` are the one exception, bounded to `>=2.15,<2.16`. `_keep_to_deadlines` in `python/src/dfo/_http/transport.py` reads the connection pool's private layout, which a minor release may move. To take a new minor, run the Python suite against it and move both bounds and the lock in one change. +CI tests both ends of each range. `python-lowest` in `.github/workflows/ci.yml` installs the package, its extras and the test tools in one `uv pip install --resolution lowest-direct` call on Python 3.12, with the test tools pinned to their `uv.lock` versions, so only the package's own direct dependencies go to their floors and a shared dependency such as `typing-extensions` lands on the package's floor. Raise a floor in `pyproject.toml` when this job fails at it. `.github/workflows/python-newest.yml` resolves every dependency to its newest release on Python 3.14 (`uv sync --upgrade`), runs on each pull request that touches Python, and runs weekly on `main`, since an unbounded range lets a new release break users with no change here. `typescript-arrow` runs the TypeScript tests with the optional `apache-arrow` peer at `21.0.0` and at the newest 21.x. The two `hyparquet` packages are exact pins, so they have no range to test. diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index 12fdd98..4d5ca2a 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this package are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `data.party_totals_left_out` names the group totals a fetch left out of each member, as `dfo.types.PartyTotalsLeftOut` entries of spec id, party role, dcids and provenance, and the repr counts them. `data.save` writes them under a new `dfo.party_totals_left_out` key, which 0.1.0 ignores, and `dfo.load` reads a file without it. + +### Changed + +- `development-finance` installs beside any `httpx2` and `httpcore2` from 2.15 on. The transport builds its connection pools through their public interfaces, so a new minor release of either needs no new client release. +- The `polars` extra needs `polars>=1.3`, the first release whose `DataFrame` reads a `Dataset` through the Arrow stream. A CI job installs every floor and runs the suite. + +### Fixed + +- `answer.timed_out` is true when a part's own no-data reason is `timed_out`, as well as when Search's diagnostics say so, and `Answer.from_json` works it out again. + ## [0.1.0] - 2026-11-13 The first release of `development-finance`, the Python client for the Development Finance Observatory APIs. It imports as `dfo` and runs on Python 3.12 or later. diff --git a/python/docs/datasets.md b/python/docs/datasets.md index 66c0c08..bfbe4b8 100644 --- a/python/docs/datasets.md +++ b/python/docs/datasets.md @@ -82,6 +82,8 @@ print(totals.nlargest(3, "value")[["financingProvider_label", "value"]]) The 15 extra rows are group totals worth 18.3 billion dollars, and each one counts donors that the other 95 rows already hold. The cut applies to the party roles a spec leaves open, and a role the spec binds keeps what it binds. `answer.fetch(party_totals=True)` keeps the totals for every reading in an answer. +`data.party_totals_left_out` names the totals a fetch left out. It holds one `dfo.types.PartyTotalsLeftOut` per member, party role and provenance that lost rows, with the role's column name, such as `financingProvider`, and the dcids of the totals, such as `DACCountriesTotal`. The repr counts them on its `Party totals left out` line, and `data.save` keeps them. The tuple is empty after `party_totals=True` and when the cut found no totals to drop. + ## Read the columns Every table leads with the columns a reader looks at, in this order: diff --git a/python/pyproject.toml b/python/pyproject.toml index 9343772..642e229 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,13 +10,8 @@ authors = [ readme = "README.md" requires-python = ">=3.12" dependencies = [ - # `_keep_to_deadlines` in src/dfo/_http/transport.py reads the pool's private - # layout and raises a RuntimeError naming both versions when a release moves it. - # The `<2.16` bounds are the one exception to the no-upper-bound policy in - # CONTRIBUTING.md. A minor release may move that private layout, so each new - # minor is tested before the bound moves. - "httpcore2>=2.15,<2.16; sys_platform != 'emscripten'", - "httpx2>=2.15,<2.16", + "httpcore2>=2.15; sys_platform != 'emscripten'", + "httpx2>=2.15", "pyarrow>=15", "typing-extensions>=4.12", ] @@ -45,7 +40,7 @@ pandas = [ "pandas>=2.2", ] polars = [ - "polars>=1.0", + "polars>=1.3", ] xlsx = [ "openpyxl>=3.1", diff --git a/python/src/dfo/_datasets/dataset.py b/python/src/dfo/_datasets/dataset.py index 557c18e..951608a 100644 --- a/python/src/dfo/_datasets/dataset.py +++ b/python/src/dfo/_datasets/dataset.py @@ -104,6 +104,12 @@ # same value as NaN whatever its payload. _NAN = object() +# The pyarrow major versions from which Arrow itself stacks struct columns whose +# fields differ by name, order or type, and finds the unique 16-bit floats. +# Below them the cells go through Python. +_NATIVE_STRUCT_PROMOTION = 21 +_NATIVE_HALF_FLOAT_UNIQUE = 26 + # The columns every artifact a dataset reads holds, in both layouts. REQUIRED_COLUMNS = ("time_period", "value") @@ -268,6 +274,32 @@ def __repr__(self) -> str: ) +@dataclass(frozen=True, slots=True) +class PartyTotalsLeftOut: + """Group totals on one party role that a fetch left out of one member's rows. + + A fetch leaves party totals out by default, so a sum over `value` counts each + flow once. `party_totals=True` keeps them. DFO names what the cut dropped in + the version's manifest, one entry per party role and provenance that lost + rows, and the Dataset gathers those entries in member order. + + Attributes: + spec_id: The spec of the member whose rows lost the totals. + role: The party role, by DFO's column name, such as `financingProvider`. + entities: The dcids of the totals left out, such as `DACCountriesTotal`. + provenance: The dataset whose rows lost the totals, or None for rows with + no provenance, or when `every_provenance` is true. + every_provenance: Whether the totals left the rows of every dataset, as + on a version DFO recorded before it named the provenance. + """ + + spec_id: str + role: str + entities: tuple[str, ...] + provenance: str | None = None + every_provenance: bool = False + + @dataclass(frozen=True, slots=True) class DerivedStep: """One step of a recipe, as a derived Dataset's repr names it. @@ -340,6 +372,10 @@ class Dataset: dropped for missing data, at any step, each naming its step. Empty for a Dataset of dataset versions. DFO lists at most 2,000; the repr says "at least" when it dropped more. + party_totals_left_out: The group totals a fetch left out of each + member's rows, one entry per member, party role and provenance that + lost rows. Empty when nothing was left out, when the fetch passed + `party_totals=True`, and for a derived Dataset. """ sentence: str @@ -363,6 +399,7 @@ class Dataset: # client's `client` or `dfo`, and `dfo` for a dataset `dfo.load` read. _caller: str = field(default="dfo", repr=False) left_out: tuple[LeftOutSeries, ...] = () + party_totals_left_out: tuple[PartyTotalsLeftOut, ...] = () # Whether `left_out` lists every series the recipe dropped. _left_out_complete: bool = field(default=True, repr=False) # The recipe's steps in the order they ran, for the repr's derived line. @@ -506,6 +543,8 @@ def _summary(self) -> list[str]: if self.sources: lines.append("Source: " + "; ".join(self.sources)) lines.extend(self._derived_summary()) + if totals := _party_totals_line(self.party_totals_left_out): + lines.append(totals) if self._shown_by is not None: lines.append(f"Shown by: {self._shown_by}") versions = [member.content_version for member in self.members] @@ -627,9 +666,28 @@ def _metadata(self) -> dict[bytes, bytes]: } ) metadata["dfo.steps"] = _dumps([asdict(step) for step in self._steps]) + if self.party_totals_left_out: + # A key of its own, since the 0.1.0 loader refuses any key in + # `dfo.dataset` it does not know, and reads only the `dfo.*` keys it names. + metadata["dfo.party_totals_left_out"] = _dumps( + [_party_totals_object(entry) for entry in self.party_totals_left_out] + ) return {key.encode(): value.encode() for key, value in metadata.items()} +def _party_totals_line(entries: Sequence[PartyTotalsLeftOut]) -> str | None: + """The repr's line counting the totals left out on each role, as + `Party totals left out: 15 on financingProvider · data.party_totals_left_out`, + or None when none were.""" + by_role: dict[str, set[str]] = {} + for entry in entries: + by_role.setdefault(entry.role, set()).update(entry.entities) + if not by_role: + return None + counts = ", ".join(f"{len(dcids):,} on {role}" for role, dcids in by_role.items()) + return f"Party totals left out: {counts} · data.party_totals_left_out" + + def _period_line(periods: Sequence[ResolvedPeriod]) -> str | None: """The years a recipe's latest years settled on, and the newest year they count back from, as `Period: 2020 to 2024, counted back from 2024, ...`.""" @@ -754,6 +812,14 @@ def _varies(column: pa.ChunkedArray) -> bool: ) first = next(keys, None) return any(key != first for key in keys) + if ( + pa.types.is_float16(column.type) + and _pyarrow_major() < _NATIVE_HALF_FLOAT_UNIQUE + ): + # Arrow has no unique kernel for 16-bit floats here, and casts them only + # from pyarrow 16, so they go through Python floats, which hold them exactly. + halves = [None if cell is None else float(cell) for cell in column.to_pylist()] + column = pa.chunked_array([pa.array(halves, pa.float64())]) unique = column.unique() if len(unique) < 2 or not pa.types.is_floating(unique.type): return len(unique) > 1 @@ -858,8 +924,12 @@ def build_dataset( layouts: list[str] = [] retentions: list[Mapping[str, Any] | None] = [] warnings: list[Notice] = [] + party_totals: list[PartyTotalsLeftOut] = [] for fetched in members: layouts.append(_layout_version(fetched, layout)) + party_totals.extend( + served_party_totals(fetched.manifest, spec_id=fetched.spec.spec_id) + ) built.append( Member( parts=fetched.parts, @@ -894,8 +964,13 @@ def build_dataset( # widen to the others' types. tables = [table for table in tables if table.num_rows] or tables[:1] try: + if _pyarrow_major() < _NATIVE_STRUCT_PROMOTION: + tables = _with_unified_structs(tables) table = pa.concat_tables(tables, promote_options="permissive") - except (pa.ArrowTypeError, pa.ArrowInvalid) as error: + # ValueError covers pyarrow's ArrowInvalid and the refusal of a nanosecond + # cell as a Python value without pandas. OverflowError covers a cell too + # large for the unified type. + except (pa.ArrowTypeError, ValueError, OverflowError) as error: raise client_error( CONTRACT_VIOLATION, f"The members' tables cannot stack into one: {error}", @@ -926,6 +1001,7 @@ def build_dataset( _table=table.replace_schema_metadata(None), _member_rows=member_rows, _caller=caller, + party_totals_left_out=tuple(party_totals), ) @@ -1032,6 +1108,94 @@ def _served_left_out( return series, complete +def served_party_totals( + manifest: Mapping[str, Any], *, spec_id: str +) -> tuple[PartyTotalsLeftOut, ...]: + """The totals a manifest's `trace.party_totals_left_out` names for the + member of `spec_id`, or none when the trace names none. A key an entry holds + beyond `role`, `entities` and `provenance` is ignored. + + Raises: + ContractError: The trace or an entry breaks the served shape + (`contract_violation`). + """ + trace = manifest.get("trace") + if trace is None: + return () + pointer = "/trace/party_totals_left_out" + try: + served = object_at(trace, "/trace").get("party_totals_left_out") + if served is None: + return () + return tuple( + read_party_totals(item, f"{pointer}/{index}", spec_id=spec_id) + for index, item in enumerate(_array(served, pointer)) + ) + except FormatError as error: + violation = client_error( + CONTRACT_VIOLATION, + f"The manifest's {error.pointer} breaks the shape this client reads", + ) + violation.pointer = error.pointer + raise violation from error + + +def read_party_totals( + value: object, pointer: str, *, spec_id: str +) -> PartyTotalsLeftOut: + """One entry of `party_totals_left_out` as the manifest's trace serves it + and a saved file keeps it: `{role, entities, provenance}`, where an absent + `provenance` means every dataset's rows. + + Raises: + FormatError: A key of that shape is missing or of the wrong type. + `pointer` locates it. + """ + obj = object_at(value, pointer) + _require(obj, pointer, ("role", "entities")) + entities = tuple( + _string(item, f"{pointer}/entities/{index}") + for index, item in enumerate(_array(obj["entities"], f"{pointer}/entities")) + ) + provenance = obj.get("provenance") + if provenance is not None: + _string(provenance, f"{pointer}/provenance") + return PartyTotalsLeftOut( + spec_id=spec_id, + role=_string(obj["role"], f"{pointer}/role"), + entities=entities, + provenance=provenance, + every_provenance="provenance" not in obj, + ) + + +def _party_totals_object(entry: PartyTotalsLeftOut) -> dict[str, Any]: + """An entry as a saved file keeps it: the served shape with its spec id.""" + obj: dict[str, Any] = { + "spec_id": entry.spec_id, + "role": entry.role, + "entities": list(entry.entities), + } + if not entry.every_provenance: + obj["provenance"] = entry.provenance + return obj + + +def _load_party_totals(text: str) -> tuple[PartyTotalsLeftOut, ...]: + """A saved file's `dfo.party_totals_left_out`.""" + pointer = "/dfo.party_totals_left_out" + loaded: list[PartyTotalsLeftOut] = [] + for index, item in enumerate(_array(decode_value(text, pointer), pointer)): + at = f"{pointer}/{index}" + obj = object_at(item, at) + check_keys( + obj, at, required=("spec_id", "role", "entities"), allowed=("provenance",) + ) + spec_id = _string(obj["spec_id"], f"{at}/spec_id") + loaded.append(read_party_totals(obj, at, spec_id=spec_id)) + return tuple(loaded) + + def _manifest_violation(pointer: str) -> ContractError: error = client_error( CONTRACT_VIOLATION, @@ -1165,7 +1329,7 @@ def _nesting_fault(data_type: pa.DataType) -> str | None: if pa.types.is_list(data_type) or pa.types.is_large_list(data_type): return _nesting_fault(data_type.value_type) if pa.types.is_struct(data_type): - fields = data_type.fields + fields = [data_type.field(index) for index in range(data_type.num_fields)] if len({field.name for field in fields}) != len(fields): return "of structs that name a field twice" return next( @@ -1256,6 +1420,11 @@ def saved_dataset(table: pa.Table, metadata: Mapping[bytes, bytes]) -> Dataset: layout_version=_string(rest["layout_version"], "/dfo.dataset/layout_version"), _table=table.replace_schema_metadata(None), _member_rows=member_rows, + party_totals_left_out=( + _load_party_totals(text) + if (text := values.get("dfo.party_totals_left_out")) is not None + else () + ), **(_load_derived(values) if derived else {}), ) @@ -1532,6 +1701,47 @@ def _layout_version(fetched: FetchedMember, layout: Layout) -> str: ) +def _pyarrow_major() -> int: + """The major version of the Arrow C++ library pyarrow runs on, which is + pyarrow's own.""" + import pyarrow as pa # noqa: PLC0415 - pyarrow loads on first use + + return pa.cpp_version_info.major + + +def _with_unified_structs(tables: list[pa.Table]) -> list[pa.Table]: + """`tables` with each column that holds structs and differs from the unified + type rebuilt as that type, structs unifying by field name in the order they are + first named. Before pyarrow `_NATIVE_STRUCT_PROMOTION` Arrow casts no struct to + one with more or reordered fields, so the cells go through Python, and the + rebuilt column drops field nullability and metadata.""" + import pyarrow as pa # noqa: PLC0415 - pyarrow loads on first use + + schema = pa.unify_schemas( + [table.schema for table in tables], promote_options="permissive" + ) + result = [] + for table in tables: + unified = table + for index, column_field in enumerate(table.schema): + target = schema.field(column_field.name).type + if _holds_struct(target) and not column_field.type.equals(target): + cells = unified.column(index).to_pylist() + column = pa.array(cells, type=target) + unified = unified.set_column(index, column_field.name, column) + result.append(unified) + return result + + +def _holds_struct(data_type: pa.DataType) -> bool: + """Whether `data_type` is a struct, or a list of them at any depth.""" + import pyarrow as pa # noqa: PLC0415 - pyarrow loads on first use + + if pa.types.is_list(data_type) or pa.types.is_large_list(data_type): + return _holds_struct(data_type.value_type) + return pa.types.is_struct(data_type) + + def _with_part(table: pa.Table, label: str) -> pa.Table: """`table` with the `part` column first, holding `label` on every row. diff --git a/python/src/dfo/_http/proxies.py b/python/src/dfo/_http/proxies.py new file mode 100644 index 0000000..f0565ef --- /dev/null +++ b/python/src/dfo/_http/proxies.py @@ -0,0 +1,139 @@ +"""The parts of the HTTP transport that name no async or sync flavour. + +`dfo._http.transport` is the async source that `scripts/gen_sync.py` copies into +its sync twin, and both import this module, so none of it is copied. It holds the +pool limits, the mapping from httpcore2's errors to httpx2's, the conversion of +an httpx2 URL to an httpcore2 one, and the reading of the proxy environment +into httpx2 mount patterns. +""" + +import ipaddress +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from urllib.request import getproxies + +import httpcore2 +import httpx2 + +# The pool limits httpx2 gives its own transport: 100 connections, 20 kept +# alive for 5 s each. +MAX_CONNECTIONS = 100 +MAX_KEEPALIVE_CONNECTIONS = 20 +KEEPALIVE_EXPIRY_S = 5.0 + +# Each httpcore2 error and the httpx2 error it raises as, the more specific +# class before the one it derives from. +HTTPCORE_ERRORS: tuple[tuple[type[Exception], type[httpx2.HTTPError]], ...] = ( + (httpcore2.ConnectTimeout, httpx2.ConnectTimeout), + (httpcore2.ReadTimeout, httpx2.ReadTimeout), + (httpcore2.WriteTimeout, httpx2.WriteTimeout), + (httpcore2.PoolTimeout, httpx2.PoolTimeout), + (httpcore2.TimeoutException, httpx2.TimeoutException), + (httpcore2.ConnectError, httpx2.ConnectError), + (httpcore2.ReadError, httpx2.ReadError), + (httpcore2.WriteError, httpx2.WriteError), + (httpcore2.NetworkError, httpx2.NetworkError), + (httpcore2.ProxyError, httpx2.ProxyError), + (httpcore2.UnsupportedProtocol, httpx2.UnsupportedProtocol), + (httpcore2.LocalProtocolError, httpx2.LocalProtocolError), + (httpcore2.RemoteProtocolError, httpx2.RemoteProtocolError), + (httpcore2.ProtocolError, httpx2.ProtocolError), +) + + +@contextmanager +def httpx_errors() -> Generator[None]: + """Raise an httpcore2 error from inside the block as its httpx2 class. + + Only `Exception` is caught, so cancellation passes through unchanged. + """ + try: + yield + except Exception as exc: + for source, target in HTTPCORE_ERRORS: + if isinstance(exc, source): + raise target(str(exc)) from exc + raise + + +def core_url(url: httpx2.URL) -> httpcore2.URL: + return httpcore2.URL( + scheme=url.raw_scheme, host=url.raw_host, port=url.port, target=url.raw_path + ) + + +def no_proxy_pattern(host: str) -> str | None: + """The httpx2 mount pattern that exempts the `NO_PROXY` entry `host` from the + proxies, or None for an entry with no pattern, following httpx2's `NO_PROXY` + mapping. + + `.example.org` maps to `all://*.example.org`, which matches the names under + example.org but not example.org itself, and `example.org` to + `all://*example.org`. An IPv4 or IPv6 address or network maps to itself and + a URL pattern to itself. An entry that maps to no pattern, such as + `example.org/path`, is skipped. An entry that maps to a pattern httpx2 + cannot read, such as `10.0.0.0/99`, is dropped by `environment_proxies`. + """ + address, slash, prefix = host.partition("/") + try: + version = ipaddress.ip_address(address).version + except ValueError: + version = None + if "://" in host: + return host + if version == 4 or host.lower() == "localhost" or host.startswith("["): + return f"all://{host}" + if version == 6: + return f"all://[{address}]{slash}{prefix}" + return f"all://*{host}" if host and "/" not in host else None + + +def _mountable(patterns: list[str]) -> list[str]: + """The patterns httpx2 takes as mount patterns. + + One client built over all of them answers in the usual case. When it + refuses, the patterns are tried one by one to find the unreadable ones. + """ + + def accepted(group: list[str]) -> bool: + try: + # A transport passed in keeps the client from reading the environment. + httpx2.AsyncClient( + transport=httpx2.AsyncBaseTransport(), + mounts=dict.fromkeys(group), + ) + except (httpx2.InvalidURL, ValueError): + return False + return True + + if accepted(patterns): + return patterns + return [pattern for pattern in patterns if accepted([pattern])] + + +def environment_proxies( + proxies: Mapping[str, str] | None = None, +) -> dict[str, str | None]: + """The proxy URL for each httpx2 mount pattern, None for a host `NO_PROXY` + exempts. + + The proxies come from `proxies`, by default `urllib.request.getproxies()`: + the `*_PROXY` variables, or on macOS and Windows the system settings. The + http, https and all proxies mount on their scheme, with `http://` before a + URL that names no scheme. `NO_PROXY=*` turns every proxy off, and each + other entry mounts as `no_proxy_pattern` says. An entry httpx2 cannot read + as a pattern is skipped, as curl and Go skip it. + """ + found = getproxies() if proxies is None else proxies + mounts: dict[str, str | None] = {} + for scheme in ("http", "https", "all"): + if url := found.get(scheme): + mounts[f"{scheme}://"] = url if "://" in url else f"http://{url}" + for entry in found.get("no", "").split(","): + host = entry.strip() + if host == "*": + return {} + if (pattern := no_proxy_pattern(host)) is not None: + mounts[pattern] = None + readable = set(_mountable(list(mounts))) + return {pattern: url for pattern, url in mounts.items() if pattern in readable} diff --git a/python/src/dfo/_http/transport.py b/python/src/dfo/_http/transport.py index 8880aaa..835baa8 100644 --- a/python/src/dfo/_http/transport.py +++ b/python/src/dfo/_http/transport.py @@ -52,7 +52,7 @@ import platform import re import ssl -from collections.abc import Generator, Iterable, Mapping +from collections.abc import AsyncIterable, AsyncIterator, Generator, Iterable, Mapping from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass @@ -63,7 +63,6 @@ import httpcore2 import httpx2 -from httpx2._utils import get_environment_proxies from dfo._config.keys import ApiKeyArgument, KeyState, ResolvedKey from dfo._core.codes import ( @@ -111,6 +110,14 @@ event_loop, monotonic, ) +from dfo._http.proxies import ( + KEEPALIVE_EXPIRY_S, + MAX_CONNECTIONS, + MAX_KEEPALIVE_CONNECTIONS, + core_url, + environment_proxies, + httpx_errors, +) from dfo._version import __version__ DEFAULT_BASE_URL = "https://developmentfinance.org/api/v1" @@ -622,42 +629,117 @@ async def sleep(self, seconds: float) -> None: await self._backend.sleep(seconds) -def _keep_to_deadlines(client: httpx2.AsyncClient) -> None: - """Route the connections of every pool `client` holds through `_DeadlineBackend`. +class _ResponseStream(httpx2.AsyncByteStream): + """A response body from httpcore2, read with httpx2's errors. + + Closing it returns the connection to the pool. + """ + + def __init__(self, stream: AsyncIterable[bytes]) -> None: + self._stream = stream + + async def __aiter__(self) -> AsyncIterator[bytes]: + with httpx_errors(): + # The generated sync loop stays a loop, as `yield from` has no async form. + async for part in self._stream: # noqa: RUF100, UP028 + yield part + + async def aclose(self) -> None: + close = getattr(self._stream, "aclose", None) + if close is not None: + with httpx_errors(): + await close() + - httpx2 builds the pools, its proxy pools from the environment included, and - takes no network backend, so the backend goes in after it builds them. A - transport the caller passed in that is not httpx2's own keeps its timing. +def _connection_pool(proxy_url: str | None) -> httpcore2.AsyncConnectionPool: + """An httpcore2 pool, direct or through `proxy_url`, whose connections keep to + the attempt's deadline through `_DeadlineBackend`. - The pools sit in private attributes of httpx2 and httpcore2, and the stall tests - run against the locked releases, 2.15 for both. + TLS verifies against the system's trust store, or `SSL_CERT_FILE` and + `SSL_CERT_DIR` when set, as httpx2's own transport does. A socks5 or socks5h + proxy needs `socksio` installed. Raises: - RuntimeError: When httpx2 or httpcore2 holds its pools differently. + ValueError: `proxy_url`'s scheme is not http, https, socks5 or socks5h. """ - try: - transports = (client._transport, *client._mounts.values()) - except AttributeError as error: - raise _pool_layout_error() from error - for transport in transports: - if not isinstance(transport, httpx2.AsyncHTTPTransport): - continue - pool = getattr(transport, "_pool", None) - if not isinstance(pool, httpcore2.AsyncConnectionPool) or not hasattr( - pool, "_network_backend" - ): - raise _pool_layout_error() - pool._network_backend = _DeadlineBackend(pool._network_backend) - - -def _pool_layout_error() -> RuntimeError: - return RuntimeError( - f"httpx2 {httpx2.__version__} with httpcore2 {httpcore2.__version__} holds " - "its connection pools where dfo cannot route them through its deadlines; " - "install httpx2 and httpcore2 2.15, the series dfo is tested on" + ssl_context = httpx2.create_ssl_context(verify=True, trust_env=True) + # httpcore2 declares a stand-in class when anyio or socksio is missing, so + # the isinstance checks narrow its types. + network = httpcore2.AnyIOBackend() + assert isinstance(network, httpcore2.AsyncNetworkBackend) + backend = _DeadlineBackend(network) + limits: dict[str, Any] = { + "max_connections": MAX_CONNECTIONS, + "max_keepalive_connections": MAX_KEEPALIVE_CONNECTIONS, + "keepalive_expiry": KEEPALIVE_EXPIRY_S, + } + if proxy_url is None: + return httpcore2.AsyncConnectionPool( + ssl_context=ssl_context, network_backend=backend, **limits + ) + proxy = httpx2.Proxy(proxy_url) + if proxy.url.scheme in ("http", "https"): + return httpcore2.AsyncHTTPProxy( + proxy_url=core_url(proxy.url), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + network_backend=backend, + **limits, + ) + if proxy.url.scheme in ("socks5", "socks5h"): + pool = httpcore2.AsyncSOCKSProxy( + proxy_url=core_url(proxy.url), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + network_backend=backend, + **limits, + ) + assert isinstance(pool, httpcore2.AsyncConnectionPool) + return pool + raise ValueError( + f"A proxy URL's scheme is http, https, socks5 or socks5h, not " + f"{proxy.url.scheme!r}" ) +class _DeadlineTransport(httpx2.AsyncBaseTransport): + """An httpx2 transport over an httpcore2 pool whose every network operation + keeps to the attempt's deadline. + + It sends as httpx2's own transport does: HTTP/1.1, no connection retries, + the request's extensions (its timeouts among them) passed to httpcore2 and + the response's passed back, and httpcore2's errors raised as httpx2's. + """ + + def __init__(self, proxy_url: str | None = None) -> None: + self._pool = _connection_pool(proxy_url) + + async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response: + assert isinstance(request.stream, httpx2.AsyncByteStream) + core_request = httpcore2.Request( + method=request.method, + url=core_url(request.url), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with httpx_errors(): + response = await self._pool.handle_async_request(core_request) + assert isinstance(response.stream, AsyncIterable) + return httpx2.Response( + status_code=response.status, + headers=response.headers, + stream=_ResponseStream(response.stream), + extensions=response.extensions, + ) + + async def aclose(self) -> None: + with httpx_errors(): + await self._pool.aclose() + + async def _check_location(response: httpx2.Response) -> None: """Refuse a redirect whose `Location` httpx2 cannot read as a URL, such as one with a malformed IPv6 host or port, before httpx2 reads it as a protocol fault. @@ -700,7 +782,7 @@ class _Pool: def __init__(self, http_transport: httpx2.AsyncBaseTransport | None) -> None: self._http_transport = http_transport - self._proxies = {} if http_transport is not None else get_environment_proxies() + self._proxies = {} if http_transport is not None else environment_proxies() self._pid = os.getpid() self._loop: object | None = None self._closed = False @@ -709,17 +791,15 @@ def __init__(self, http_transport: httpx2.AsyncBaseTransport | None) -> None: def _build(self) -> httpx2.AsyncClient: # An explicit transport keeps httpx2 from looking the proxies up again. transport = self._http_transport - client = httpx2.AsyncClient( - transport=httpx2.AsyncHTTPTransport() if transport is None else transport, + return httpx2.AsyncClient( + transport=_DeadlineTransport() if transport is None else transport, mounts={ - pattern: None if url is None else httpx2.AsyncHTTPTransport(proxy=url) + pattern: None if url is None else _DeadlineTransport(url) for pattern, url in self._proxies.items() }, follow_redirects=False, event_hooks={"response": [_check_location]}, ) - _keep_to_deadlines(client) - return client @property def client(self) -> httpx2.AsyncClient: @@ -791,9 +871,9 @@ def __init__( headers: Headers sent with every API request. header_provider: Called before every API request for more headers, such as an identity token. - http_transport: The httpx2 transport, for tests and proxies. An - `httpx2.AsyncHTTPTransport` keeps to the attempt's deadline at - every network operation; any other transport times itself. + http_transport: The httpx2 transport, for tests. A transport + passed in times itself, and dfo's own keeps to the attempt's + deadline at every network operation. sleep: Waits between attempts. clock: A monotonic clock, in seconds, for deadlines. caller: What a hint names the owner of a call it suggests: `client` diff --git a/python/src/dfo/_search/answer.py b/python/src/dfo/_search/answer.py index 2a21502..9643256 100644 --- a/python/src/dfo/_search/answer.py +++ b/python/src/dfo/_search/answer.py @@ -118,7 +118,8 @@ class AsyncAnswer: question: The question asked, when the answer came from one. reason: Why Search declined, for a declined answer. warnings: Search's warnings for the whole answer. - timed_out: Whether Search stopped at its time limit. + timed_out: Whether Search stopped at its time limit, by its diagnostics + or by a part's own no-data reason `timed_out`. trace_id: The Search response's trace id, to quote when reporting it. raw: The decoded response body, untouched. """ @@ -236,7 +237,7 @@ async def choose( ContractError: The chosen option's `data_query` is not a parsed input of one part; it carries the answer's trace id. """ - # A caller without a type checker can still pass the earlier form. + # A caller without a type checker can pass the `choose(part, option)` form. if old_form: # ty: ignore[redundant-condition] raise TypeError( "answer.choose takes the option first and the part as a keyword: " @@ -499,7 +500,10 @@ def answer_from_body( if isinstance(sentence, str) and sentence else " ".join(part.sentence for part in parts if part.sentence) ) - timed_out = diagnostics.get("timed_out") is True + # Search can stop at its time limit on one part without setting the flag. + timed_out = diagnostics.get("timed_out") is True or any( + isinstance(part, NoDataPart) and part.reason == "timed_out" for part in parts + ) gaps = tuple( gap for index, part in enumerate(parts) diff --git a/python/src/dfo/_sync/_http/transport.py b/python/src/dfo/_sync/_http/transport.py index ac5ee9d..3c6d9fa 100644 --- a/python/src/dfo/_sync/_http/transport.py +++ b/python/src/dfo/_sync/_http/transport.py @@ -53,7 +53,7 @@ import platform import re import ssl -from collections.abc import Generator, Iterable, Mapping +from collections.abc import Generator, Iterable, Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass @@ -64,7 +64,6 @@ import httpcore2 import httpx2 -from httpx2._utils import get_environment_proxies from dfo._config.keys import ApiKeyArgument, KeyState, ResolvedKey from dfo._core.codes import ( @@ -112,6 +111,14 @@ sync_sleep, sync_within, ) +from dfo._http.proxies import ( + KEEPALIVE_EXPIRY_S, + MAX_CONNECTIONS, + MAX_KEEPALIVE_CONNECTIONS, + core_url, + environment_proxies, + httpx_errors, +) from dfo._version import __version__ DEFAULT_BASE_URL = "https://developmentfinance.org/api/v1" @@ -621,42 +628,117 @@ def sleep(self, seconds: float) -> None: self._backend.sleep(seconds) -def _keep_to_deadlines(client: httpx2.Client) -> None: - """Route the connections of every pool `client` holds through `_DeadlineBackend`. +class _ResponseStream(httpx2.SyncByteStream): + """A response body from httpcore2, read with httpx2's errors. + + Closing it returns the connection to the pool. + """ + + def __init__(self, stream: Iterable[bytes]) -> None: + self._stream = stream + + def __iter__(self) -> Iterator[bytes]: + with httpx_errors(): + # The generated sync loop stays a loop, as `yield from` has no async form. + for part in self._stream: # noqa: RUF100, UP028 + yield part + + def close(self) -> None: + close = getattr(self._stream, "close", None) + if close is not None: + with httpx_errors(): + close() + - httpx2 builds the pools, its proxy pools from the environment included, and - takes no network backend, so the backend goes in after it builds them. A - transport the caller passed in that is not httpx2's own keeps its timing. +def _connection_pool(proxy_url: str | None) -> httpcore2.ConnectionPool: + """An httpcore2 pool, direct or through `proxy_url`, whose connections keep to + the attempt's deadline through `_DeadlineBackend`. - The pools sit in private attributes of httpx2 and httpcore2, and the stall tests - run against the locked releases, 2.15 for both. + TLS verifies against the system's trust store, or `SSL_CERT_FILE` and + `SSL_CERT_DIR` when set, as httpx2's own transport does. A socks5 or socks5h + proxy needs `socksio` installed. Raises: - RuntimeError: When httpx2 or httpcore2 holds its pools differently. + ValueError: `proxy_url`'s scheme is not http, https, socks5 or socks5h. """ - try: - transports = (client._transport, *client._mounts.values()) - except AttributeError as error: - raise _pool_layout_error() from error - for transport in transports: - if not isinstance(transport, httpx2.HTTPTransport): - continue - pool = getattr(transport, "_pool", None) - if not isinstance(pool, httpcore2.ConnectionPool) or not hasattr( - pool, "_network_backend" - ): - raise _pool_layout_error() - pool._network_backend = _DeadlineBackend(pool._network_backend) - - -def _pool_layout_error() -> RuntimeError: - return RuntimeError( - f"httpx2 {httpx2.__version__} with httpcore2 {httpcore2.__version__} holds " - "its connection pools where dfo cannot route them through its deadlines; " - "install httpx2 and httpcore2 2.15, the series dfo is tested on" + ssl_context = httpx2.create_ssl_context(verify=True, trust_env=True) + # httpcore2 declares a stand-in class when anyio or socksio is missing, so + # the isinstance checks narrow its types. + network = httpcore2.SyncBackend() + assert isinstance(network, httpcore2.NetworkBackend) + backend = _DeadlineBackend(network) + limits: dict[str, Any] = { + "max_connections": MAX_CONNECTIONS, + "max_keepalive_connections": MAX_KEEPALIVE_CONNECTIONS, + "keepalive_expiry": KEEPALIVE_EXPIRY_S, + } + if proxy_url is None: + return httpcore2.ConnectionPool( + ssl_context=ssl_context, network_backend=backend, **limits + ) + proxy = httpx2.Proxy(proxy_url) + if proxy.url.scheme in ("http", "https"): + return httpcore2.HTTPProxy( + proxy_url=core_url(proxy.url), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + network_backend=backend, + **limits, + ) + if proxy.url.scheme in ("socks5", "socks5h"): + pool = httpcore2.SOCKSProxy( + proxy_url=core_url(proxy.url), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + network_backend=backend, + **limits, + ) + assert isinstance(pool, httpcore2.ConnectionPool) + return pool + raise ValueError( + f"A proxy URL's scheme is http, https, socks5 or socks5h, not " + f"{proxy.url.scheme!r}" ) +class _DeadlineTransport(httpx2.BaseTransport): + """An httpx2 transport over an httpcore2 pool whose every network operation + keeps to the attempt's deadline. + + It sends as httpx2's own transport does: HTTP/1.1, no connection retries, + the request's extensions (its timeouts among them) passed to httpcore2 and + the response's passed back, and httpcore2's errors raised as httpx2's. + """ + + def __init__(self, proxy_url: str | None = None) -> None: + self._pool = _connection_pool(proxy_url) + + def handle_request(self, request: httpx2.Request) -> httpx2.Response: + assert isinstance(request.stream, httpx2.SyncByteStream) + core_request = httpcore2.Request( + method=request.method, + url=core_url(request.url), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with httpx_errors(): + response = self._pool.handle_request(core_request) + assert isinstance(response.stream, Iterable) + return httpx2.Response( + status_code=response.status, + headers=response.headers, + stream=_ResponseStream(response.stream), + extensions=response.extensions, + ) + + def close(self) -> None: + with httpx_errors(): + self._pool.close() + + def _check_location(response: httpx2.Response) -> None: """Refuse a redirect whose `Location` httpx2 cannot read as a URL, such as one with a malformed IPv6 host or port, before httpx2 reads it as a protocol fault. @@ -699,7 +781,7 @@ class _Pool: def __init__(self, http_transport: httpx2.BaseTransport | None) -> None: self._http_transport = http_transport - self._proxies = {} if http_transport is not None else get_environment_proxies() + self._proxies = {} if http_transport is not None else environment_proxies() self._pid = os.getpid() self._loop: object | None = None self._closed = False @@ -708,17 +790,15 @@ def __init__(self, http_transport: httpx2.BaseTransport | None) -> None: def _build(self) -> httpx2.Client: # An explicit transport keeps httpx2 from looking the proxies up again. transport = self._http_transport - client = httpx2.Client( - transport=httpx2.HTTPTransport() if transport is None else transport, + return httpx2.Client( + transport=_DeadlineTransport() if transport is None else transport, mounts={ - pattern: None if url is None else httpx2.HTTPTransport(proxy=url) + pattern: None if url is None else _DeadlineTransport(url) for pattern, url in self._proxies.items() }, follow_redirects=False, event_hooks={"response": [_check_location]}, ) - _keep_to_deadlines(client) - return client @property def client(self) -> httpx2.Client: @@ -790,9 +870,9 @@ def __init__( headers: Headers sent with every API request. header_provider: Called before every API request for more headers, such as an identity token. - http_transport: The httpx2 transport, for tests and proxies. An - `httpx2.HTTPTransport` keeps to the attempt's deadline at - every network operation; any other transport times itself. + http_transport: The httpx2 transport, for tests. A transport + passed in times itself, and dfo's own keeps to the attempt's + deadline at every network operation. sleep: Waits between attempts. clock: A monotonic clock, in seconds, for deadlines. caller: What a hint names the owner of a call it suggests: `client` diff --git a/python/src/dfo/_sync/_search/answer.py b/python/src/dfo/_sync/_search/answer.py index ac8344f..58a2080 100644 --- a/python/src/dfo/_sync/_search/answer.py +++ b/python/src/dfo/_sync/_search/answer.py @@ -119,7 +119,8 @@ class Answer: question: The question asked, when the answer came from one. reason: Why Search declined, for a declined answer. warnings: Search's warnings for the whole answer. - timed_out: Whether Search stopped at its time limit. + timed_out: Whether Search stopped at its time limit, by its diagnostics + or by a part's own no-data reason `timed_out`. trace_id: The Search response's trace id, to quote when reporting it. raw: The decoded response body, untouched. """ @@ -237,7 +238,7 @@ def choose( ContractError: The chosen option's `data_query` is not a parsed input of one part; it carries the answer's trace id. """ - # A caller without a type checker can still pass the earlier form. + # A caller without a type checker can pass the `choose(part, option)` form. if old_form: # ty: ignore[redundant-condition] raise TypeError( "answer.choose takes the option first and the part as a keyword: " @@ -500,7 +501,10 @@ def answer_from_body( if isinstance(sentence, str) and sentence else " ".join(part.sentence for part in parts if part.sentence) ) - timed_out = diagnostics.get("timed_out") is True + # Search can stop at its time limit on one part without setting the flag. + timed_out = diagnostics.get("timed_out") is True or any( + isinstance(part, NoDataPart) and part.reason == "timed_out" for part in parts + ) gaps = tuple( gap for index, part in enumerate(parts) diff --git a/python/src/dfo/types.py b/python/src/dfo/types.py index 40ac19b..ac9f22b 100644 --- a/python/src/dfo/types.py +++ b/python/src/dfo/types.py @@ -55,7 +55,7 @@ ) from dfo._datasets.build import Progress from dfo._datasets.bundle import BundleLimits -from dfo._datasets.dataset import LeftOutSeries, LeftOutStep +from dfo._datasets.dataset import LeftOutSeries, LeftOutStep, PartyTotalsLeftOut from dfo._datasets.publication import ( Card, CardKind, @@ -126,6 +126,7 @@ "Option", "Pairing", "Part", + "PartyTotalsLeftOut", "Progress", "RawResponse", "Recipe", diff --git a/python/tests/_async/http/test_transport.py b/python/tests/_async/http/test_transport.py index a7cfd00..5b2921d 100644 --- a/python/tests/_async/http/test_transport.py +++ b/python/tests/_async/http/test_transport.py @@ -4,13 +4,17 @@ the sync transport passes the same tests. """ +import contextlib import gzip import json import math +import socket import ssl +import threading import time import warnings -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from typing import Any import httpcore2 @@ -43,7 +47,6 @@ AsyncTransport, BaseUrl, _DeadlineBackend, - _keep_to_deadlines, attempt_deadline, resolve_base_url, ) @@ -391,22 +394,70 @@ async def test_a_body_that_stalls_after_its_first_chunk_raises_at_the_attempt_en assert 0.55 < elapsed < 0.85 -@pytest.mark.parametrize("attribute", ["_pool", "_network_backend"]) -async def test_a_pool_layout_the_pins_do_not_cover_raises_naming_the_versions( - attribute: str, +async def test_a_body_that_stalls_through_an_environment_proxy_raises_at_the_attempt_end( + monkeypatch: pytest.MonkeyPatch, ) -> None: - async with httpx2.AsyncClient() as client: - transport = client._transport - assert isinstance(transport, httpx2.AsyncHTTPTransport) - holder = transport._pool if attribute == "_network_backend" else transport - held = getattr(holder, attribute) - delattr(holder, attribute) - with pytest.raises(RuntimeError) as caught: - _keep_to_deadlines(client) - # The client's close needs the attribute back. - setattr(holder, attribute, held) - assert f"httpx2 {httpx2.__version__}" in str(caught.value) - assert f"httpcore2 {httpcore2.__version__}" in str(caught.value) + # The stalling server stands in for the proxy, so the request crosses the + # proxy pool the environment names. + for name in ("NO_PROXY", "no_proxy", "ALL_PROXY", "all_proxy", "REQUEST_METHOD"): + monkeypatch.delenv(name, raising=False) + with stalling_server(first_chunk_after=0.4) as proxy: + monkeypatch.setenv("HTTP_PROXY", proxy) + monkeypatch.setenv("http_proxy", proxy) + async with AsyncTransport( + api_key=False, + base_url="http://api.example.org/api/v1", + timeout=0.6, + max_retries=0, + ) as transport: + start = time.monotonic() + with pytest.raises(DfoTimeoutError) as caught: + await transport.request("GET", "/datasets/schema") + elapsed = time.monotonic() - start + assert (caught.value.code, caught.value.retryable) == ("deadline_exceeded", True) + assert 0.55 < elapsed < 0.85 + + +@contextmanager +def keep_alive_server() -> Iterator[tuple[str, list[int]]]: + """Serve `{}` to every request over HTTP/1.1, keeping each connection open. + + Yields: + The server's `http://127.0.0.1:`, and a list that gains a 1 for + each connection it accepts. + """ + accepted: list[int] = [] + listener = socket.create_server(("127.0.0.1", 0)) + listener.settimeout(5) + + def serve(connection: socket.socket) -> None: + with connection: + while connection.recv(65536): + connection.sendall(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}") + + def accept() -> None: + with contextlib.suppress(OSError): + while True: + connection, _ = listener.accept() + accepted.append(1) + threading.Thread(target=serve, args=(connection,), daemon=True).start() + + threading.Thread(target=accept, daemon=True).start() + try: + yield f"http://127.0.0.1:{listener.getsockname()[1]}", accepted + finally: + listener.close() + + +async def test_a_read_response_returns_its_connection_to_the_pool( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NO_PROXY", "*") + with keep_alive_server() as (origin, accepted): + async with AsyncTransport(api_key=False, base_url=origin) as transport: + for _ in range(3): + await transport.request("GET", "/datasets/schema") + assert len(accepted) == 1 class Stream(httpcore2.AsyncNetworkStream): diff --git a/python/tests/_async/search/test_answer.py b/python/tests/_async/search/test_answer.py index 2d4934e..f84a2db 100644 --- a/python/tests/_async/search/test_answer.py +++ b/python/tests/_async/search/test_answer.py @@ -34,7 +34,8 @@ from dfo._datasets.dataset import build_dataset from dfo._datasets.fetch import Session from dfo._http.transport import AsyncTransport -from dfo._search.answer import AsyncAnswer +from dfo._search.answer import AsyncAnswer, answer_from_body +from dfo._search.answer_json import read_answer_json from dfo._search.search import search from tests._data import load from tests._datasets import recorded_member @@ -273,6 +274,19 @@ async def test_a_timed_out_answer_keeps_each_no_data_parts_gap_kind() -> None: assert answer.timed_out +async def test_a_part_that_timed_out_marks_the_answer_and_its_json_timed_out() -> None: + wire = load("search-4.json") + wire["diagnostics"]["timed_out"] = False + wire["parts"][1]["no_data"]["reason"] = "timed_out" + answer, _ = await ask("debt", wire) + assert answer.timed_out + question, body, trace_id = read_answer_json(answer.to_json()) + revived = answer_from_body( + body, question=question, source=answer._source, trace_id=trace_id + ) + assert revived.timed_out + + async def test_unknown_and_unsupported_parts_are_gaps() -> None: wire = load("search-4.json") wire["parts"][0]["status"] = "partly_definite" diff --git a/python/tests/_sync/http/test_transport.py b/python/tests/_sync/http/test_transport.py index f772b3e..49ef1cc 100644 --- a/python/tests/_sync/http/test_transport.py +++ b/python/tests/_sync/http/test_transport.py @@ -5,13 +5,17 @@ the sync transport passes the same tests. """ +import contextlib import gzip import json import math +import socket import ssl +import threading import time import warnings -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from typing import Any import httpcore2 @@ -44,7 +48,6 @@ BaseUrl, Transport, _DeadlineBackend, - _keep_to_deadlines, attempt_deadline, resolve_base_url, ) @@ -384,22 +387,70 @@ def test_a_body_that_stalls_after_its_first_chunk_raises_at_the_attempt_end( assert 0.55 < elapsed < 0.85 -@pytest.mark.parametrize("attribute", ["_pool", "_network_backend"]) -def test_a_pool_layout_the_pins_do_not_cover_raises_naming_the_versions( - attribute: str, +def test_a_body_that_stalls_through_an_environment_proxy_raises_at_the_attempt_end( + monkeypatch: pytest.MonkeyPatch, ) -> None: - with httpx2.Client() as client: - transport = client._transport - assert isinstance(transport, httpx2.HTTPTransport) - holder = transport._pool if attribute == "_network_backend" else transport - held = getattr(holder, attribute) - delattr(holder, attribute) - with pytest.raises(RuntimeError) as caught: - _keep_to_deadlines(client) - # The client's close needs the attribute back. - setattr(holder, attribute, held) - assert f"httpx2 {httpx2.__version__}" in str(caught.value) - assert f"httpcore2 {httpcore2.__version__}" in str(caught.value) + # The stalling server stands in for the proxy, so the request crosses the + # proxy pool the environment names. + for name in ("NO_PROXY", "no_proxy", "ALL_PROXY", "all_proxy", "REQUEST_METHOD"): + monkeypatch.delenv(name, raising=False) + with stalling_server(first_chunk_after=0.4) as proxy: + monkeypatch.setenv("HTTP_PROXY", proxy) + monkeypatch.setenv("http_proxy", proxy) + with Transport( + api_key=False, + base_url="http://api.example.org/api/v1", + timeout=0.6, + max_retries=0, + ) as transport: + start = time.monotonic() + with pytest.raises(DfoTimeoutError) as caught: + transport.request("GET", "/datasets/schema") + elapsed = time.monotonic() - start + assert (caught.value.code, caught.value.retryable) == ("deadline_exceeded", True) + assert 0.55 < elapsed < 0.85 + + +@contextmanager +def keep_alive_server() -> Iterator[tuple[str, list[int]]]: + """Serve `{}` to every request over HTTP/1.1, keeping each connection open. + + Yields: + The server's `http://127.0.0.1:`, and a list that gains a 1 for + each connection it accepts. + """ + accepted: list[int] = [] + listener = socket.create_server(("127.0.0.1", 0)) + listener.settimeout(5) + + def serve(connection: socket.socket) -> None: + with connection: + while connection.recv(65536): + connection.sendall(b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\n{}") + + def accept() -> None: + with contextlib.suppress(OSError): + while True: + connection, _ = listener.accept() + accepted.append(1) + threading.Thread(target=serve, args=(connection,), daemon=True).start() + + threading.Thread(target=accept, daemon=True).start() + try: + yield f"http://127.0.0.1:{listener.getsockname()[1]}", accepted + finally: + listener.close() + + +def test_a_read_response_returns_its_connection_to_the_pool( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("NO_PROXY", "*") + with keep_alive_server() as (origin, accepted): + with Transport(api_key=False, base_url=origin) as transport: + for _ in range(3): + transport.request("GET", "/datasets/schema") + assert len(accepted) == 1 class Stream(httpcore2.NetworkStream): diff --git a/python/tests/_sync/search/test_answer.py b/python/tests/_sync/search/test_answer.py index 5b71065..1493919 100644 --- a/python/tests/_sync/search/test_answer.py +++ b/python/tests/_sync/search/test_answer.py @@ -33,9 +33,10 @@ from dfo._core.types import CandidatesPart, DefinitePart from dfo._datasets.build import Reading from dfo._datasets.dataset import build_dataset +from dfo._search.answer_json import read_answer_json from dfo._sync._datasets.fetch import Session from dfo._sync._http.transport import Transport -from dfo._sync._search.answer import Answer +from dfo._sync._search.answer import Answer, answer_from_body from dfo._sync._search.search import search from tests._data import load from tests._datasets import recorded_member @@ -272,6 +273,19 @@ def test_a_timed_out_answer_keeps_each_no_data_parts_gap_kind() -> None: assert answer.timed_out +def test_a_part_that_timed_out_marks_the_answer_and_its_json_timed_out() -> None: + wire = load("search-4.json") + wire["diagnostics"]["timed_out"] = False + wire["parts"][1]["no_data"]["reason"] = "timed_out" + answer, _ = ask("debt", wire) + assert answer.timed_out + question, body, trace_id = read_answer_json(answer.to_json()) + revived = answer_from_body( + body, question=question, source=answer._source, trace_id=trace_id + ) + assert revived.timed_out + + def test_unknown_and_unsupported_parts_are_gaps() -> None: wire = load("search-4.json") wire["parts"][0]["status"] = "partly_definite" diff --git a/python/tests/datasets/test_dataset.py b/python/tests/datasets/test_dataset.py index 3e5742b..c5f7f2e 100644 --- a/python/tests/datasets/test_dataset.py +++ b/python/tests/datasets/test_dataset.py @@ -5,6 +5,7 @@ import math import pickle import struct +import subprocess import sys from collections.abc import Callable from dataclasses import replace @@ -37,6 +38,7 @@ Rendering, RequestedOperation, ) +from dfo._datasets import dataset as dataset_module from dfo._datasets.bundle import load as load_saved from dfo._datasets.cache import sha256_digest from dfo._datasets.dataset import ( @@ -44,6 +46,7 @@ FetchedMember, LeftOutSeries, LeftOutStep, + PartyTotalsLeftOut, build_dataset, derived_dataset, in_reading_order, @@ -51,6 +54,15 @@ from tests._data import DATA, load from tests._datasets import PARQUET, manifest, other_member, recorded_member, scenario from tests._fetch_support import FakeCache +from tests.contract.test_replay import ( + BASE_URLS, + SCENARIOS, + Clock, + Exchanges, + Pollers, + _client, + _run, +) from tests.rule_vectors import ids, vectors EXPECTED = scenario()["expect"][0]["dataset"] @@ -303,14 +315,41 @@ def test_a_column_of_another_nesting_is_refused(column: pa.Array) -> None: assert caught.value.code == "contract_violation" +@pytest.fixture(params=["native", "python"]) +def stacking(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> str: + """Stack struct columns through Arrow, where the installed pyarrow can, or + through Python cells, as older pyarrow does.""" + if request.param == "python": + monkeypatch.setattr(dataset_module, "_pyarrow_major", lambda: 15) + elif dataset_module._pyarrow_major() < dataset_module._NATIVE_STRUCT_PROMOTION: + pytest.skip("this pyarrow stacks struct columns through Python") + return request.param + + +@pytest.mark.usefixtures("stacking") def test_struct_columns_stack_by_field_name() -> None: """Two members' structs unify by field name, in the order the first names them, a field one member lacks null in its rows.""" + # Typed in full, since older pyarrow sorts the keys of an inferred struct. first = pa.table( - {"time_period": ["2020"], "value": [1.0], "s": [{"b": "x", "a": 1}]} + { + "time_period": ["2020"], + "value": [1.0], + "s": pa.array( + [{"b": "x", "a": 1}], + pa.struct([("b", pa.string()), ("a", pa.int64())]), + ), + } ) second = pa.table( - {"time_period": ["2021"], "value": [2.0], "s": [{"a": 2.5, "c": True}]} + { + "time_period": ["2021"], + "value": [2.0], + "s": pa.array( + [{"a": 2.5, "c": True}], + pa.struct([("a", pa.float64()), ("c", pa.bool_())]), + ), + } ) data = built( with_table(recorded_member(), first), with_table(other_member(), second) @@ -325,16 +364,94 @@ def test_struct_columns_stack_by_field_name() -> None: ] +def _struct_member(base: FetchedMember, column: pa.Array) -> FetchedMember: + """`base` holding one row whose `s` column holds `column`.""" + table = pa.table({"time_period": ["2020"], "value": [1.0], "s": column}) + return with_table(base, table) + + +@pytest.mark.usefixtures("stacking") +def test_a_struct_field_too_large_to_unify_raises_a_contract_error() -> None: + """A uint64 above 2**63 unified with an int64 fits neither.""" + members = [ + _struct_member( + recorded_member(), + pa.array([{"a": 2**63 + 5}], pa.struct([("a", pa.uint64())])), + ), + _struct_member( + other_member(), + pa.array( + [{"a": -1, "c": True}], + pa.struct([("a", pa.int64()), ("c", pa.bool_())]), + ), + ), + ] + with pytest.raises(ContractError) as caught: + built(*members) + assert caught.value.code == "contract_violation" + + +# Stacks a sub-microsecond timestamp in a struct through Python cells with pandas +# unimportable, since pandas would hold the cell as a Timestamp. +_NANOSECOND_STACK = """ +import sys + +class HidePandas: + def find_spec(self, name, path=None, target=None): + if name.partition(".")[0] == "pandas": + raise ModuleNotFoundError(name) + +sys.meta_path.insert(0, HidePandas()) +import pyarrow as pa +from dfo._core.errors import ContractError +from dfo._datasets import dataset +from tests._datasets import other_member, recorded_member +from tests.datasets.test_dataset import _struct_member, built + +dataset._pyarrow_major = lambda: 15 +nanosecond = pa.struct([("t", pa.timestamp("ns"))]) +try: + built( + _struct_member(recorded_member(), pa.array([{"t": 1}], nanosecond)), + _struct_member(other_member(), pa.array([{"u": 2}])), + ) +except ContractError as error: + print(error.code) +""" + + +def test_a_nanosecond_struct_field_python_cannot_hold_raises_a_contract_error() -> None: + """Without pandas, pyarrow refuses a sub-microsecond timestamp as a Python + value. A fresh interpreter hides pandas, since this process has loaded it.""" + result = subprocess.run( + [sys.executable, "-c", _NANOSECOND_STACK], + capture_output=True, + text=True, + check=False, + cwd=Path(__file__).parents[2], + ) + assert result.returncode == 0, result.stderr + assert result.stdout.split() == ["contract_violation"] + + def test_a_sort_key_the_order_cannot_compare_is_refused() -> None: table = pq.read_table(PARQUET).replace_schema_metadata(None) # Arrow sorts no 16-bit float, and a varying column is a sort key. - halves = pa.array([float(row) for row in range(table.num_rows)]).cast(pa.float16()) + halves = _halves([float(row) for row in range(table.num_rows)]) table = table.append_column("half", halves) with pytest.raises(ContractError) as caught: built(with_table(recorded_member(), table)) assert caught.value.code == "contract_violation" +def _halves(values: list[float]) -> pa.Array: + """A 16-bit float array. pyarrow casts to one only from 16 on, so it is built + from numpy, which pandas brings in.""" + import numpy as np # noqa: PLC0415 - pandas, a test dependency, requires it + + return pa.array(np.array(values, dtype=np.float16)) + + # A NaN with another payload than the one Python writes. _OTHER_NAN = struct.unpack(" None: @pytest.mark.parametrize( ("share", "varies"), [ - (pa.array([0.0, -0.0]).cast(pa.float16()), False), + (_halves([0.0, -0.0]), False), (pa.array([0.0, -0.0], pa.float32()), False), (pa.array([0.0, -0.0]), False), (pa.array([math.nan, _OTHER_NAN]), False), @@ -698,7 +815,7 @@ def test_polars_reads_the_arrow_stream() -> None: def test_duckdb_reads_the_arrow_stream() -> None: duckdb = pytest.importorskip("duckdb") data = built() # noqa: F841 - DuckDB finds it by name - assert duckdb.sql("select count(*) from data").fetchone() == (4,) + assert duckdb.sql("select count(*) from data").fetchall() == [(4,)] def test_a_dataset_pickles() -> None: @@ -1048,6 +1165,137 @@ def test_a_manifest_without_left_out_reads_as_none_left_out() -> None: assert " Derived: from 2 readings" in repr(dataset).splitlines() +# The keys the 0.1.0 loader requires in `dfo.dataset` and refuses beyond. +DATASET_KEYS_0_1_0 = { + "member_citations", + "warnings", + "departures", + "requested_operations", + "assumptions", + "complete", + "missing", + "held_until", + "pinned", + "stale", + "layout_version", + "member_rows", +} + + +def test_a_fetch_that_left_party_totals_out_names_them_and_saves_them( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + record = SCENARIOS["fetch-spec-party-totals-left-out"] + exchanges = Exchanges(record, BASE_URLS["api-path"]) + clock = Clock(Pollers(monkeypatch).poll_waits) + client = _client("sync", BASE_URLS["api-path"], exchanges, clock) + (step,) = record["call"] + data = _run(step, client, None, None) + (served,) = record["exchanges"][0]["response"]["json"]["trace"][ + "party_totals_left_out" + ] + assert "provenance" not in served + assert data.party_totals_left_out == ( + PartyTotalsLeftOut( + spec_id=data.members[0].spec.spec_id, + role="financingProvider", + entities=tuple(served["entities"]), + every_provenance=True, + ), + ) + count = len(served["entities"]) + assert ( + f" Party totals left out: {count} on financingProvider · " + "data.party_totals_left_out" + ) in repr(data).splitlines() + + path = tmp_path / "saved.parquet" + data.save(path) + stored = pq.read_metadata(path).metadata + assert set(json.loads(stored[b"dfo.dataset"])) == DATASET_KEYS_0_1_0 + assert b"dfo.party_totals_left_out" in stored + _assert_same(load_saved(path), data) + + +@pytest.mark.parametrize( + ("served", "provenance", "every"), + [ + ({"provenance": "provenance/OECD_DAC2a"}, "provenance/OECD_DAC2a", False), + ({"provenance": None}, None, False), + ({}, None, True), + ], + ids=["named", "rows-without-provenance", "every-provenance"], +) +def test_party_totals_keep_the_provenance_through_save_and_load( + tmp_path: Path, served: dict[str, Any], provenance: str | None, *, every: bool +) -> None: + body = manifest() + entry = {"role": "financingRecipient", "entities": ["AfricaTotal"], **served} + body["trace"] = {"party_totals_left_out": [entry], "other": 1} + data = built(recorded_member(manifest=body)) + (read,) = data.party_totals_left_out + assert (read.provenance, read.every_provenance) == (provenance, every) + path = tmp_path / "saved.parquet" + data.save(path) + assert load_saved(path).party_totals_left_out == data.party_totals_left_out + + +def test_a_dataset_without_party_totals_left_out_saves_no_key(tmp_path: Path) -> None: + data = built() + assert data.party_totals_left_out == () + assert not any("Party totals" in line for line in repr(data).splitlines()) + path = tmp_path / "saved.parquet" + data.save(path) + assert b"dfo.party_totals_left_out" not in pq.read_metadata(path).metadata + + +@pytest.mark.parametrize( + ("trace", "pointer"), + [ + ([], "/trace"), + ({"party_totals_left_out": {}}, "/trace/party_totals_left_out"), + ({"party_totals_left_out": [[]]}, "/trace/party_totals_left_out/0"), + ( + {"party_totals_left_out": [{"entities": []}]}, + "/trace/party_totals_left_out/0/role", + ), + ( + {"party_totals_left_out": [{"role": "financingProvider"}]}, + "/trace/party_totals_left_out/0/entities", + ), + ( + {"party_totals_left_out": [{"role": "financingProvider", "entities": [1]}]}, + "/trace/party_totals_left_out/0/entities/0", + ), + ( + { + "party_totals_left_out": [ + {"role": "financingProvider", "entities": [], "provenance": 2} + ] + }, + "/trace/party_totals_left_out/0/provenance", + ), + ], + ids=[ + "trace-not-object", + "list-not-array", + "entry-not-object", + "role-missing", + "entities-missing", + "entity-not-text", + "provenance-not-text", + ], +) +def test_a_malformed_party_totals_trace_raises_contract_error( + trace: object, pointer: str +) -> None: + body = manifest() + body["trace"] = trace + with pytest.raises(ContractError) as caught: + built(recorded_member(manifest=body)) + assert (caught.value.code, caught.value.pointer) == ("contract_violation", pointer) + + def _saved() -> Dataset: """A dataset of two members and two gaps, with every provenance member filled.""" (part,) = parse_parts(load("search-1.json")) @@ -1139,6 +1387,16 @@ def edit(metadata: dict[bytes, bytes]) -> None: _dataset_edit(lambda d: d.update(member_rows=[9, -1])), "/dfo.dataset/member_rows", ), + ( + lambda m: m.update({b"dfo.party_totals_left_out": b"{}"}), + "/dfo.party_totals_left_out", + ), + ( + lambda m: m.update( + {b"dfo.party_totals_left_out": b'[{"role": "r", "entities": []}]'} + ), + "/dfo.party_totals_left_out/0/spec_id", + ), ], ids=[ "citation-removed", @@ -1160,6 +1418,8 @@ def edit(metadata: dict[bytes, bytes]) -> None: "member-rows-not-summing-to-the-rows", "member-rows-not-one-per-member", "member-rows-negative", + "party-totals-not-array", + "party-totals-without-spec-id", ], ) def test_a_damaged_saved_file_raises_format_error( @@ -1269,6 +1529,7 @@ def _assert_same(loaded: Dataset, data: Dataset) -> None: "licence", "ref", "left_out", + "party_totals_left_out", ): assert getattr(loaded, name) == getattr(data, name), name assert len(loaded) == len(data) diff --git a/python/tests/e2e/test_local_stack.py b/python/tests/e2e/test_local_stack.py index 9a6ed2d..cd6f165 100644 --- a/python/tests/e2e/test_local_stack.py +++ b/python/tests/e2e/test_local_stack.py @@ -71,6 +71,8 @@ "cv_96556e3d3e312acb219d633392cd1d05", "Kenya GNI (PPP), 2015 to 2024", ) +# The codes a server without the seed answers a seeded reading with. +NOT_FOUND_CODES = {"spec_not_found", "snapshot_not_found"} PAIRING = {"destination": "place"} # A breakdown by recipient, the sum data-mcp's `get_data` reads a frame with. BREAKDOWN = {"by": ["time_period", "financingRecipient"], "op": "sum"} @@ -174,6 +176,31 @@ def e2e_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: return path +@pytest.fixture(scope="module", autouse=True) +def seed(client: Client) -> None: + """Stops the run once, before any step, when the server lacks the seed. + + Each seeded reading is probed once. Only the not-found codes count as + missing. Any other error propagates as itself. `pytest.exit` ends the run + with one failure, where `pytest.fail` here would error every step. + """ + missing = [] + for spec_id, content_version, _ in (ODA, POPULATION): + try: + client.request("GET", f"/datasets/{spec_id}/{content_version}") + except NotFoundError as error: + if error.code not in NOT_FOUND_CODES: + raise + missing.append(f"{spec_id} at {content_version}") + if missing: + pytest.exit( + "The server lacks the local-stack seed; missing readings: " + + ", ".join(missing) + + ". Run `just local-stack seed` in dfo-apis.", + returncode=1, + ) + + def _seeded(client: Client, reading: tuple[str, str, str]) -> Dataset: spec_id, content_version, sentence = reading manifest = client.request("GET", f"/datasets/{spec_id}/{content_version}").json() @@ -345,8 +372,10 @@ def test_06_cards_draw_the_derived_frame( for card in cards: payload = _payload(client, card.id) described = payload["description"]["sources"] - # The description names the root readings the card draws, one source each. - assert sorted(source["ref"] for source in described) == roots + # The description names each root reading the card draws once, beside the + # publishers it credits. + refs = [source["ref"] for source in described] + assert all(refs.count(root) == 1 for root in roots), refs assert all(source["label"] for source in described) provenance = payload["provenance"] assert provenance["inputs"] == [ diff --git a/python/tests/http/test_environment_proxies.py b/python/tests/http/test_environment_proxies.py new file mode 100644 index 0000000..0f13646 --- /dev/null +++ b/python/tests/http/test_environment_proxies.py @@ -0,0 +1,51 @@ +"""Tests for the proxy mounts the transport reads from the environment.""" + +import pytest + +from dfo._http.proxies import environment_proxies + +PROXY = "http://proxy.example.net:3128" + + +@pytest.mark.parametrize( + ("no_proxy", "pattern"), + [ + (".example.org", "all://*.example.org"), + ("example.org", "all://*example.org"), + ("192.168.1.10", "all://192.168.1.10"), + ("10.0.0.0/8", "all://10.0.0.0/8"), + ("::1", "all://[::1]"), + ("fd00::/8", "all://[fd00::]/8"), + ("[::1]", "all://[::1]"), + ("localhost", "all://localhost"), + ("LOCALHOST", "all://LOCALHOST"), + ("https://internal.example.org", "https://internal.example.org"), + # Entries httpx2 cannot read as a pattern, and blanks, mount nothing. + ("10.0.0.0/99", None), + ("proxy.example.org:port", None), + ("example.org/path", None), + (" ", None), + ], +) +def test_each_no_proxy_entry_mounts_its_pattern( + no_proxy: str, pattern: str | None +) -> None: + mounts = environment_proxies({"http": PROXY, "no": f"{no_proxy},"}) + expected = ( + {"http://": PROXY} if pattern is None else {"http://": PROXY, pattern: None} + ) + assert mounts == expected + + +def test_no_proxy_star_turns_every_proxy_off() -> None: + proxies = {"http": PROXY, "https": PROXY, "no": "example.org, *"} + assert environment_proxies(proxies) == {} + + +def test_each_scheme_mounts_its_proxy_with_http_when_it_names_none() -> None: + proxies = {"http": "proxy:3128", "https": PROXY, "all": "socks5://s:1080"} + assert environment_proxies(proxies) == { + "http://": "http://proxy:3128", + "https://": PROXY, + "all://": "socks5://s:1080", + } diff --git a/python/tests/test_codegen.py b/python/tests/test_codegen.py index 6bcaf56..db39b0e 100644 --- a/python/tests/test_codegen.py +++ b/python/tests/test_codegen.py @@ -98,6 +98,8 @@ def test_async_source_may_not_import_an_async_module_from_its_package( "x = httpx2.AsyncByteStream\n", "await response.aread()\n", "def __aiter__(self): ...\n", + "backend = httpcore2.AnyIOBackend()\n", + "pool.handle_async_request(request)\n", ], ) def test_an_async_name_left_in_the_output_is_refused(source: str) -> None: diff --git a/python/tests/test_dfo.py b/python/tests/test_dfo.py index fe5476f..337f775 100644 --- a/python/tests/test_dfo.py +++ b/python/tests/test_dfo.py @@ -160,6 +160,7 @@ def test_the_top_level_holds_the_verbs_the_core_types_and_the_task_modules() -> "Option", "Pairing", "Part", + "PartyTotalsLeftOut", "Progress", "RawResponse", "Recipe", diff --git a/python/uv.lock b/python/uv.lock index 5a354d7..9040009 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15' and sys_platform != 'emscripten'", @@ -259,11 +259,11 @@ dev = [ [package.metadata] requires-dist = [ - { name = "httpcore2", marker = "sys_platform != 'emscripten'", specifier = ">=2.15,<2.16" }, - { name = "httpx2", specifier = ">=2.15,<2.16" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'", specifier = ">=2.15" }, + { name = "httpx2", specifier = ">=2.15" }, { name = "openpyxl", marker = "extra == 'xlsx'", specifier = ">=3.1" }, { name = "pandas", marker = "extra == 'pandas'", specifier = ">=2.2" }, - { name = "polars", marker = "extra == 'polars'", specifier = ">=1.0" }, + { name = "polars", marker = "extra == 'polars'", specifier = ">=1.3" }, { name = "pyarrow", specifier = ">=15" }, { name = "typing-extensions", specifier = ">=4.12" }, ] @@ -441,20 +441,20 @@ wheels = [ [[package]] name = "httpcore2" -version = "2.15.1" +version = "2.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "h11" }, { name = "truststore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c2/a9/11bc39967bb9dfaab0981ab4d49f519818acbee662a7e76d06813319c838/httpcore2-2.15.1.tar.gz", hash = "sha256:a94b6b3e6260e1a3f035915408d3dcd7dc8e844c2b544b642e227cf98a747af0", size = 69859, upload-time = "2026-10-19T20:14:23.824Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/61/afa9d6b7452f9eca73f73d45324eb0bb52f098fd5e6049bf6b8263be9b4e/httpcore2-2.16.0.tar.gz", hash = "sha256:c72cd0413563b17e1cd8354677dff887466f8ec2551679759e93126534428643", size = 69886, upload-time = "2026-10-21T07:09:45.31Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/9b/e2b556e26fbc67716ab1dfc18832eb6fe196d88363f7094cb1da3d7c0d3e/httpcore2-2.15.1-py3-none-any.whl", hash = "sha256:730f603058c260a3288378217b464713350c23e3b8fe8bba753701054032d98a", size = 85055, upload-time = "2026-10-19T20:14:21.363Z" }, + { url = "https://files.pythonhosted.org/packages/af/c2/ce417c397afa5181f0e3c6ff01a4d01639414e1adef08730d97571297151/httpcore2-2.16.0-py3-none-any.whl", hash = "sha256:5c4358319f2602d984ad93182c9fee5160f22e9da7babdda870a5e7c4e2dc34f", size = 85070, upload-time = "2026-10-21T07:09:42.802Z" }, ] [[package]] name = "httpx2" -version = "2.15.1" +version = "2.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform != 'emscripten'" }, @@ -464,9 +464,9 @@ dependencies = [ { name = "truststore", marker = "sys_platform != 'emscripten'" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/85/07978f4afd168d6b408fa75c1dc13e3010653a81e182aa1b6d89ab94d9c6/httpx2-2.15.1.tar.gz", hash = "sha256:ff75485308fbe441e75b1218546e4e78d884df41e1a9741c1f79335ed8fcfef7", size = 101673, upload-time = "2026-10-19T20:14:24.587Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/73/5ca2cedf5e4e50423d1c258877865a6b62793633b561c58499f8aeb33b7e/httpx2-2.16.0.tar.gz", hash = "sha256:0d54c98128e8f2ff7fbbde0fca8c359e5f032edeaaaac443cdb7f31f685bcd7f", size = 103116, upload-time = "2026-10-21T07:09:46.348Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/bf/56068f774620330c8330e4b4915fae889a5b2d4b52667b3b7c8f66c4a9ad/httpx2-2.15.1-py3-none-any.whl", hash = "sha256:75841cd83d16b417f7d6da8f2089ce8120806efdbf96af37d46c5129c49015be", size = 96411, upload-time = "2026-10-19T20:14:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3d/2e4bc4fd1ef9770871669adb348d612a6e4786b303a23f9935b81c48dd60/httpx2-2.16.0-py3-none-any.whl", hash = "sha256:fd2c5d7909e8d86fe92d53020c310ca0e4e431b838c207c5754b9bbf0308d49a", size = 97880, upload-time = "2026-10-21T07:09:44.01Z" }, ] [[package]] @@ -1389,7 +1389,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.14.11" +version = "21.14.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -1398,7 +1398,7 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/be/4d1d59d309db76b5808fcfd78bd5996f4e7241366691c38fc331a3136398/virtualenv-21.14.11.tar.gz", hash = "sha256:593052d8477ea8c55a842387ce111bd08c72618f0c1133c302cafe335e570a57", size = 5503091, upload-time = "2026-10-18T07:06:34.716Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/aa/bff50daee0488e16d611258a9b6c5a1d0c9e128f8ca424e9f4e1c0444ef2/virtualenv-21.14.13.tar.gz", hash = "sha256:de5307ba4b9a0001d9c8ea02832d38f63ca432880f416c1bface61c4323fd905", size = 5570301, upload-time = "2026-10-22T20:10:16.952Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/6b/d85da8982b41190eafcb9df91dff9d83fc6989e98af4b5be4c8f85510c95/virtualenv-21.14.11-py3-none-any.whl", hash = "sha256:b319f8590a84e5de5f8f2c803ecd2237e9bc0ffa7c1e875e3169c107cecb7454", size = 5495268, upload-time = "2026-10-18T07:06:32.719Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a9/c10d8a44a29fb94a18b43fdf11f38caca53ae7c9406f11a72b1cb8f624f1/virtualenv-21.14.13-py3-none-any.whl", hash = "sha256:5fbb04138a4861580b78ba9110d529a26bc48fcc93c8ec36bc0202eeed96a788", size = 5557094, upload-time = "2026-10-22T20:10:15.096Z" }, ] diff --git a/scripts/gen_sync.py b/scripts/gen_sync.py index 2771121..379b7d1 100755 --- a/scripts/gen_sync.py +++ b/scripts/gen_sync.py @@ -18,8 +18,9 @@ at the sync twin; - a token in `SUBS` that no source uses, so a renamed async name in httpx2 or in this package fails here; -- an async name left in the output (`async`, `await`, `Async*`, `__aenter__`, - `aclose`, ...), which means `SUBS` lacks a token; +- an async name left in the output (a name holding `async` or `Async`, such + as `handle_async_request`, `await`, `AnyIOBackend`, `__aenter__`, `aclose`, + ...), which means `SUBS` lacks a token; - a sync directory without its hand-written `__init__.py`, without which the sync tests collide with the async ones of the same name. """ @@ -77,11 +78,17 @@ ("AsyncAnalysis", "Analysis"), ("AsyncCards", "Cards"), ("AsyncIterator", "Iterator"), + ("AsyncIterable", "Iterable"), ("AsyncBaseTransport", "BaseTransport"), - ("AsyncHTTPTransport", "HTTPTransport"), + # httpx2's byte streams and httpcore2's backends take other names in sync code. + ("AsyncByteStream", "SyncByteStream"), + ("AnyIOBackend", "SyncBackend"), ("AsyncNetworkBackend", "NetworkBackend"), ("AsyncNetworkStream", "NetworkStream"), ("AsyncConnectionPool", "ConnectionPool"), + ("AsyncHTTPProxy", "HTTPProxy"), + ("AsyncSOCKSProxy", "SOCKSProxy"), + ("handle_async_request", "handle_request"), ("AsyncHeaderProvider", "HeaderProvider"), ("AsyncSleep", "Sleep"), ("async_sleep", "sync_sleep"), @@ -92,14 +99,18 @@ ("aread", "read"), ("aiter_bytes", "iter_bytes"), ("__aenter__", "__enter__"), + ("__aiter__", "__iter__"), ("__aexit__", "__exit__"), ("pytestmark = pytest.mark.anyio", ""), ) FORBIDDEN_IMPORTS = frozenset({"asyncio", "anyio", "trio"}) +# A name left in the output that only async code uses: one holding `async` or +# `Async`, `await`, an anyio backend, or an async dunder or method. ASYNC_NAME = re.compile( - r"async|await|Async[A-Z]\w*|__a(enter|exit|iter|next)__|a(close|read|send|iter\w*)" + r"\w*async\w*|await|\w*Async[A-Z]\w*|AnyIO\w*|__a(enter|exit|iter|next)__" + r"|a(close|read|send|iter\w*)" ) diff --git a/typescript/CHANGELOG.md b/typescript/CHANGELOG.md index e7048a0..3e96581 100644 --- a/typescript/CHANGELOG.md +++ b/typescript/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this package are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `data.partyTotalsLeftOut` names the group totals a fetch left out of each member, as `PartyTotalsLeftOut` entries of spec id, party role, dcids and provenance, and `String(data)` counts them. `data.bytes()` writes them under a new `dfo.party_totals_left_out` key, which Python's `dfo.load` reads and its 0.1.0 release ignores. + +- `answer.facts` gives each part's assumptions, departures, requested operations and warnings as typed `PartFacts`, one entry per part in the order of `answer.parts`, with a candidates part's `readingAssumptions` by spec id. `partFacts(part)` reads one part, and `PartFacts` is exported from the package root. + +### Fixed + +- `answer.timedOut` is true when a part's own no-data reason is `timed_out`, as well as when Search's diagnostics say so, and `Answer.fromJSON` works it out again. + ## [0.1.0] - 2026-11-13 The first release of `@developmentfinance/client`, the TypeScript client for the Development Finance Observatory APIs. It is ESM only and runs on Node 22.18 or later, Bun, Deno, Workers, and browsers. diff --git a/typescript/README.md b/typescript/README.md index ccaeb1f..65c9d31 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -34,7 +34,9 @@ const table = await toArrow(data); console.log(table.numRows); const growth = await client.analysis.growth(data, { window: [2015, 2024] }); -const chart = await client.cards.line(data, { title: "German aid to Nigeria for basic health care" }); +const chart = await client.cards.line(data, { + title: "German aid to Nigeria for basic health care", +}); const share = await client.share(chart, { latest: true }); console.log(share.url); ``` @@ -72,6 +74,18 @@ console.log(data.sentence); `choose` takes the option's index, its label, or its line as `String(answer)` prints it, such as `"Kenya's percent of nominal GDP spent on education, 1971 to 2024 · World Development Indicators"`, and returns Search's answer for that reading. When several parts await a choice, name the part: `answer.choose(0, { part: "education spending" })`. A label or part that names nothing raises `InvalidRequestError` with the code `choice_invalid`, the nearest match and the valid choices. `answer.gaps` holds the same choices as data, for a page that draws its own buttons. Each gap's `options` carry a `label`. +Search states more about each part than its reading. `answer.facts` holds one entry per part, in the order of `answer.parts`, and each has typed fields: + +```ts +const [facts] = answer.facts; +console.log(facts.assumptions?.map((item) => item.text)); +console.log(facts.departures.map((item) => item.code)); +console.log(facts.operations.map((item) => item.phrase)); +console.log(facts.warnings.map((item) => item.message)); +``` + +`assumptions` lists what Search assumed to read the part, and is undefined when the part holds no list. `departures` lists how the reading differs from the question, `operations` the computations the question asked for that Search does not run, and `warnings` the part's own. A candidates part also has `readingAssumptions`, a map from each reading's `spec_id` to its assumptions. An item that lacks a required member is dropped. `partFacts(part)` reads a single part and gives the same result. + ## Narrow a fetch `client.fetch(spec, options)` builds one of the answer's readings, `answer.specs[0]` here, with a narrower window, other places, or a fixed facet: @@ -131,6 +145,8 @@ const withTotals = await answer.fetch({ partyTotals: true }); console.log(members.rowCount, withTotals.rowCount); ``` +`data.partyTotalsLeftOut` names the totals a fetch left out. It holds one `PartyTotalsLeftOut` per member, party role and provenance that lost rows, with the role's column name, such as `financingProvider`, and the dcids of the totals, such as `DACCountriesTotal`. `String(data)` counts them on its `Party totals left out` line, and `data.bytes()` keeps them. The array is empty after `partyTotals: true` and when the cut found no totals to drop. + ## Stop a call Every call that goes to the server takes a `signal` and a `timeout` in milliseconds. For `search` and `fetch` the `timeout` bounds the whole call, and `search` stops after 60 seconds by default. For a lookup or `client.request` it bounds one attempt, which is retried up to `maxRetries` times. `fetch` waits as long as the build takes, showing progress to `onProgress`, unless you give it a `timeout`. diff --git a/typescript/src/client.ts b/typescript/src/client.ts index 78eb449..a63b476 100644 --- a/typescript/src/client.ts +++ b/typescript/src/client.ts @@ -5,6 +5,7 @@ import type { Answer, AnswerContext, SearchOptions } from "./search/answer.ts"; import type { Dataset } from "./datasets/dataset.ts"; import { clientError, DfoError, InvalidRequestError, messageOf, optionsOf } from "./core/errors.ts"; import { present } from "./core/json.ts"; +import { trimTrailingSlashes } from "./core/text.ts"; import { checkOptions, declareOptions } from "./core/options.ts"; import { Analysis } from "./datasets/analysis.ts"; import { Cards } from "./datasets/cards.ts"; @@ -147,7 +148,7 @@ export class Client { /** The API's base URL, less any trailing slashes. */ get baseUrl(): string { - return this.#transport.baseUrl.href.replace(/\/+$/, ""); + return trimTrailingSlashes(this.#transport.baseUrl.href); } /** The client as ``, naming its base URL, with `(from DFO_BASE_URL)` when diff --git a/typescript/src/config/credentials.ts b/typescript/src/config/credentials.ts index fed1256..112d85f 100644 --- a/typescript/src/config/credentials.ts +++ b/typescript/src/config/credentials.ts @@ -1,3 +1,4 @@ +import { trimEnd } from "../core/text.ts"; import type { EnvReader } from "./env.ts"; /** The platforms the credentials path rule distinguishes. */ @@ -56,7 +57,11 @@ export function parseCredentialsFile(text: string): ParsedCredentialsFile { const profiles: CredentialsProfiles = Object.create(null) as CredentialsProfiles; let open: Record | undefined; for (const [index, raw] of text.split("\n").entries()) { - const line = raw.replace(/[ \t]*\r?[ \t]*$/, "").replace(/^[ \t]+/, ""); + const bare = trimEnd(raw, " \t"); + const line = (bare.endsWith("\r") ? trimEnd(bare.slice(0, -1), " \t") : bare).replace( + /^[ \t]+/, + "", + ); if (line === "" || line.startsWith("#")) continue; const table = TABLE.exec(line); if (table !== null) { diff --git a/typescript/src/core/parse.ts b/typescript/src/core/parse.ts index 258278e..9e0e440 100644 --- a/typescript/src/core/parse.ts +++ b/typescript/src/core/parse.ts @@ -19,7 +19,7 @@ import type { WarningModel, } from "../wire.ts"; -/** What one part states beside its reading. */ +/** What one part states beside its reading, as `Answer.facts` and `partFacts` give it. */ export type PartFacts = { /** The part's `assumptions`, or undefined when it holds no list; `[]` states none. */ readonly assumptions: readonly Assumption[] | undefined; @@ -30,7 +30,7 @@ export type PartFacts = { readonly warnings: readonly Warning[]; }; -/** A part's facts, read afresh on each call. */ +/** A part's facts, read afresh on each call. `answer.facts[i]` is `partFacts(answer.parts[i])`. */ export function partFacts(part: Readonly>): PartFacts { const byReading = part["reading_assumptions"]; const readings = new Map(); diff --git a/typescript/src/core/text.ts b/typescript/src/core/text.ts index cd8befb..a5226b0 100644 --- a/typescript/src/core/text.ts +++ b/typescript/src/core/text.ts @@ -106,3 +106,19 @@ export function quoted(value: unknown): string { return "a value JSON cannot hold"; } } + +/** The characters a regular expression's `\s` matches. */ +export const WHITESPACE = + "\t\n\v\f\r \u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"; + +/** `text` without the trailing characters that appear in `chars`, found by one scan from the + * end. A pattern such as `/x+$/` rescans every run of `x` that is not at the end, which takes + * quadratic time. */ +export function trimEnd(text: string, chars: string): string { + let end = text.length; + while (end > 0 && chars.includes(text.charAt(end - 1))) end -= 1; + return text.slice(0, end); +} + +/** `text` without its trailing slashes. */ +export const trimTrailingSlashes = (text: string): string => trimEnd(text, "/"); diff --git a/typescript/src/datasets/cards.ts b/typescript/src/datasets/cards.ts index 61d3e1b..3105cec 100644 --- a/typescript/src/datasets/cards.ts +++ b/typescript/src/datasets/cards.ts @@ -29,7 +29,7 @@ import type { InvalidRequestError } from "../core/errors.ts"; import { isRecord } from "../core/json.ts"; import { checkOptions, declareOptions } from "../core/options.ts"; import { periodObject } from "../core/periods.ts"; -import { quoted } from "../core/text.ts"; +import { quoted, trimEnd } from "../core/text.ts"; import { DatasetRef } from "../core/ref.ts"; import { describing } from "../core/trace.ts"; import type { Transport } from "../http/transport.ts"; @@ -114,7 +114,7 @@ async function createCard( export function mintToken(prefix: string): string { const bytes = crypto.getRandomValues(new Uint8Array(16)); const base64 = btoa(String.fromCharCode(...bytes)); - return prefix + base64.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); + return prefix + trimEnd(base64.replace(/\+/g, "-").replace(/\//g, "_"), "="); } /** The ref of the one version `data` holds. */ diff --git a/typescript/src/datasets/dataset.ts b/typescript/src/datasets/dataset.ts index 5acaf43..bef5252 100644 --- a/typescript/src/datasets/dataset.ts +++ b/typescript/src/datasets/dataset.ts @@ -15,6 +15,7 @@ import type { PartFacts } from "../core/parse.ts"; import type { Assumption, CitationSource, Departure, Notice, RequestedOperation } from "../wire.ts"; import { isRecord, list, objects, text } from "../core/json.ts"; import { inline } from "../core/text.ts"; +import { clientError } from "../core/errors.ts"; /** The version of the provenance a file from `bytes()` carries under `dfo.*` keys, the one * Python's `Dataset.save` writes: 1 for a dataset of dataset versions, and 2 for a derived @@ -163,6 +164,92 @@ export function stepsText(steps: readonly DerivedStep[], source: string): string return done === "" ? `from ${source}` : `${done}, from ${source}`; } +/** + * Group totals on one party role that a fetch left out of one member's rows. A fetch leaves + * party totals out by default, so a sum over `value` counts each flow once. `partyTotals: true` + * keeps them. DFO names what the cut dropped in the version's manifest, one entry per party role + * and provenance that lost rows, and the dataset gathers those entries in member order. + */ +export type PartyTotalsLeftOut = { + /** The spec of the member whose rows lost the totals. */ + readonly specId: string; + /** The party role, by DFO's column name, such as `financingProvider`. */ + readonly role: string; + /** The dcids of the totals left out, such as `DACCountriesTotal`. */ + readonly entities: readonly string[]; + /** The dataset whose rows lost the totals, or null for rows with no provenance, or when + * `everyProvenance` is true. */ + readonly provenance: string | null; + /** Whether the totals left the rows of every dataset, as on a version DFO recorded before it + * named the provenance. */ + readonly everyProvenance: boolean; +}; + +/** The pointer of the served list of party totals left out. */ +const PARTY_TOTALS_POINTER = "/trace/party_totals_left_out"; + +/** A manifest's `trace` breaks the served shape at `pointer`. */ +const traceViolation = (pointer: string) => + clientError("contract_violation", { + detail: `The manifest's ${pointer} breaks the shape this client reads.`, + pointer, + }); + +/** + * The totals a manifest's `trace.party_totals_left_out` names for the member of `specId`, each + * `{role, entities, provenance}`, where an absent `provenance` means every dataset's rows. The + * list is empty when the trace names none. Other keys in an entry are ignored. + * + * @throws {ContractError} The trace or an entry breaks the served shape (`contract_violation`). + */ +function partyTotalsOf( + manifest: Readonly>, + specId: string, +): PartyTotalsLeftOut[] { + const trace = manifest["trace"]; + if (trace === undefined || trace === null) return []; + if (!isRecord(trace)) throw traceViolation("/trace"); + const served = trace["party_totals_left_out"]; + if (served === undefined || served === null) return []; + if (!Array.isArray(served)) throw traceViolation(PARTY_TOTALS_POINTER); + return served.map((entry: unknown, index): PartyTotalsLeftOut => { + const pointer = `${PARTY_TOTALS_POINTER}/${index}`; + if (!isRecord(entry)) throw traceViolation(pointer); + const { role, entities, provenance } = entry; + if (typeof role !== "string") throw traceViolation(`${pointer}/role`); + if (!Array.isArray(entities)) throw traceViolation(`${pointer}/entities`); + entities.forEach((entity: unknown, at) => { + if (typeof entity !== "string") throw traceViolation(`${pointer}/entities/${at}`); + }); + if (provenance !== undefined && provenance !== null && typeof provenance !== "string") { + throw traceViolation(`${pointer}/provenance`); + } + return Object.freeze({ + specId, + role, + entities: Object.freeze([...(entities as string[])]), + provenance: provenance ?? null, + everyProvenance: !Object.hasOwn(entry, "provenance"), + }); + }); +} + +/** The text form's line counting the totals left out on each role, as + * `Party totals left out: 15 on financingProvider · data.partyTotalsLeftOut`. */ +function partyTotalsLine(entries: readonly PartyTotalsLeftOut[]): string | undefined { + const byRole = new Map>(); + for (const { role, entities } of entries) { + const dcids = byRole.get(role) ?? new Set(); + for (const entity of entities) dcids.add(entity); + byRole.set(role, dcids); + } + if (byRole.size === 0) return undefined; + const counts = [...byRole] + .map(([role, dcids]) => `${dcids.size.toLocaleString("en-GB")} on ${role}`) + .join(", "); + return `Party totals left out: ${counts} · data.partyTotalsLeftOut`; +} + /** What a derived dataset adds to its member: every series its recipe left out, whether that * list is whole, its steps in the order they ran, and the role word the call named each * column by. */ @@ -346,6 +433,10 @@ export class Dataset { readonly leftOut: readonly LeftOutSeries[]; /** Whether `leftOut` lists every series the recipe dropped. */ readonly #leftOutComplete: boolean; + /** The group totals a fetch left out of each member's rows, one entry per member, party role + * and provenance that lost rows. Empty when nothing was left out, when the fetch passed + * `partyTotals: true`, and for a derived dataset. */ + readonly partyTotalsLeftOut: readonly PartyTotalsLeftOut[]; /** The recipe's steps in the order they ran, for the text form's derived line. */ readonly #steps: readonly DerivedStep[]; /** The members and options the dataset was made from, less their bytes, for a copy. */ @@ -443,6 +534,11 @@ export class Dataset { (derived?.leftOut ?? []).map((series) => new LeftOutSeries(series, roles)), ); this.#leftOutComplete = derived?.leftOutComplete ?? true; + this.partyTotalsLeftOut = Object.freeze( + members.flatMap((member) => + isDerived(member) ? [] : partyTotalsOf(member.manifest, member.spec.spec_id), + ), + ); this.#steps = Object.freeze( (derived?.steps ?? []).map((step) => Object.freeze({ ...step, by: step.by.map((column) => roles[column] ?? column) }), @@ -503,6 +599,8 @@ export class Dataset { const sources = this.citation.sources.map((source) => source.label); if (sources.length > 0) lines.push(`Source: ${sources.join("; ")}`); lines.push(...this.#derivedSummary()); + const totals = partyTotalsLine(this.partyTotalsLeftOut); + if (totals !== undefined) lines.push(totals); if (this.#inputs.options.shownBy !== undefined) { lines.push(`Shown by: ${this.#inputs.options.shownBy}`); } @@ -630,6 +728,22 @@ export class Dataset { }), "dfo.ref": JSON.stringify(this.ref), ...derived, + // A key of its own, since Python 0.1.0's loader refuses any key in `dfo.dataset` it does + // not know and reads only the `dfo.*` keys it names. + ...(this.partyTotalsLeftOut.length === 0 + ? {} + : { + "dfo.party_totals_left_out": JSON.stringify( + this.partyTotalsLeftOut.map( + ({ specId, role, entities, provenance, everyProvenance }) => ({ + spec_id: specId, + role, + entities, + ...(everyProvenance ? {} : { provenance }), + }), + ), + ), + }), }; } } diff --git a/typescript/src/datasets/fetch.ts b/typescript/src/datasets/fetch.ts index f5d795c..0440020 100644 --- a/typescript/src/datasets/fetch.ts +++ b/typescript/src/datasets/fetch.ts @@ -44,6 +44,7 @@ import { mintOptions, renderingFor, renderingMatches } from "../core/ref.ts"; import type { Layout, Rendering, Spec } from "../core/ref.ts"; import { checkStrict } from "../core/strict.ts"; import { describing } from "../core/trace.ts"; +import { trimEnd, WHITESPACE } from "../core/text.ts"; import { DEFAULT_MAX_BYTES, passedDeadline } from "../http/transport.ts"; import type { Target, Transport } from "../http/transport.ts"; import type { EntityGroup, Part } from "../wire.ts"; @@ -748,7 +749,7 @@ export async function finish( * keeps the field the refusal's pointer names and the bracket with its code and trace id. */ function withHint(error: DfoError, ...hints: readonly string[]): DfoError { const ErrorClass = error.constructor as new (options: DfoErrorOptions) => DfoError; - const hinted = (text: string) => `${text.replace(/[.\s]*$/, ".")} ${hints.join(" ")}`; + const hinted = (text: string) => `${trimEnd(text, `.${WHITESPACE}`)}. ${hints.join(" ")}`; return new ErrorClass({ ...optionsOf(error), detail: hinted(error.detail), diff --git a/typescript/src/datasets/pin.ts b/typescript/src/datasets/pin.ts index 9013d3c..0fb770e 100644 --- a/typescript/src/datasets/pin.ts +++ b/typescript/src/datasets/pin.ts @@ -19,6 +19,7 @@ import { invalid } from "./fetch.ts"; import { clientError, DfoError, messageOf, optionsOf } from "../core/errors.ts"; import type { DfoErrorOptions } from "../core/errors.ts"; import { isRecord } from "../core/json.ts"; +import { trimEnd, WHITESPACE } from "../core/text.ts"; import { isDerived } from "../core/ref.ts"; import type { Transport } from "../http/transport.ts"; @@ -51,7 +52,7 @@ function partWay(error: DfoError, pinned: readonly Member[]): DfoError { const ErrorClass = error.constructor as new (options: DfoErrorOptions) => DfoError; const named = pinned.map((member) => `${member.spec.spec_id} at ${member.contentVersion}`); const hint = `Pinned before it failed, and still pinned: ${named.join(", ")}.`; - const hinted = (text: string) => `${text.replace(/[.\s]*$/, ".")} ${hint}`; + const hinted = (text: string) => `${trimEnd(text, `.${WHITESPACE}`)}. ${hint}`; return new ErrorClass({ ...optionsOf(error), detail: hinted(error.detail), diff --git a/typescript/src/datasets/publication.ts b/typescript/src/datasets/publication.ts index 24e26e6..2b94065 100644 --- a/typescript/src/datasets/publication.ts +++ b/typescript/src/datasets/publication.ts @@ -18,7 +18,7 @@ import { clientError } from "../core/errors.ts"; import type { ContractError } from "../core/errors.ts"; import { isRecord } from "../core/json.ts"; import type { DatasetRef } from "../core/ref.ts"; -import { inline, quoted } from "../core/text.ts"; +import { inline, quoted, trimTrailingSlashes } from "../core/text.ts"; /** What a card draws: a line chart, a bar chart, a table, a headline number, or the source and * citation of its data. */ @@ -466,7 +466,7 @@ export function shareIdIn(target: unknown): string | null { if (target instanceof Share) return target.id; if (typeof target !== "string") return null; const path = pathOf(target); - const segments = path.replace(/\/+$/, "").split("/"); + const segments = trimTrailingSlashes(path).split("/"); const [page, id] = segments.slice(-2); return segments.length >= 2 && SHARE_PAGES.has(page ?? "") && SHARE_ID.test(id ?? "") ? id! diff --git a/typescript/src/http/transport.ts b/typescript/src/http/transport.ts index 72d259c..783d0ee 100644 --- a/typescript/src/http/transport.ts +++ b/typescript/src/http/transport.ts @@ -30,6 +30,7 @@ import { inline, keyFault, secretPattern, + trimTrailingSlashes, } from "../core/text.ts"; /** The public API's base URL. */ @@ -922,7 +923,7 @@ export class Transport { if (target.path === undefined) { url = new URL(target.url, this.baseUrl.origin); } else { - const base = this.baseUrl.pathname.replace(/\/+$/, ""); + const base = trimTrailingSlashes(this.baseUrl.pathname); const fault = controlFault(target.path); if (fault !== null) { throw clientError("invalid_argument", { detail: `The path holds ${fault}.` }); diff --git a/typescript/src/index.ts b/typescript/src/index.ts index a2cb5a1..362a2aa 100644 --- a/typescript/src/index.ts +++ b/typescript/src/index.ts @@ -69,9 +69,16 @@ export type { Severity, UnknownPart, UnsupportedPart, + Warning, } from "./wire.ts"; export { Dataset, LeftOutSeries } from "./datasets/dataset.ts"; -export type { Citation, DerivedMember, LeftOutStep, Member } from "./datasets/dataset.ts"; +export type { + Citation, + DerivedMember, + LeftOutStep, + Member, + PartyTotalsLeftOut, +} from "./datasets/dataset.ts"; export { Analysis } from "./datasets/analysis.ts"; export type { RunOptions } from "./datasets/analysis.ts"; export { Cards } from "./datasets/cards.ts"; @@ -116,6 +123,8 @@ export { Catalogue, Code, Entity, Match, Node } from "./search/lookup-results.ts export type { GraphNode } from "./search/lookup-results.ts"; export type { Progress, ProgressEvent, ProgressHandler } from "./datasets/operation.ts"; export type { Row, Value } from "./datasets/parquet.ts"; +export { partFacts } from "./core/parse.ts"; +export type { PartFacts } from "./core/parse.ts"; export { isKnownPart, PART_STATUSES } from "./core/parts.ts"; export { specRoles } from "./core/roles.ts"; export { DatasetRef, FormatError, isDatasetMember, isDerived } from "./core/ref.ts"; diff --git a/typescript/src/search/answer.ts b/typescript/src/search/answer.ts index 6fad07a..9ad1fc0 100644 --- a/typescript/src/search/answer.ts +++ b/typescript/src/search/answer.ts @@ -29,6 +29,7 @@ import type { import { CODE_GAP_KINDS } from "../generated/actions.ts"; import type { GAP_KINDS } from "../generated/actions.ts"; import { parsedInputOf, partFacts, warningsOf } from "../core/parse.ts"; +import type { PartFacts } from "../core/parse.ts"; import { definiteParts, isKnownPart, partSentence, partText } from "../core/parts.ts"; import { checkOptions, declareOptions } from "../core/options.ts"; import { checkStrict } from "../core/strict.ts"; @@ -239,7 +240,8 @@ export class Answer { readonly reason: DeclinedReason | (string & {}) | null; /** Search's warnings for the whole answer. */ readonly warnings: readonly Notice[]; - /** Whether Search stopped at its time limit. */ + /** Whether Search stopped at its time limit: its diagnostics say so, or a part's own no-data + * reason is `timed_out`. */ readonly timedOut: boolean; /** Search's timing, token counts and build for the response; empty when it sent none. An * answer read by `Answer.fromJSON` holds only `timed_out` and `warnings`. */ @@ -250,6 +252,7 @@ export class Answer { * body it held. */ readonly raw: Readonly>; readonly #context: AnswerContext; + #facts: readonly PartFacts[] | undefined; /** The members `fetch` builds, one per distinct spec. */ readonly #members: readonly MemberRequest[]; @@ -277,6 +280,18 @@ export class Answer { this.specs = members.map((member) => member.spec); } + /** + * What each part states beside its reading, one entry per part in the order of `parts`: its + * `assumptions`, `departures`, requested `operations` and `warnings`, and a candidates part's + * `readingAssumptions` by `spec_id`. A part whose member has the wrong type reads as absent + * (`[]`, or undefined for `assumptions` when the part holds no list), and an item that lacks a + * required member is dropped, as in the Python client's `Part` attributes. + */ + get facts(): readonly PartFacts[] { + this.#facts ??= Object.freeze(this.parts.map(partFacts)); + return this.#facts; + } + /** * Settles a part that awaits a choice and returns Search's answer to it, in one Search request. * Picking a reading needs no model call. A place choice, and any part restated beside it, asks @@ -502,7 +517,8 @@ export function answerFromBody( parts: typed, question, warnings, - timedOut: diagnostics["timed_out"] === true, + // Search can stop at its time limit on one part without setting the flag. + timedOut: diagnostics["timed_out"] === true || typed.some(partTimedOut), diagnostics, traceId, raw: body, @@ -510,6 +526,11 @@ export function answerFromBody( }); } +/** Whether a part's own no-data reason says Search stopped at its time limit. */ +function partTimedOut(part: Part): boolean { + return isKnownPart(part) && part.status === "no_data" && part.no_data.reason === "timed_out"; +} + /** A part, once it passes the checks the client makes beyond the strict fields: that it is an * object, that it has a `status`, which the strict check lets through when absent, and that each * spec's definition names it. */ diff --git a/typescript/src/search/choices.ts b/typescript/src/search/choices.ts index 047496f..12477c0 100644 --- a/typescript/src/search/choices.ts +++ b/typescript/src/search/choices.ts @@ -26,7 +26,7 @@ import { isRecord } from "../core/json.ts"; import { nearest } from "../core/nearest.ts"; import { assumptionsReadOf, operationsOf, parsedPartOf } from "../core/parse.ts"; import { isKnownPart, partText } from "../core/parts.ts"; -import { inline } from "../core/text.ts"; +import { inline, trimEnd } from "../core/text.ts"; import type { Assumption, DatasetResubmitInput, @@ -122,7 +122,7 @@ const labelFold = (label: string): string => label.trim().toLowerCase(); /** Option text as an answer prints it: on one line, trimmed, without a closing full stop. The * printout and the label match both read it, so a label copied from `String(answer)` names its * option. */ -const shownText = (text: string): string => inline(text).trim().replace(/\.+$/, ""); +const shownText = (text: string): string => trimEnd(inline(text).trim(), "."); /** An option as `String(answer)` prints it: a candidate's label, source and unit joined by * ` · `, as `Kenya's GDP, 1960 to 2024 · World Development Indicators`, its sentence standing diff --git a/typescript/test/answer.test.ts b/typescript/test/answer.test.ts index 47463a9..c8bc5d4 100644 --- a/typescript/test/answer.test.ts +++ b/typescript/test/answer.test.ts @@ -5,6 +5,7 @@ import { inspect } from "node:util"; import { parquetMetadata } from "hyparquet"; import { Answer, answerFromBody, SearchWarning } from "../src/search/answer.ts"; +import { partFacts } from "../src/core/parse.ts"; import { optionLine } from "../src/search/choices.ts"; import type { AnswerContext, Candidate, Gap, QueryOption } from "../src/search/answer.ts"; import { @@ -239,6 +240,17 @@ describe("gaps", () => { ); }); + test("a part whose own reason is timed_out marks the answer and its JSON timed out", () => { + const edited = body("answer-mixed-fetch-raises"); + (edited["diagnostics"] as Record)["timed_out"] = false; + const parts = edited["parts"] as { no_data?: { reason: string } }[]; + parts[1]!.no_data!.reason = "timed_out"; + const answer = answerFromBody(edited, offline(), null); + assert.equal(answer.timedOut, true); + const client = clientOver(offline().transport); + assert.equal(Answer.fromJSON(JSON.stringify(answer), client).timedOut, true); + }); + test("diagnostics holds the response's diagnostics, or nothing when it sent none", () => { for (const name of ["answer-mixed-fetch-raises", "answer-declined"]) { assert.deepEqual(answerOf(name).diagnostics, body(name)["diagnostics"], name); @@ -583,6 +595,16 @@ describe("fetch", () => { ' Not done: rank ("top 5")', " Fetch: answer.fetch()", ]); + assert.deepEqual(answer.facts[0], partFacts(answer.parts[0]!)); + assert.deepEqual(answer.facts[0].assumptions, [ + { kind: "declared_default", binding_ref: "how:aid", text: "aid", refs: ["A"] }, + ]); + assert.deepEqual(answer.facts[0].departures, [ + { code: "operation_not_applied", text: "Not ranked." }, + ]); + assert.deepEqual(answer.facts[0].operations, [{ verb: "rank", phrase: "top 5" }]); + assert.deepEqual(answer.facts[0].warnings, []); + assert.equal(answer.facts, answer.facts); const dataset = await answer.fetch(); assert.deepEqual(remaining(), []); assert.deepEqual(dataset.assumptions, [ diff --git a/typescript/test/dataset.test.ts b/typescript/test/dataset.test.ts index 27ee1b7..12ee6b4 100644 --- a/typescript/test/dataset.test.ts +++ b/typescript/test/dataset.test.ts @@ -16,7 +16,8 @@ import { derivedDataset } from "../src/datasets/derived.ts"; import { DatasetRef, isDerived } from "../src/core/ref.ts"; import type { DerivedRefMember } from "../src/core/ref.ts"; import type { Gap } from "../src/search/answer.ts"; -import { scenarioBytes } from "./replay.ts"; +import { replay, scenario, scenarioBytes, stepOf } from "./replay.ts"; +import { clientOver } from "../src/client.ts"; import { recordedBytes, recordedJson } from "./support.ts"; const SCENARIOS = new URL("../../contract/scenarios/", import.meta.url); @@ -841,6 +842,111 @@ describe("a dataset of dataset versions", () => { }); }); +describe("party totals left out", () => { + test("a fetch names the totals its cut left out and writes them under a key of their own", async () => { + const record = await scenario("fetch-spec-party-totals-left-out"); + const { spec, window, entities } = stepOf(record, 0, "fetch"); + assert.ok(spec); + const client = clientOver(replay(record).transport); + const dataset = await client.fetch(spec, { window, entities }); + const trace = (record.exchanges[0]!.response.json as Record)["trace"] as { + party_totals_left_out: { role: string; entities: string[] }[]; + }; + const [served] = trace.party_totals_left_out; + assert.ok(served !== undefined && !("provenance" in served)); + assert.deepEqual(dataset.partyTotalsLeftOut, [ + { + specId: dataset.members[0]!.spec.spec_id, + role: "financingProvider", + entities: served.entities, + provenance: null, + everyProvenance: true, + }, + ]); + assert.ok( + String(dataset) + .split("\n") + .includes( + ` Party totals left out: ${served.entities.length} on financingProvider · ` + + "data.partyTotalsLeftOut", + ), + ); + const metadata = await savedMetadata(dataset); + assert.deepEqual(JSON.parse(metadata["dfo.party_totals_left_out"]!), [ + { spec_id: dataset.members[0]!.spec.spec_id, ...served }, + ]); + // Python 0.1.0 refuses a `dfo.dataset` key it does not know. + assert.deepEqual(Object.keys(JSON.parse(metadata["dfo.dataset"]!) as object).toSorted(), [ + "assumptions", + "complete", + "departures", + "held_until", + "layout_version", + "member_citations", + "member_rows", + "missing", + "pinned", + "requested_operations", + "stale", + "warnings", + ]); + }); + + for (const [name, served, provenance, everyProvenance] of [ + ["a named provenance", { provenance: "provenance/OECD_DAC2a" }, "provenance/OECD_DAC2a", false], + ["rows without provenance", { provenance: null }, null, false], + ["every provenance", {}, null, true], + ] as const) { + test(`keeps ${name} through bytes()`, async () => { + const entry = { role: "financingRecipient", entities: ["AfricaTotal"], ...served }; + const data = await scenarioBytes("fetch-ref.parquet"); + const manifest = { trace: { party_totals_left_out: [entry], other: 1 } }; + const dataset = await Dataset.create([member(data, { manifest })], { layout: "long" }); + const [read] = dataset.partyTotalsLeftOut; + assert.deepEqual([read?.provenance, read?.everyProvenance], [provenance, everyProvenance]); + const metadata = await savedMetadata(dataset); + assert.deepEqual(JSON.parse(metadata["dfo.party_totals_left_out"]!), [ + { spec_id: dataset.members[0]!.spec.spec_id, ...entry }, + ]); + }); + } + + test("a dataset with none left out writes no key and no line", async () => { + const data = await scenarioBytes("fetch-ref.parquet"); + const dataset = await Dataset.create([member(data)], { layout: "long" }); + assert.deepEqual(dataset.partyTotalsLeftOut, []); + assert.ok(!String(dataset).includes("Party totals")); + assert.equal((await savedMetadata(dataset))["dfo.party_totals_left_out"], undefined); + }); + + for (const [trace, pointer] of [ + [[], "/trace"], + [{ party_totals_left_out: {} }, "/trace/party_totals_left_out"], + [{ party_totals_left_out: [[]] }, "/trace/party_totals_left_out/0"], + [{ party_totals_left_out: [{ entities: [] }] }, "/trace/party_totals_left_out/0/role"], + [ + { party_totals_left_out: [{ role: "financingProvider" }] }, + "/trace/party_totals_left_out/0/entities", + ], + [ + { party_totals_left_out: [{ role: "financingProvider", entities: [1] }] }, + "/trace/party_totals_left_out/0/entities/0", + ], + [ + { party_totals_left_out: [{ role: "financingProvider", entities: [], provenance: 2 }] }, + "/trace/party_totals_left_out/0/provenance", + ], + ] as const) { + test(`a trace broken at ${pointer} raises ContractError`, async () => { + const data = await scenarioBytes("fetch-ref.parquet"); + await assert.rejects( + Dataset.create([member(data, { manifest: { trace } })], { layout: "long" }), + { name: "ContractError", code: "contract_violation", pointer }, + ); + }); + } +}); + describe("members whose column types differ", () => { type Spec = { type: string; diff --git a/typescript/test/e2e/local-stack.local.ts b/typescript/test/e2e/local-stack.local.ts index 252653d..224a018 100644 --- a/typescript/test/e2e/local-stack.local.ts +++ b/typescript/test/e2e/local-stack.local.ts @@ -50,6 +50,8 @@ const POPULATION = { contentVersion: "cv_3f610869372f93965119e3c0221847f6", sentence: "Kenya population, 2015 to 2024", }; +// The codes a server without the seed answers a seeded reading with. +const NOT_FOUND_CODES = new Set(["spec_not_found", "snapshot_not_found"]); const PAIRING = { destination: "place" }; // A breakdown by recipient, the sum data-mcp's `get_data` reads a frame with. const BREAKDOWN = { by: ["time_period", "financingRecipient"], op: "sum" }; @@ -153,6 +155,26 @@ async function fromPython(): Promise { } } +// Stops the file once, before any step is registered, when the server lacks the seed. Only the +// not-found codes count as missing, and any other error propagates as itself. +await (async () => { + const missing: string[] = []; + for (const { specId, contentVersion } of [ODA, POPULATION]) { + try { + await client.request("GET", `/datasets/${specId}/${contentVersion}`); + } catch (error) { + if (!(error instanceof NotFoundError) || !NOT_FOUND_CODES.has(error.code)) throw error; + missing.push(`${specId} at ${contentVersion}`); + } + } + if (missing.length > 0) { + throw new Error( + `The server lacks the local-stack seed; missing readings: ${missing.join(", ")}. ` + + "Run `just local-stack seed` in dfo-apis.", + ); + } +})(); + // State the later steps read from the earlier ones. let quotient: Dataset; let chained: Dataset; diff --git a/typescript/test/fetch.test.ts b/typescript/test/fetch.test.ts index 10bb599..5d4395e 100644 --- a/typescript/test/fetch.test.ts +++ b/typescript/test/fetch.test.ts @@ -320,7 +320,7 @@ describe("arguments a fetch refuses before any request", () => { await assert.rejects(fetchDataset(transport, await storedRef(), options as FetchOptions), { name: "InvalidRequestError", code: "invalid_argument", - message: new RegExp(named.replace(/[.[\](){}]/g, "\\$&")), + message: new RegExp(named.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), }); }); } diff --git a/typescript/test/package.test.ts b/typescript/test/package.test.ts index 31eda50..1ee25cd 100644 --- a/typescript/test/package.test.ts +++ b/typescript/test/package.test.ts @@ -63,6 +63,7 @@ test("the root entry exports these values and no others", async () => { "isDfoError", "isKnownPart", "latestYear", + "partFacts", "recipe", "specRoles", ]); diff --git a/typescript/test/published-types.test.ts b/typescript/test/published-types.test.ts index 80027cb..faa07fa 100644 --- a/typescript/test/published-types.test.ts +++ b/typescript/test/published-types.test.ts @@ -25,6 +25,7 @@ import { isDerived, isDfoError, latestYear, + partFacts, recipe, SeriesLeftOutWarning, } from "@developmentfinance/client"; @@ -32,6 +33,7 @@ import type { Card, CardKind, LeftOutSeries, + PartyTotalsLeftOut, Recipe, Served, Share, @@ -46,6 +48,7 @@ import type { Node, Notice, Part, + PartFacts, ProblemIssue, Severity, Spec, @@ -81,6 +84,18 @@ export function described(answer: Answer): string { return [kind, status, kinds.join(), total ?? 0, tokens ?? 0].join(" "); } +// The facts an answer's parts state are typed from the package root. +export function stated(answer: Answer): number { + const facts: readonly PartFacts[] = answer.facts; + const first: PartFacts = partFacts(answer.parts[0]!); + const kinds: readonly string[] = (facts[0]?.assumptions ?? []).map((item) => item.kind); + const codes: readonly string[] = first.departures.map((item) => item.code); + const verbs: readonly string[] = first.operations.map((item) => item.verb); + const severities: readonly string[] = first.warnings.map((item) => item.severity); + const byReading = [...first.readingAssumptions.values()].flat().length; + return kinds.length + codes.length + verbs.length + severities.length + byReading; +} + export const known = (error: unknown): boolean => isDfoError(error); // A refusal's issues and alternatives are typed from the package root. @@ -110,8 +125,9 @@ export async function analysed(client: Client, spec: Spec): Promise { const newest = await client.fetch(top.ref, { latest: true }); const dropped: readonly LeftOutSeries[] = newest.leftOut; const kept = await client.pin(await client.fetch(spec)); + const totals: readonly PartyTotalsLeftOut[] = kept.partyTotalsLeftOut; const loud = new SeriesLeftOutWarning("series_left_out", "x").code === "series_left_out"; - return dropped.length + share.rowCount + (kept.pinned && loud ? 1 : 0); + return dropped.length + totals.length + share.rowCount + (kept.pinned && loud ? 1 : 0); } // A member's spec is a Spec or a DerivedSpec; the guards narrow it, and only a Spec fetches. diff --git a/typescript/test/reprs.test.ts b/typescript/test/reprs.test.ts index b4550f4..3727f53 100644 --- a/typescript/test/reprs.test.ts +++ b/typescript/test/reprs.test.ts @@ -534,7 +534,7 @@ const TAINTED_ROWS: [string, (taint: string) => { toString(): string }][] = [ (taint) => new RawResponse( 200, - new Headers({ "content-type": `application/json${taint.replace("\n", " ")}` }), + new Headers({ "content-type": `application/json${taint.replaceAll("\n", " ")}` }), new Uint8Array([123, 125]), ), ], diff --git a/typescript/test/text.test.ts b/typescript/test/text.test.ts new file mode 100644 index 0000000..87eb744 --- /dev/null +++ b/typescript/test/text.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { trimEnd, trimTrailingSlashes, WHITESPACE } from "../src/core/text.ts"; + +test("trimTrailingSlashes drops every trailing slash and nothing else", () => { + for (const [text, trimmed] of [ + ["", ""], + ["/", ""], + ["///", ""], + ["/api/v1", "/api/v1"], + ["/api/v1/", "/api/v1"], + ["/api//v1///", "/api//v1"], + ["//a", "//a"], + ] as const) { + assert.equal(trimTrailingSlashes(text), trimmed, JSON.stringify(text)); + } +}); + +test("trimTrailingSlashes takes linear time on a long run of slashes that does not end the text", () => { + const text = `${"/".repeat(200_000)}x`; + const start = performance.now(); + assert.equal(trimTrailingSlashes(text), text); + assert.ok(performance.now() - start < 1000); +}); + +test("trimEnd strips only the trailing characters it is given", () => { + for (const [text, chars, trimmed] of [ + ["", "=", ""], + ["abc==", "=", "abc"], + ["a=b==", "=", "a=b"], + ["...", ".", ""], + ["Done.. .", `.${WHITESPACE}`, "Done"], + ["Done \u00a0\n\ufeff", `.${WHITESPACE}`, "Done"], + ["a b", `.${WHITESPACE}`, "a b"], + ["key = 1 \t ", " \t", "key = 1"], + ["a \r", " \t", "a \r"], + ] as const) { + assert.equal(trimEnd(text, chars), trimmed, JSON.stringify([text, chars])); + } +}); + +test("WHITESPACE holds exactly the characters a regular expression's \\s matches", () => { + for (let code = 0; code < 0x10000; code += 1) { + const char = String.fromCharCode(code); + assert.equal(WHITESPACE.includes(char), /\s/.test(char), code.toString(16)); + } +}); + +test("trimEnd takes linear time on a long run that does not end the text", () => { + const text = `${"=".repeat(200_000)}x`; + const start = performance.now(); + assert.equal(trimEnd(text, "="), text); + assert.ok(performance.now() - start < 1000); +}); From 032c11ca24722af1bb586421ba2d97f1d821b404 Mon Sep 17 00:00:00 2001 From: Jorge Rivera Date: Fri, 13 Nov 2026 18:44:18 +0100 Subject: [PATCH 02/11] feat!: settle the public surface of both clients for 0.2.0 Both packages move to 0.2.0. Every breaking change is listed under Unreleased in python/CHANGELOG.md and typescript/CHANGELOG.md. - Errors: every error a public call raises is a DfoError, FormatError and BundleError included, and a damaged file given to dfo.load raises one. Calls that raised ValueError or TypeError still do. Messages end in [ trace=]. The old-form shims in Answer.choose and the TypeScript options map are gone. - Timeouts: the client's timeout bounds a whole call unless the call sets its own, and attempt_timeout (attemptTimeout) bounds one attempt. docs/settings.md in each package lists every setting. - entities= takes dcids or what lookup.resolve returns. - fetch is overloaded per target in both languages, and type tests pin the arguments each target refuses. - Datasets: Python gains to_bytes() and load(bytes); TypeScript renames bytes() to toBytes(), gains save(path) on Node, and spells licence as Python does. A cache hit says so in the repr, and saved files record dfo.question and dfo.fetched_at. - TypeScript: Answer.parts types only the members the client checks, isKnownPart checks them, QueryOption is Option, and the root exports the nine names that match Python's top level, with the rest under ./errors and ./types. - Docs: the recipes run without warnings a reader should ignore, and a nightly workflow runs every documented example against dev. Closes #6 Refs #19 --- .github/workflows/docs-examples.yml | 88 ++++++ CONTRIBUTING.md | 14 +- python/CHANGELOG.md | 17 ++ python/README.md | 22 +- python/docs/answers.md | 24 +- python/docs/cache.md | 26 +- python/docs/compute.md | 4 +- python/docs/datasets.md | 37 ++- python/docs/errors.md | 74 ++--- python/docs/keys.md | 8 +- python/docs/long-builds.md | 34 ++- python/docs/settings.md | 104 +++++++ python/docs/shares.md | 7 +- python/llms.txt | 13 +- python/pyproject.toml | 2 +- python/src/dfo/_client/analysis.py | 31 ++- python/src/dfo/_client/cli.py | 25 +- python/src/dfo/_client/client.py | 157 ++++++++--- python/src/dfo/_client/default.py | 116 ++++++-- python/src/dfo/_config/credentials.py | 32 ++- python/src/dfo/_config/keys.py | 14 +- python/src/dfo/_core/base_error.py | 94 +++++++ python/src/dfo/_core/errors.py | 139 ++++------ python/src/dfo/_core/raw_response.py | 58 ++++ python/src/dfo/_core/refs.py | 17 +- python/src/dfo/_core/timeouts.py | 45 +++ python/src/dfo/_core/types.py | 43 +-- python/src/dfo/_datasets/analysis.py | 53 ++-- python/src/dfo/_datasets/build.py | 96 ++++++- python/src/dfo/_datasets/bundle.py | 130 ++++++--- python/src/dfo/_datasets/cache.py | 98 ++++++- python/src/dfo/_datasets/cards.py | 20 +- python/src/dfo/_datasets/dataset.py | 155 +++++++++-- python/src/dfo/_datasets/fetch.py | 71 +++-- python/src/dfo/_datasets/operation.py | 32 +-- python/src/dfo/_datasets/pin.py | 15 +- python/src/dfo/_datasets/resolve.py | 66 ++++- python/src/dfo/_http/transport.py | 204 +++++++++----- python/src/dfo/_search/answer.py | 64 ++--- python/src/dfo/_search/search.py | 6 +- python/src/dfo/_sync/_client/client.py | 157 ++++++++--- python/src/dfo/_sync/_datasets/analysis.py | 51 ++-- python/src/dfo/_sync/_datasets/cards.py | 20 +- python/src/dfo/_sync/_datasets/fetch.py | 71 +++-- python/src/dfo/_sync/_datasets/operation.py | 32 +-- python/src/dfo/_sync/_datasets/pin.py | 15 +- python/src/dfo/_sync/_datasets/resolve.py | 66 ++++- python/src/dfo/_sync/_http/transport.py | 204 +++++++++----- python/src/dfo/_sync/_search/answer.py | 66 +++-- python/src/dfo/_sync/_search/search.py | 6 +- python/src/dfo/_version.py | 2 +- python/src/dfo/errors.py | 2 +- python/src/dfo/types.py | 9 +- python/tests/_async/client/test_client.py | 4 +- python/tests/_async/datasets/test_analysis.py | 4 + python/tests/_async/datasets/test_fetch.py | 88 +++++- .../tests/_async/datasets/test_operation.py | 51 +++- python/tests/_async/http/test_transport.py | 64 +++-- python/tests/_async/search/test_answer.py | 48 ++-- python/tests/_fetch_support.py | 17 +- python/tests/_sync/client/test_client.py | 4 +- python/tests/_sync/datasets/test_analysis.py | 4 + python/tests/_sync/datasets/test_fetch.py | 88 +++++- python/tests/_sync/datasets/test_operation.py | 49 +++- python/tests/_sync/http/test_transport.py | 62 ++++- python/tests/_sync/search/test_answer.py | 48 ++-- python/tests/config/test_credentials.py | 15 +- python/tests/config/test_keys.py | 9 +- python/tests/core/test_errors.py | 135 ++++++++- python/tests/core/test_types.py | 6 +- python/tests/datasets/test_build.py | 16 +- python/tests/datasets/test_bundle.py | 48 +++- python/tests/datasets/test_cache.py | 80 +++++- python/tests/datasets/test_dataset.py | 142 +++++++++- python/tests/e2e/test_journey.py | 2 +- python/tests/http/test_in_flight.py | 4 +- python/tests/test_check_docs_examples.py | 33 ++- python/tests/test_dfo.py | 4 +- python/tests/test_reprs.py | 4 +- python/tests/test_typing.py | 104 ++++++- python/uv.lock | 2 +- scripts/check_docs_examples.py | 53 +++- typescript/CHANGELOG.md | 33 ++- typescript/README.md | 107 +++++--- typescript/docs/compute.md | 21 +- typescript/docs/settings.md | 57 ++++ typescript/docs/shares.md | 9 +- typescript/docs/web-apps.md | 10 +- typescript/eslint.config.js | 12 +- typescript/package.json | 10 +- typescript/src/client.ts | 84 ++++-- typescript/src/config/keys.ts | 10 +- typescript/src/core/errors.ts | 20 +- typescript/src/core/options.ts | 22 +- typescript/src/core/parts.ts | 91 +++++- typescript/src/core/ref.ts | 23 +- typescript/src/core/strict.ts | 13 +- typescript/src/core/text.ts | 6 +- typescript/src/core/trace.ts | 5 +- typescript/src/core/unknown-part.ts | 8 +- typescript/src/datasets/analysis.ts | 9 +- typescript/src/datasets/cards.ts | 9 +- typescript/src/datasets/dataset.ts | 108 ++++++-- typescript/src/datasets/fetch.ts | 108 ++++++-- typescript/src/datasets/operation.ts | 49 ++-- typescript/src/datasets/parquet.ts | 9 +- typescript/src/datasets/pin.ts | 8 +- typescript/src/datasets/resolve.ts | 14 +- typescript/src/datasets/save-node.ts | 6 + typescript/src/datasets/save.ts | 14 + typescript/src/errors.ts | 40 +++ typescript/src/http/transport.ts | 138 +++++++--- typescript/src/index.node.ts | 6 +- typescript/src/index.ts | 164 +---------- typescript/src/search/answer.ts | 67 ++--- typescript/src/search/choices.ts | 30 +- typescript/src/search/lookup-results.ts | 40 +++ typescript/src/search/lookups.ts | 21 +- typescript/src/types.ts | 128 +++++++++ typescript/src/version.ts | 2 +- typescript/src/wire.ts | 4 +- typescript/test/answer.test.ts | 21 +- typescript/test/builds.ts | 6 +- typescript/test/client.test.ts | 95 ++++++- typescript/test/contract.test.ts | 4 +- typescript/test/dataset.test.ts | 158 +++++++++-- typescript/test/e2e/journey.e2e.ts | 12 +- typescript/test/e2e/local-stack.local.ts | 20 +- typescript/test/errors.test.ts | 43 +++ typescript/test/fetch-options.types.ts | 81 ++++++ typescript/test/fetch.test.ts | 75 ++++- typescript/test/operation.test.ts | 47 ++-- typescript/test/options.test.ts | 15 - typescript/test/package.test.ts | 258 ++++++++++++++---- typescript/test/parts.types.ts | 112 ++++++++ typescript/test/published-types.test.ts | 64 ++--- typescript/test/ref.test.ts | 5 +- typescript/test/replay.ts | 3 +- typescript/test/reprs.test.ts | 6 +- typescript/test/transport.test.ts | 67 ++--- typescript/test/wire.test.ts | 38 +++ 141 files changed, 5117 insertions(+), 1672 deletions(-) create mode 100644 .github/workflows/docs-examples.yml create mode 100644 python/docs/settings.md create mode 100644 python/src/dfo/_core/base_error.py create mode 100644 python/src/dfo/_core/raw_response.py create mode 100644 python/src/dfo/_core/timeouts.py create mode 100644 typescript/docs/settings.md create mode 100644 typescript/src/datasets/save-node.ts create mode 100644 typescript/src/datasets/save.ts create mode 100644 typescript/src/errors.ts create mode 100644 typescript/src/types.ts create mode 100644 typescript/test/fetch-options.types.ts create mode 100644 typescript/test/parts.types.ts diff --git a/.github/workflows/docs-examples.yml b/.github/workflows/docs-examples.yml new file mode 100644 index 0000000..25842b5 --- /dev/null +++ b/.github/workflows/docs-examples.yml @@ -0,0 +1,88 @@ +name: Docs examples + +# scripts/check_docs_examples.py runs every python block in README.md, +# python/README.md and python/docs/*.md, and every ts block in +# typescript/README.md and typescript/docs/*.md, against the dev API through the +# dev website's /api/v1 proxy, which needs no ID token. One job per language. The +# Markdown report lists each block's outcome, the warnings each passed Python +# block showed, and each failure; the job writes it to the run summary and +# uploads it. + +on: + schedule: + - cron: "47 5 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-examples + cancel-in-progress: false + +jobs: + docs-examples: + name: docs examples (${{ matrix.language }}) + runs-on: ubuntu-latest + # The script bounds each Python file and each TypeScript program at 1,800 s. + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + language: [python, typescript] + env: + DFO_BASE_URL: https://dfo-dev-website-staging-zms5lj4zhq-nn.a.run.app/api/v1 + steps: + - uses: actions/checkout@v7 + + # The script runs under the python project's environment for both languages. + - name: Install uv + uses: astral-sh/setup-uv@v10.0.0 + + - name: Set up Python + run: uv python install 3.12 + + - name: Install Python dependencies + working-directory: python + run: uv sync --frozen --python 3.12 + + # Reads the pnpm version from typescript/package.json's packageManager. + - name: Install pnpm + if: matrix.language == 'typescript' + uses: pnpm/action-setup@v6 + with: + package_json_file: typescript/package.json + + - name: Set up Node + if: matrix.language == 'typescript' + uses: actions/setup-node@v7 + with: + node-version: "24" + cache: pnpm + cache-dependency-path: typescript/pnpm-lock.yaml + + - name: Install TypeScript dependencies + if: matrix.language == 'typescript' + working-directory: typescript + run: pnpm install --frozen-lockfile + + - name: Run the docs examples + working-directory: python + run: >- + uv run --no-sync python ../scripts/check_docs_examples.py + --language ${{ matrix.language }} + --report "$RUNNER_TEMP/docs-check-${{ matrix.language }}.md" + + - name: Write the report to the run summary + if: ${{ !cancelled() }} + run: | + report="$RUNNER_TEMP/docs-check-${{ matrix.language }}.md" + if [ -f "$report" ]; then cat "$report" >> "$GITHUB_STEP_SUMMARY"; fi + + - name: Upload the report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: docs-check-${{ matrix.language }} + path: ${{ runner.temp }}/docs-check-${{ matrix.language }}.md + if-no-files-found: warn diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea07c40..2b90beb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ Both packages group their modules into the same seven layers. A module imports a | core | `dfo/_core/` | `src/core/` | errors and warnings, refusal mapping, the public data classes, wire parsing and the DatasetRef format | | generated | `dfo/_generated/` | `src/generated/` | literals and types the generators write from the contract | -The public names sit above every layer. In Python they are `dfo/__init__.py` and the task modules beside it, `dfo/errors.py`, `dfo/types.py`, `dfo/lookup.py` and `dfo/auth.py`. Each layer's `__init__.py` holds only a docstring. In TypeScript they are the entry points at the `src/` root (`index.ts`, `index.node.ts`, `wire.ts` and `arrow.ts`), which re-export from the layers. `_version.py` and `src/version.ts` hold the version constant and sit below every layer. The sync twin of each Python async module, under `dfo/_sync/`, sits in the layer of its source. +The public names sit above every layer. In Python they are `dfo/__init__.py` and the task modules beside it, `dfo/errors.py`, `dfo/types.py`, `dfo/lookup.py` and `dfo/auth.py`. Each layer's `__init__.py` holds only a docstring. In TypeScript they are the entry points at the `src/` root (`index.ts`, `index.node.ts`, `types.ts`, `errors.ts`, `wire.ts` and `arrow.ts`), which re-export from the layers. `_version.py` and `src/version.ts` hold the version constant and sit below every layer. The sync twin of each Python async module, under `dfo/_sync/`, sits in the layer of its source. Two checks enforce the rule, and each fails on a runtime import from a higher layer: @@ -131,7 +131,15 @@ The async modules are the source of both clients, and `scripts/gen_sync.py` deri ## Docs examples -`scripts/check_docs_examples.py` runs every `python` block in `README.md`, `python/README.md`, and `python/docs/*.md`, and every `ts` or `typescript` block in `typescript/README.md` and `typescript/docs/*.md`, against the API `DFO_BASE_URL` names. It needs the network and spends Search calls, so CI leaves it out. Run it against dev after a docs change. The dev API is private, so start a local proxy that signs each request as your gcloud account, which needs `run.routes.invoke` on the service: +`scripts/check_docs_examples.py` runs every `python` block in `README.md`, `python/README.md`, and `python/docs/*.md`, and every `ts` or `typescript` block in `typescript/README.md` and `typescript/docs/*.md`, against the API `DFO_BASE_URL` names. It needs the network and spends Search calls, so pull requests do not run it. `.github/workflows/docs-examples.yml` runs it every night, and on demand from the Actions tab, against the dev API through the dev website's `/api/v1` proxy, one job per language, and attaches the Markdown report to the run. Run it yourself against dev after a docs change. The website proxy needs no credentials: + +```bash +cd python +DFO_BASE_URL=https://dfo-dev-website-staging-zms5lj4zhq-nn.a.run.app/api/v1 \ + uv run python ../scripts/check_docs_examples.py --report docs-check.md +``` + +To reach the dev API directly, which is private, start a local proxy that signs each request as your gcloud account, which needs `run.routes.invoke` on the service: ```bash gcloud run services proxy dfo-dev-apis --region northamerica-northeast1 --project dev-fin-observatory --port 8083 @@ -145,7 +153,7 @@ DFO_BASE_URL=http://localhost:8083/api \ uv run python ../scripts/check_docs_examples.py --report docs-check.md ``` -It prints a line per block, with `file:line`, `pass`, `fail` or `skip`, and the error or the reason, then the counts. A failure names the docs line the error came from. The script exits 1 when a block fails, a process fails after its last block, or none ran. Name files, such as `python/docs/errors.md`, to check only those; a name that matches no file stops the script with exit 2, and pass `--language python` or `--language typescript` to check one client. `--list` prints each block and how it will run, offline. `--timeout` bounds each Python file and each TypeScript program, 1,800 seconds by default. `python/tests/test_check_docs_examples.py` tests the parsing and planning offline. +It prints a line per block, with `file:line`, `pass`, `fail` or `skip`, and the error or the reason, then the counts. A failure names the docs line the error came from. A Python block that passes but shows a warning prints the warning with the docs line that caused it, and the report lists each such block under Warnings, so a recipe that warns shows up without failing the run. Read each one. A warning a reader would learn to ignore means the example should change, either to a question Search answers as asked or to `dfo.fetch(spec)` after the answer is read. The script exits 1 when a block fails, a process fails after its last block, or none ran. Name files, such as `python/docs/errors.md`, to check only those. A name that matches no file stops the script with exit 2, and pass `--language python` or `--language typescript` to check one client. `--list` prints each block and how it will run, offline. `--timeout` bounds each Python file and each TypeScript program, 1,800 seconds by default. `python/tests/test_check_docs_examples.py` tests the parsing and planning offline. The Python blocks of a file run in order in one process and one namespace, as a notebook runs them, so a block can use the names an earlier block set. The TypeScript blocks of a file form programs. A block that holds an `import` statement starts a program, and each block after it without one continues it. The script joins a program's blocks into one module, so a later block sees what the earlier ones declared. Imports of `@developmentfinance/client` load the TypeScript source, so no build is needed, but Node 22.18 or later and `pnpm --dir typescript install` are. diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index 4d5ca2a..a7d9fe3 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -9,14 +9,31 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added - `data.party_totals_left_out` names the group totals a fetch left out of each member, as `dfo.types.PartyTotalsLeftOut` entries of spec id, party role, dcids and provenance, and the repr counts them. `data.save` writes them under a new `dfo.party_totals_left_out` key, which 0.1.0 ignores, and `dfo.load` reads a file without it. +- `entities=` on `fetch` and `answer.fetch` takes the `Match` and `Entity` objects `dfo.lookup.resolve` returns beside dcids, as in `entities={"destination": dfo.lookup.resolve("Nigeria")}`. A match that is not `resolved` raises `InvalidRequestError` (`invalid_argument`) before any request, printing the match and a candidate to pass instead. An awaitable in its place, such as `client.lookup.resolve(...)` on an `AsyncClient` without `await`, raises the same error and names the missing `await`. +- `docs/settings.md` lists every client option and environment variable with its default and which source wins. +- `data.to_bytes()` returns the Parquet file `data.save(path)` writes, and `dfo.load` reads those bytes as it reads a path. +- `data.from_cache` says whether a fetch read any member's file from the cache, and `data.fetched_at` when the data left the server, in UTC. The repr adds `Cached: read from the cache, fetched