Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions prototypes/app/debt-thresholds/_data/coverage.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,13 @@ IDS December 2025 (archived bulk files), WEO October 2025, projected to 2030.

## Base year 2025

IDS November 2026 (DFO API, filled from the current bulk files), WEO October 2026, projected to 2031.
IDS November 2026 (DFO API), WEO October 2026, projected to 2031.

- Countries lacking a core series: 0
- Countries without a projection: 2 (SEN, PAK)
- Countries without a capacity: 0
- Creditor-group guard, 1%: 869 of 870 country-years pass
- NIC 2017: sum of DFO creditors 199,338,848 against PPG 205,334,848
- DFO against the IDS bulk files, 1%: 4470 of 4470 values agree
- No 2015-19 commitments from a source, so the 58-country average terms stand in: BTN private, DJI private, GNB private, BGD private, CAF private, COM private, NER private, BDI private, KHM private, TCD private, SWZ private, HTI private, KGZ private, LSO private, MWI private, MLI private, MRT private, NPL private, SLE private, SLB private, VUT private

| Country | ISO3 | Debt stock | Debt service | GDP | Revenue | Projection | Capacity |
Expand Down
2,290 changes: 2,225 additions & 65 deletions prototypes/app/debt-thresholds/_data/inputs.json

Large diffs are not rendered by default.

6 changes: 2 additions & 4 deletions prototypes/app/debt-thresholds/_data/pipeline/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@
December 2025 release (actuals to 2024), and IMF WEO October 2025. The DFO holds
only the current IDS release, so this base year reads the World Bank's IDS bulk
files as the Internet Archive saved them on 3 and 4 January 2026.
- 2025 on current data: the IDS release the DFO holds, through the DFO API (the pinned readings),
filled from the current IDS bulk files for what the API does not carry (China's
official and private split, IMF repurchases after 2025, lending terms), and IMF
WEO October 2026.
- 2025 on current data: the IDS release the DFO holds, read entirely through the DFO
API (the pinned readings in `pipeline.sources.dfo_api`), and IMF WEO October 2026.

The files it writes, and the shape of each, are described in `pipeline.run`; the
creditor groups in `pipeline.creditors`; the coverage report in `pipeline.coverage`.
Expand Down
25 changes: 9 additions & 16 deletions prototypes/app/debt-thresholds/_data/pipeline/config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Constants of the method, the two base-year views and the output paths."""

from dataclasses import dataclass
from datetime import date, datetime
from datetime import date
from pathlib import Path
from typing import Literal, Self

Expand All @@ -21,34 +21,29 @@ class View:

base: int
ids: str
ids_dir: str
# Cache folder of the IDS bulk files; None when the view reads IDS through the DFO.
ids_dir: str | None
archived: bool
weo: tuple[Literal["April", "October"], int]
capacity_as_of: str
use_dfo: bool

@property
def end(self) -> int:
"""The last projected year."""
return self.base + HORIZON


def release_dir(release: str) -> str:
"""Return the cache folder name for an IDS release named "November 2026": `ids-2026-11`.

The current view's bulk files and parsed panel live in this folder, so a new
release downloads and parses afresh instead of reusing an older one's files.
"""
return f"ids-{datetime.strptime(release, '%B %Y'):%Y-%m}"
@property
def use_dfo(self) -> bool:
"""True when the view reads IDS and its base-year GDP through the DFO."""
return self.ids_dir is None


def views(today: date, *, current_ids: str) -> tuple[View, View]:
"""The 2024 view on the archived releases and the 2025 view on current data.

`today` stamps the current view's capacity vintage, so the clock is read by
the caller and not at import. `current_ids` names the IDS release the DFO
holds (`sources.dfo_api.ids_release`), so the label and the cache folder
follow the data.
holds (`sources.dfo_api.ids_release`), so the label follows the data.
"""
return (
View(
Expand All @@ -58,16 +53,14 @@ def views(today: date, *, current_ids: str) -> tuple[View, View]:
archived=True,
weo=("October", 2025),
capacity_as_of="2026-03-14",
use_dfo=False,
),
View(
base=2025,
ids=current_ids,
ids_dir=release_dir(current_ids),
ids_dir=None,
archived=False,
weo=("October", 2026),
capacity_as_of=today.isoformat(),
use_dfo=True,
),
)

Expand Down
15 changes: 2 additions & 13 deletions prototypes/app/debt-thresholds/_data/pipeline/coverage.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""The coverage report: which series each country has in each base year.

`coverage.md` lists the 58 countries by series for both base years, then the
creditor-group guard and the DFO-against-bulk check. The run fails when more than
creditor-group guard and the lending-terms fallback. The run fails when more than
`MAX_MISSING` countries lack a core series (debt stock, debt service, GDP,
revenue) in the default year.
"""
Expand Down Expand Up @@ -63,11 +63,7 @@ def render_coverage(
]
for view in views:
rows, check = coverage[view.base], checks[view.base]
source = (
"archived bulk files"
if view.archived
else "DFO API, filled from the current bulk files"
)
source = "DFO API" if view.use_dfo else "archived bulk files"
lines += [
f"## Base year {view.base}",
"",
Expand All @@ -94,13 +90,6 @@ def render_coverage(
lines += [f" - {failure}" for failure in guard["failures"][:20]]
if guard["failed"] > 20:
lines.append(f" - and {guard['failed'] - 20} more")
if check["cross"]:
cross = check["cross"]
lines.append(
f"- DFO against the IDS bulk files, 1%: {cross['compared'] - cross['differ']} "
f"of {cross['compared']} values agree"
)
lines += [f" - {example}" for example in cross["examples"]]
if check["terms_fallback"]:
lines.append(
"- No 2015-19 commitments from a source, so the 58-country average "
Expand Down
48 changes: 17 additions & 31 deletions prototypes/app/debt-thresholds/_data/pipeline/panel.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,43 +2,29 @@

from collections.abc import Mapping

from pipeline.config import TOLERANCE, View
from pipeline.config import MAX_MISSING
from pipeline.countries import ISO3
from pipeline.errors import DataError

type Series = dict[int, float]
type Panel = dict[str, dict[str, Series]] # ISO3 -> panel key -> year -> value


def merge_panels(primary: Panel, fill: Panel) -> Panel:
"""Return a panel where `primary` wins and `fill` supplies the keys and years it lacks."""
merged: Panel = {}
for iso3 in ISO3:
keys = {key: dict(series) for key, series in fill.get(iso3, {}).items()}
for key, series in primary.get(iso3, {}).items():
keys.setdefault(key, {}).update(series)
merged[iso3] = keys
return merged


def cross_check(from_dfo: Panel, bulk: Panel, view: View) -> dict:
"""Count the values where the DFO and the bulk files differ by more than 1%."""
keys = ("ppg_service", "ppg_stock", "mlat_service", "blat_service", "china_service")
compared = differ = 0
examples = []
for iso3 in ISO3:
for key in keys:
for year, amount in from_dfo[iso3].get(key, {}).items():
other = bulk[iso3].get(key, {}).get(year)
if other is None or year > view.end:
continue
compared += 1
if abs(amount - other) > TOLERANCE * max(abs(other), 1.0):
differ += 1
if len(examples) < 10:
examples.append(
f"{iso3} {key} {year}: DFO {amount:,.0f}, bulk {other:,.0f}"
)
return {"compared": compared, "differ": differ, "examples": examples}
def check_actuals(panel: Panel, base: int, *, source: str, hint: str = "") -> None:
"""Raise `DataError` unless the release's actual debt stocks end in `base`.

At most `MAX_MISSING` countries may lack a `base` stock, and none may have one
for the year after. `source` names the release in the message, and `hint`
follows it.
"""
stocks = [keys.get("ppg_stock", {}) for keys in panel.values()]
actual = sum(1 for s in stocks if base in s)
later = sum(1 for s in stocks if base + 1 in s)
if actual < len(ISO3) - MAX_MISSING or later > 0:
raise DataError(
f"{source} does not end its actuals in {base}: {actual} countries have a "
f"{base} debt stock and {later} one for {base + 1}.{hint}"
)


def amount_of(keys: Mapping[str, Series], key: str, year: int) -> float:
Expand Down
13 changes: 5 additions & 8 deletions prototypes/app/debt-thresholds/_data/pipeline/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
logging or exits. It writes, into the output folder:

- `coverage.md`: the countries by series for both base years, the creditor-group
guard and the DFO-against-bulk check. The run returns 1 when more than
guard and the lending-terms fallback. The run returns 1 when more than
`MAX_MISSING` countries lack a core series in the default year.
- `inputs.json`: an envelope whose `data` maps base year, then ISO3 code, to the
`inputs` dict `pipeline.project.project_service` takes, shaped as that module's
Expand Down Expand Up @@ -38,7 +38,7 @@
from pipeline.creditors import estimate_imf_charges, guard_groups
from pipeline.errors import DataError
from pipeline.output import write_envelope, write_text
from pipeline.panel import Series, cross_check, merge_panels
from pipeline.panel import Series
from pipeline.provenance import Ledger, citation, envelope_provenance
from pipeline.sources.dfo_api import fetch_dfo, ids_release
from pipeline.sources.ids_bulk import load_ids
Expand Down Expand Up @@ -101,13 +101,11 @@ def build_thresholds(out: Path, *, client: httpx.Client, now: datetime, ids: str
inputs: dict[str, dict[str, dict]] = {}
checks: dict[int, dict] = {}
for view in both:
panel = load_ids(view, paths=paths, client=client, ledger=ledger)
creditor_sums: dict[str, Series] = {}
cross = None
if view.use_dfo:
from_dfo, creditor_sums = fetch_dfo(view, ledger=ledger)
cross = cross_check(from_dfo, panel, view)
panel = merge_panels(from_dfo, panel)
panel, creditor_sums = fetch_dfo(view, ledger=ledger)
else:
panel = load_ids(view, paths=paths, client=client, ledger=ledger)
weo = load_weo(view, ledger=ledger)
panel = estimate_imf_charges(panel, view)
terms, fallback = lending_terms(panel)
Expand All @@ -122,7 +120,6 @@ def build_thresholds(out: Path, *, client: httpx.Client, now: datetime, ids: str
)
checks[view.base] = {
"guard": guard_groups(panel, view, creditor_sums),
"cross": cross,
"terms_fallback": fallback,
}

Expand Down
48 changes: 36 additions & 12 deletions prototypes/app/debt-thresholds/_data/pipeline/sources/dfo_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,15 @@
from pipeline.config import FIRST_FLOW_YEAR, View
from pipeline.countries import ISO3
from pipeline.errors import DataError
from pipeline.panel import Panel, Series
from pipeline.panel import Panel, Series, check_actuals
from pipeline.provenance import Ledger

log = logging.getLogger("pipeline")

PINNED_ORIGIN = {
"imf_stock": ("InternationalMonetaryFund",),
"imf_flows": ("InternationalMonetaryFund",),
"imf_principal": ("InternationalMonetaryFund",),
"imf_interest": ("InternationalMonetaryFund",),
}
DFO_KEYS = {
Expand All @@ -38,7 +39,29 @@
"stock": "ppg_stock",
"imf_stock": "imf_stock",
"imf_interest": "imf_interest",
"imf_principal": "imf_principal",
"service_private": "prvt_service",
"gdp": "gdp",
"concessional_stock": "ppg_concessional_stock",
"grant_element": "ppg_grant_element",
"rate_official": "inr_offt",
"rate_private": "inr_prvt",
"maturity_official": "mat_offt",
"maturity_private": "mat_prvt",
"grace_official": "gpa_offt",
"grace_private": "gpa_prvt",
"commitments_official": "com_offt",
"commitments_private": "com_prvt",
}
# Readings by single creditor, kept for China's part: all Chinese creditors, and the
# bilateral (official) part.
CHINA_KEYS = {
"service_by_creditor": "china_service",
"principal_by_creditor": "china_principal",
"disbursements_by_creditor": "china_disb",
"service_bilateral_by_creditor": "china_blat_service",
"principal_bilateral_by_creditor": "china_blat_principal",
"disbursements_bilateral_by_creditor": "china_blat_disb",
}
IDS_NODE = "provenance/WorldBankInternationalDebtStatistics"

Expand All @@ -60,7 +83,8 @@ def fetch_dfo(view: View, *, ledger: Ledger) -> tuple[Panel, dict[str, Series]]:
"""Return the current IDS release and WEO GDP through the DFO, one call per pinned reading.

The result is the panel and, per ISO3, the sum of debt service over single
creditors, for the guard. Each reading is recorded in `ledger`.
creditors, for the guard. Each reading is recorded in `ledger`. Raises
`DataError` when the release's actual debt stocks do not end in the base year.
"""
# Imported here so the parsing helpers load without the SDK installed.
import dfo
Expand Down Expand Up @@ -114,26 +138,26 @@ def fetch_dfo(view: View, *, ledger: Ledger) -> tuple[Panel, dict[str, Series]]:
frame["year"] = frame["time_period"].astype(int)
frame = frame[frame["iso3"].isin(ISO3) & frame["value"].notna()]

if key.endswith("_by_creditor"):
flow = key.removesuffix("_by_creditor")
if key in CHINA_KEYS:
if "InternationalMonetaryFund" in set(frame["financingProvider"]):
raise DataError(f"the IMF is among the creditors in {key}.")
if flow == "service":
if key == "service_by_creditor":
sums = frame.groupby(["iso3", "year"])["value"].sum()
for (iso3, year), total in sums.items():
creditor_sums[iso3][int(year)] = float(total)
china = frame[frame["financingProvider"] == "country/CHN"]
put(collected, _rows(china), f"china_{flow}")
put(collected, _rows(china), CHINA_KEYS[key])
elif key == "imf_flows":
for word, target in (
("Disbursement", "imf_disb"),
("PrincipalRepayment", "imf_principal"),
):
put(collected, _rows(frame[frame["stat_var"].str.contains(word)]), target)
# Repurchases come from the `imf_principal` reading, which runs to the
# end of the projection; the ones in this reading stop at the base year.
disbursed = frame[frame["stat_var"].str.contains("Disbursement")]
put(collected, _rows(disbursed), "imf_disb")
else:
put(collected, _rows(frame), DFO_KEYS[key])

return {iso3: dict(keys) for iso3, keys in collected.items()}, dict(creditor_sums)
panel = {iso3: dict(keys) for iso3, keys in collected.items()}
check_actuals(panel, view.base, source=f"the DFO's IDS {view.ids} release")
return panel, dict(creditor_sums)


def _rows(frame) -> Iterable[tuple[str, int, float]]:
Expand Down
21 changes: 10 additions & 11 deletions prototypes/app/debt-thresholds/_data/pipeline/sources/ids_bulk.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@
from pathlib import Path
from typing import TYPE_CHECKING

from pipeline.config import MAX_MISSING, Paths, View
from pipeline.config import Paths, View
from pipeline.countries import ISO3
from pipeline.errors import DataError
from pipeline.panel import Panel, Series
from pipeline.panel import Panel, Series, check_actuals
from pipeline.provenance import Ledger

if TYPE_CHECKING:
Expand Down Expand Up @@ -83,6 +83,8 @@ def load_ids(view: View, *, paths: Paths, client: httpx.Client, ledger: Ledger)
# Imported here so the parsing helpers load without the HTTP client installed.
from pipeline.sources.http import download

if view.ids_dir is None:
raise DataError(f"the {view.base} view reads IDS through the DFO, not the bulk files.")
folder = paths.cache / view.ids_dir
files = []
for name, capture in IDS_FILES.items():
Expand Down Expand Up @@ -174,12 +176,9 @@ def parse_ids_sheets(sheets: Iterable[Sheet]) -> tuple[Panel, int]:

def check_release(panel: Panel, view: View, *, paths: Paths) -> None:
"""Raise `DataError` when the files' last actual year is not the view's base year."""
stocks = [keys.get("ppg_stock", {}) for keys in panel.values()]
actual = sum(1 for s in stocks if view.base in s)
later = sum(1 for s in stocks if view.base + 1 in s)
if actual < len(ISO3) - MAX_MISSING or later > 0:
raise DataError(
f"the IDS files in {paths.cache / view.ids_dir} are not the "
f"{view.ids} release: {actual} countries have a {view.base} debt stock "
f"and {later} one for {view.base + 1}. Delete the folder to download again."
)
check_actuals(
panel,
view.base,
source=f"the {view.ids} release in {paths.cache / str(view.ids_dir)}",
hint=" Delete the folder to download again.",
)
Loading
Loading