From f42deae6a86e0dd7f2d581f1ad326fb63cd60d65 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Tue, 29 Sep 2026 22:41:44 +0200 Subject: [PATCH 1/4] =?UTF-8?q?docs(falsify):=2040-lesson=20run=20readines?= =?UTF-8?q?s=20=E2=80=94=20FALSIFIED,=20two=20hard,=20two=20soft?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claim audited: "we are ready for a new full (40-lesson) run." HARD 1 — views-models still pins VIEWS_POSTPROCESSING_PIN="1.1.1" in both launchers. A run launched now installs the build whose findability guard checks 2 of 110 artefacts and cannot see the sidecar by construction — the exact defect that made today's delivery unservable. 1.4.0 exists, is tagged, is on PyPI, and is referenced by nothing. views-models#439 carries the one-line diff. HARD 2 — a fresh launcher environment is unbuildable (views-models#516, open). xarray is unpinned in postprocessors/{un_fao,un_crafd}/requirements.txt, and views-datafactory's `xarray>=2024.1,<2026` cap permits 2025.12.0, which requires pandas>=2.1 against the platform's pandas 1.5.3. Today's pod works only because xarray was hand-pinned to 2024.3.0 on it. So the claim holds only if the run reuses that exact pod and its env survives the launcher's dry-run check. SOFT 3 — the SELECTION half of the findability guard still depends on an ordering guarantee that does not exist. #314 made the per-object half order-independent; the legs loop still resolves through pipeline-core's get_latest_file_id, which documents "newest by creation timestamp" and takes files_list[0] from an unsorted result. Held since August, so order has been favourable rather than guaranteed, and the set it indexes into grows ~120 documents per run at 40 lessons. The failure mode is a FALSE invisible-delivery on a healthy run. Stub S7 below. SOFT 4 — ADR-013 §4.6's capacity inequality (assembled-run size x safety factor <= consumer serving RAM) is an open maintainer item, and its reference figure is 28.6 GB at 36 months, already stated as larger than the serving host's RAM. 40 lessons raises it ~11%. Whether that matters depends on the run's S, which this seat does not know — recorded rather than asserted. PASSED — P4, the one I most expected to break: the new per-object query works on the live path. `_build_partner_read_store` sets store.model_path = None, and get_predictions_by_metadata injects `filters["name"]` only when model_path has a model_name, so the C-77 suppression holds for documents() as well as for latest_file_id. Verified in pipeline-core at datastore.py:423-432. Only S7 is assertable in this repo. HARD 1 and HARD 2 are other repositories' state and views-models is not in this repo's CI sibling checkout (ADR-016), so a test reaching for them would pass vacuously — a guard that cannot fire (C-102). They are recorded in the module docstring with their issue numbers instead. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ANY1CCy9Xo7zjMY4XJ69v9 --- tests/test_falsification_release_readiness.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_falsification_release_readiness.py b/tests/test_falsification_release_readiness.py index ac4a2fc..4705c6b 100644 --- a/tests/test_falsification_release_readiness.py +++ b/tests/test_falsification_release_readiness.py @@ -146,3 +146,57 @@ def test_the_publish_job_cannot_ship_untested_code(): # and are recorded in the sprint rather than as assertions: views-faoapi is blocked # downstream (#294), and #272's second question is unanswered. # ───────────────────────────────────────────────────────────────────────────── + + +# ───────────────────────────────────────────────────────────────────────────── +# Fourth audit, 2026-09-29. Claim: *"we are ready for a new full (40-lesson) run."* +# +# Verdict: FALSIFIED — two hard, two soft. Only ONE is assertable in this repo and +# it is below. The others are facts about other repositories' state, and a test +# here that reached for them would be a guard that cannot fire in CI (C-102): +# +# HARD 1 — views-models still pins VIEWS_POSTPROCESSING_PIN="1.1.1", so a run +# launched now installs the build whose guard checks 2 of 110 artefacts. +# Lives on views-models#439. Not assertable here: views-models is not in +# this repo's CI sibling checkout (ADR-016), so the assertion would pass +# vacuously. This is C-112's whole subject. +# HARD 2 — a fresh launcher environment is unbuildable (views-models#516, open): +# xarray is unpinned in the launcher requirements and datafactory's +# `>=2024.1,<2026` cap permits 2025.12.0, which needs pandas>=2.1 against +# the platform's pandas 1.5.3. Not ours and not assertable here. +# SOFT 4 — ADR-013 §4.6's capacity inequality is an open maintainer item and +# 40 lessons raises the assembled-run figure ~11%. Needs the run's S, +# which this seat does not know. Recorded, not tested. +# ───────────────────────────────────────────────────────────────────────────── + + +@pytest.mark.xfail(reason="S7: unaddressed falsification — see the audit report", strict=True) +def test_the_selection_check_does_not_depend_on_an_order_nothing_guarantees(): + """S7 (soft). The per-object half stopped depending on document order in #314. + The SELECTION half still does, through a guarantee that does not exist. + + `verify`'s legs loop calls the port's `latest_file_id`, which is pipeline-core's + `get_latest_file_id`. That function DOCUMENTS *"the file ID of the newest matching + file based on creation timestamp"* and implements `files_list[0]` over the result of + `search_files_by_metadata`, which appends only `Query.equal` per filter and never an + `order_desc`/`order_asc` (`modules/appwrite/file.py:1045-1050`). + + So the check that decides whether the consumer's selection lands on THIS run takes + an arbitrary element of every `category="forecast"` document ever written, and + compares it against this run's manifest id. It has held since August, which means + the order has been favourable rather than guaranteed — and the set it indexes into + grows by ~110 documents per run, ~120 at 40 lessons. + + The failure is a **false** `DeliveryNotFindableError` on a healthy delivery, which + is the direction this module exists to avoid. + + Fails until either the selection half is made order-independent here, or + pipeline-core's sort lands and the pin moves. Filed upstream after the #312 + re-review; the fix is not ours and the dependency is. + """ + source = (_REPO / "views_postprocessing" / "delivery" / "findability.py").read_text() + legs_loop = source[source.index("for category, expected in legs.items():"):] + assert "resolve_latest(" not in legs_loop.split("assert_all_findable")[0], ( + "the selection check still resolves through `latest_file_id`, i.e. through " + "pipeline-core's unsorted `files_list[0]`" + ) From 7c4ca881ddc8b059e636d1322a5e9330ce0794f1 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Fri, 30 Oct 2026 00:17:55 +0100 Subject: [PATCH 2/4] =?UTF-8?q?chore(creds):=20both=20platform=20keys=20ro?= =?UTF-8?q?tated=20=E2=80=94=20the=20tripwire=20goes=20green=20on=20the=20?= =?UTF-8?q?fix,=20not=20the=20escape?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C-84 predicted a total seam outage on 2026-11-17 and C-110 predicted that the tripwire guarding it would close `main` from 2026-10-18. Both happened, on the day, and `main` was gated for eleven days with nothing queued behind it. Cleared by rotating, not by acknowledging. `ACKNOWLEDGED_UNTIL` was never set. VIEWS Pipeline Core (36 scopes) -> pipeline-core-writer (6 load-bearing) UN FAO / faoapi-legacy-shared-RETIRING (36) -> faoapi-server (4, all reads) Each successor was verified live BEFORE its predecessor was stood down: appwrite_store STORE_ACTIVE (1007 files), unfao_delivery DELIVERING, 3 collections / 1006 documents, and faoapi's own smoke test ALL PASS with appwrite=True and both legs serving 1,030 cells. KEY_EXPIRY now carries 2027-10-29 for both. The dates are DECLARED (one-year expiry set at creation), not read back from the console, and the comment says so — the times are approximate and immaterial at a 30-day window. The warning text is rewritten because it described a world that no longer exists. "Neither can carry traffic while the other is replaced" was a property of the keys being SHARED across every identity on the seam. The successors serve different consumers with different scopes, so the next rotation is two independent operations; the message now says to rotate them separately, and a guard asserts that it does. The finding worth keeping: scope sets derived from source were wrong TWICE — once by the pipeline-core seat tracing every call site, once independently here. Appwrite split the vocabulary into documentsdb.* / tables.*+rows.* / vectorsdb.*, and what authorises appwrite==19.2.0 is the three scopes the console tags Deprecated. The server named them itself: 401 general_unauthorized_scope missing scopes (["collections.read"]) 401 general_unauthorized_scope missing scopes (["documents.read"]) views-faoapi's own guard had already written the rule, and their C-282 was the first instance of this exact error. Create-before-delete made the rediscovery free: the old key carried production throughout. ADRs requested at views-pipeline-core#572 and views-faoapi#543. Also recorded in C-110: shortening LEAD_DAYS from 30 to 7 was attempted during the incident and does not clear the check — measured, it reddens two other guards instead. The obvious shortcut is closed, deliberately. Residual, not chased: files.write/documents.write on pipeline-core-writer are unproven (verifying means writing to production); whether the FAO have adopted faoapi-caller-fao is unconfirmed and tracked at views-faoapi#543. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ANY1CCy9Xo7zjMY4XJ69v9 --- reports/technical_risk_register.md | 152 ++++++++++++++++++----------- tests/test_credential_expiry.py | 65 ++++++++---- 2 files changed, 143 insertions(+), 74 deletions(-) diff --git a/reports/technical_risk_register.md b/reports/technical_risk_register.md index afcddbb..6c81483 100644 --- a/reports/technical_risk_register.md +++ b/reports/technical_risk_register.md @@ -4,10 +4,10 @@ |-------------------|--------------------------------------| | Project | views-postprocessing | | Owner | Dylan Pinheiro / PRIO MD&D Team | -| Last Updated | 2026-09-14 | +| Last Updated | 2026-10-30 | | Total Concerns | 113 | -| Open Concerns | 32 | -| Resolved Concerns | 81 | +| Open Concerns | 30 | +| Resolved Concerns | 83 | --- @@ -710,37 +710,6 @@ It is a worked example wearing a risk's clothes. **Give it a real trigger or mov --- -### C-84: Every identity this repo delivers under dies on 2026-11-17, within 3h35m of the other - -| Field | Value | -|-------|-------| -| ID | C-84 | -| Tier | 2 — not silent. The delivery fails loudly and completely, which is the correct behaviour and also the whole problem: there is no degraded mode, no fallback identity, and the date is known in advance. A foreseeable total outage that nobody has scheduled work against is a structural risk, not an operational surprise. | -| Source | views-appwrite coordinate registry v1.4.3/v1.4.4 — operator console read, 2026-08-05 (þing-02 A3(i)) | -| Trigger | **A date, unusually — 2026-11-17.** The registry records `VIEWS Pipeline Core` expiring 12:35 and `UN FAO` 16:10 that afternoon. Act when the un_fao delivery is next scheduled within a month of it, or when anyone plans a rotation, whichever is first. **Since 2026-08-19 the first arm fires by itself**: `tests/test_credential_expiry.py` goes red from 2026-10-18, so the date no longer depends on anyone remembering it. | -| Owner | Simon, and only Simon — issuing and installing keys is a console action. This entry exists so the date is visible from *this* repo's planning surface rather than only from the platform's. | -| Location | `views_postprocessing/{unfao,crafd}/appwrite_env.py` — the declared coordinates; the values live in the environment and the registry, never here. | - -The FAO delivery authenticates with the `UN FAO` key. That key expires **2026-11-17 16:10**, and the platform's other key three and a half hours earlier. Read from the console rather than inferred, and recorded in the registry this repo pins. - -**Why the gap is the finding and not the dates.** Three and a half hours is not a stagger — it is close enough that the two keys cannot cover for one another under any realistic response, and both are on the same seam. After 16:10 that afternoon every identity on it is dead at once: model and ensemble writes, the un_fao delivery, the CRAF'd delivery, all preflights, and FAO's own read access. A rotation that assumes one key can carry traffic while the other is replaced has no such window. - -**What this repo can and cannot do.** It cannot rotate anything; it holds no credentials and must not (þing-01 D3). What it can do is fail early and legibly rather than mid-delivery — and it does not currently. `appwrite_env.py` validates that the declared variables are *present*, which an expired key still is. An expired key is indistinguishable from a valid one until the first request comes back unauthorised, by which point a delivery is part-way through. - -**Deliberately not fixed here, and the reason is C-84's own shape.** A preflight that checks key validity means an authenticated call at startup, and the only project to make it against is production — which **þing-01 D2** forbids for tests and this would not quite be — and which that verdict explicitly permits as *read-only preflight validation*, so the obstacle here is the authenticated call, not the prohibition (see C-95). The honest position is that this is a *date to act on*, not a mechanism to build, and inventing a mechanism would be building the wrong thing to feel busy. Registered so the date is not discovered by an outage. - -**The trigger now fires on its own (2026-08-19), and this is not the mechanism above.** The first arm of the trigger — *"act when the un_fao delivery is next scheduled within a month of it"* — was a trigger nobody could notice: it fired in someone's memory or not at all, which is the same defect that withdrew the third arm of C-94's trigger and which ADR-014 §4 exists to forbid. `tests/test_credential_expiry.py` declares the two expiries and fails from 30 days out, naming the dates, the 3h35m gap, who owns the rotation (operator; views-appwrite#12, key split views-faoapi#338), and the three ways to make it pass — rotate and update the constant, update the constant if a key was replaced early, or set `ACKNOWLEDGED_UNTIL`. **The third is the only one available to someone without console access**, which is most people who will meet this gate; omitting it here would reproduce the merge-queue-hostage outcome the acknowledgement exists to prevent. - -**It is emphatically not the key-validity preflight this entry rejected.** No authenticated call, no credential, no network — a calendar and two declared datetimes. The rejection above stands and is unaffected: what was wrong was building a mechanism to *discover* a fact already known; what was missing was making the known fact impossible to forget. **The acknowledgement is the load-bearing part, and the first draft did not have it.** `/code-review high` found two design faults that would each have ended with the test deleted. (a) A literal pin on the two datetimes made the tripwire's own prescribed remediation — *rotate, then update `KEY_EXPIRY`* — fail a second test whose message said not to adjust the constant. A guard that refuses its own documented fix is worse than no guard, and it would have landed on the one person who could not route around it. The pin is gone. (b) From 2026-10-18 the gate would have been red for **every unrelated pull request**, clearable only by an operator console action the repository cannot perform — which is precisely what `pyproject.toml` says about ruff, citing ADR-014 §3: *a gate that starts red gets switched off*. `ACKNOWLEDGED_UNTIL` is the in-repo escape: a declared, reviewed, dated edit meaning *seen, and being acted on*, which **cannot be set on or after the expiry** — so it postpones attention and can never replace it. - -Four companion tests keep it honest rather than decorative: the firing branch is exercised against a probe **derived from** `KEY_EXPIRY` (**C-102** — a guard that has never run is unproven; deriving it rather than hardcoding means the proof survives a rotation instead of quietly expiring with it); an acknowledgement past the expiry is refused; `LEAD_DAYS` is floored, because shaving a week off the warning neuters the guard while leaving it looking present; and the outage-day text is checked, since the first draft would have told an operator the keys expired *"in -3 days"* while the seam was down. All verified by mutation. - -Cross-refs: **C-81** (the same operator session's other half — branch protection and the CI token), **C-27** (no rotation mechanism for a secret value upstream), **C-57** (the pinned-registry detector, which is how this arrived here at all — it demanded the v1.4.4 bump and the bump is what surfaced the expiry), þing-02 A3(i), views-appwrite C-65 and C-66. - -**The tripwire gates the release path, which this entry did not say when it was added — see C-110.** From 2026-10-18 it reddens `test`, which `protect_main` makes a required check with zero bypass actors, so no release can be tagged until the keys rotate or `ACKNOWLEDGED_UNTIL` is set. - ---- - ### C-15: Upload metadata lacks enrichment provenance and carries test description | Field | Value | @@ -1295,29 +1264,6 @@ Cross-refs: **C-103** (twice stale in four days — the case that made this visi --- -### C-110: The release path will block itself from 2026-10-18, and nothing tells the person it blocks - -| Field | Value | -|-------|-------| -| ID | C-110 | -| Tier | 3 — the ability to ship is interrupted on a known date, but the escape is in-repo and reachable. That is what separates it from **C-86**'s Tier 2, where the only responses are console actions by one person. If the acknowledgement were ever removed, or the tripwire made unconditional, this becomes C-86's tier. | -| Source | `/falsify` release-readiness audit, 2026-08-21 (probe P5) | -| Trigger | A release is cut on or after **2026-10-18**, or anyone reports a red `test` check on `main` they cannot explain — read `tests/test_credential_expiry.py` before diagnosing anything else. | -| Owner | This repository for the documentation; the operator for the rotation that removes the cause. | -| Location | `tests/test_credential_expiry.py::test_the_platform_keys_are_not_about_to_expire`; the `protect_main` ruleset; `docs/operations/` (where the runbook that would say this does not exist). | - -Measured 2026-08-21 against the live ruleset: `protect_main` is **active**, the `test` job is a **required status check**, and `bypass_actors` is **empty** — C-86's finding, re-confirmed. `test_credential_expiry` fails from 30 days before the 2026-11-17 expiry, i.e. **2026-10-18**. From that date the required check is red, so no pull request merges to `main` and no release can be tagged. - -**This entry exists because it falls between two records and is in neither.** C-84 registers the expiry and the tripwire and does not mention the release path or required checks. C-86 registers that a red build cannot be bypassed and does not mention the tripwire. The interaction — *the guard we added will redden the check that cannot be bypassed, on a date we chose* — is the product of the two, and was not noticed when the tripwire was added four days earlier. The ruleset was never queried at the time; the audit queried it. - -**The escape is real and was verified, which is why this is Tier 3.** A pull request that sets `ACKNOWLEDGED_UNTIL` is green on its own branch (measured: `1 failed` → `6 passed`), so the block is not a trap and the fix is not gated behind the thing it fixes. What is missing is that **nothing tells a releaser any of this**. There is no release runbook in `docs/operations/`, and the failure message names `ACKNOWLEDGED_UNTIL` without saying that a release is what it is blocking. - -**Closing this is documentation, not code.** Either a release runbook that names the interaction, or a cross-reference in C-84 and C-86 so that whoever reads one meets the other. - -Cross-refs: **C-84** (the expiry and the tripwire), **C-86** (no way past a red build; zero bypass actors), **C-81** (what actually gates `main`). - ---- - ### C-112: Nothing here can see what production actually runs, and it ran a defect we had already fixed for eight days | Field | Value | @@ -1448,6 +1394,98 @@ See also C-40 (the inheritance/representation coupling this migration unwinds), ## Resolved Concerns +### C-84: Every identity this repo delivers under dies on 2026-11-17, within 3h35m of the other — RESOLVED + +| Field | Value | +|-------|-------| +| ID | C-84 | +| Tier | 2 — not silent. The delivery fails loudly and completely, which is the correct behaviour and also the whole problem: there is no degraded mode, no fallback identity, and the date is known in advance. A foreseeable total outage that nobody has scheduled work against is a structural risk, not an operational surprise. | +| Source | views-appwrite coordinate registry v1.4.3/v1.4.4 — operator console read, 2026-08-05 (þing-02 A3(i)) | +| Trigger | **A date, unusually — 2026-11-17.** The registry records `VIEWS Pipeline Core` expiring 12:35 and `UN FAO` 16:10 that afternoon. Act when the un_fao delivery is next scheduled within a month of it, or when anyone plans a rotation, whichever is first. **Since 2026-08-19 the first arm fires by itself**: `tests/test_credential_expiry.py` goes red from 2026-10-18, so the date no longer depends on anyone remembering it. | +| Owner | Simon, and only Simon — issuing and installing keys is a console action. This entry exists so the date is visible from *this* repo's planning surface rather than only from the platform's. | +| Location | `views_postprocessing/{unfao,crafd}/appwrite_env.py` — the declared coordinates; the values live in the environment and the registry, never here. | + +**RESOLVED 2026-10-29 — both keys rotated, 19 days before the date.** The tripwire fired on 2026-10-18 exactly as predicted and held `main` closed for eleven days, which is the mechanism working rather than failing: nobody had to remember. + +Both 36-scope keys were replaced with least-privilege successors, each verified live **before** its predecessor was stood down: + +| was | now | scopes | verified | +|---|---|---|---| +| `VIEWS Pipeline Core` | `pipeline-core-writer` | 6 load-bearing | `appwrite_store` STORE_ACTIVE (1007 files); `unfao_delivery` DELIVERING; 3 collections, 1006 documents | +| `UN FAO` / `faoapi-legacy-shared-RETIRING` | `faoapi-server` | 4, all reads | faoapi smoke test ALL PASS, `appwrite=True`, both legs serving 1,030 cells | + +**What this closes that the date alone would not.** The entry's framing — *"no degraded mode, no fallback identity"* — was a property of the keys being **shared**, not of their expiry. Two 36-scope keys carrying every identity on the seam could not be rotated one at a time. The successors serve different consumers with different scopes, so the next rotation is two independent, routine operations. The structural risk is gone, not merely postponed; `tests/test_credential_expiry.py` now carries 2027-10-29 and its warning text was rewritten to say so. + +**The finding worth keeping, because it will recur at the next SDK bump.** Scope sets derived from source were **wrong twice** — once by the pipeline-core seat tracing every call site, once independently here. Appwrite had split the vocabulary into `documentsdb.*` / `tables.*`/`rows.*` / `vectorsdb.*`, and what actually authorises `appwrite==19.2.0` is the three scopes the console tags **Deprecated**. The server named them itself: + +``` +401 general_unauthorized_scope missing scopes (["collections.read"]) +401 general_unauthorized_scope missing scopes (["documents.read"]) +``` + +views-faoapi's own guard had already written the rule this cost us a round trip to rediscover: *"That is Appwrite's contract, not ours, and no local test can establish it — only a live round trip can."* Their **views-faoapi C-282** was the first instance. Create-before-delete is what made the rediscovery free: the old key carried production throughout. + +ADRs requested at **views-pipeline-core#572** and **views-faoapi#543**, because a scope table that records the names without recording *why source analysis cannot produce them* will be wrong again. + +*Residual, deliberately not chased here:* `files.write` and `documents.write` on `pipeline-core-writer` are unproven — verifying them means writing to the production store. The old key remains available until a real producer run exercises them. Whether the FAO have adopted `faoapi-caller-fao`, or still hold only the retired shared key, is unconfirmed and is tracked at views-faoapi#543; the operator held a meeting and sent the key, and adoption is the partner's to complete. + +Cross-refs: **C-110** (the self-inflicted release block this resolves), **C-86** (no way past a red build), views-pipeline-core#572, views-faoapi#543, views-faoapi C-282. + +The FAO delivery authenticates with the `UN FAO` key. That key expires **2026-11-17 16:10**, and the platform's other key three and a half hours earlier. Read from the console rather than inferred, and recorded in the registry this repo pins. + +**Why the gap is the finding and not the dates.** Three and a half hours is not a stagger — it is close enough that the two keys cannot cover for one another under any realistic response, and both are on the same seam. After 16:10 that afternoon every identity on it is dead at once: model and ensemble writes, the un_fao delivery, the CRAF'd delivery, all preflights, and FAO's own read access. A rotation that assumes one key can carry traffic while the other is replaced has no such window. + +**What this repo can and cannot do.** It cannot rotate anything; it holds no credentials and must not (þing-01 D3). What it can do is fail early and legibly rather than mid-delivery — and it does not currently. `appwrite_env.py` validates that the declared variables are *present*, which an expired key still is. An expired key is indistinguishable from a valid one until the first request comes back unauthorised, by which point a delivery is part-way through. + +**Deliberately not fixed here, and the reason is C-84's own shape.** A preflight that checks key validity means an authenticated call at startup, and the only project to make it against is production — which **þing-01 D2** forbids for tests and this would not quite be — and which that verdict explicitly permits as *read-only preflight validation*, so the obstacle here is the authenticated call, not the prohibition (see C-95). The honest position is that this is a *date to act on*, not a mechanism to build, and inventing a mechanism would be building the wrong thing to feel busy. Registered so the date is not discovered by an outage. + +**The trigger now fires on its own (2026-08-19), and this is not the mechanism above.** The first arm of the trigger — *"act when the un_fao delivery is next scheduled within a month of it"* — was a trigger nobody could notice: it fired in someone's memory or not at all, which is the same defect that withdrew the third arm of C-94's trigger and which ADR-014 §4 exists to forbid. `tests/test_credential_expiry.py` declares the two expiries and fails from 30 days out, naming the dates, the 3h35m gap, who owns the rotation (operator; views-appwrite#12, key split views-faoapi#338), and the three ways to make it pass — rotate and update the constant, update the constant if a key was replaced early, or set `ACKNOWLEDGED_UNTIL`. **The third is the only one available to someone without console access**, which is most people who will meet this gate; omitting it here would reproduce the merge-queue-hostage outcome the acknowledgement exists to prevent. + +**It is emphatically not the key-validity preflight this entry rejected.** No authenticated call, no credential, no network — a calendar and two declared datetimes. The rejection above stands and is unaffected: what was wrong was building a mechanism to *discover* a fact already known; what was missing was making the known fact impossible to forget. **The acknowledgement is the load-bearing part, and the first draft did not have it.** `/code-review high` found two design faults that would each have ended with the test deleted. (a) A literal pin on the two datetimes made the tripwire's own prescribed remediation — *rotate, then update `KEY_EXPIRY`* — fail a second test whose message said not to adjust the constant. A guard that refuses its own documented fix is worse than no guard, and it would have landed on the one person who could not route around it. The pin is gone. (b) From 2026-10-18 the gate would have been red for **every unrelated pull request**, clearable only by an operator console action the repository cannot perform — which is precisely what `pyproject.toml` says about ruff, citing ADR-014 §3: *a gate that starts red gets switched off*. `ACKNOWLEDGED_UNTIL` is the in-repo escape: a declared, reviewed, dated edit meaning *seen, and being acted on*, which **cannot be set on or after the expiry** — so it postpones attention and can never replace it. + +Four companion tests keep it honest rather than decorative: the firing branch is exercised against a probe **derived from** `KEY_EXPIRY` (**C-102** — a guard that has never run is unproven; deriving it rather than hardcoding means the proof survives a rotation instead of quietly expiring with it); an acknowledgement past the expiry is refused; `LEAD_DAYS` is floored, because shaving a week off the warning neuters the guard while leaving it looking present; and the outage-day text is checked, since the first draft would have told an operator the keys expired *"in -3 days"* while the seam was down. All verified by mutation. + +Cross-refs: **C-81** (the same operator session's other half — branch protection and the CI token), **C-27** (no rotation mechanism for a secret value upstream), **C-57** (the pinned-registry detector, which is how this arrived here at all — it demanded the v1.4.4 bump and the bump is what surfaced the expiry), þing-02 A3(i), views-appwrite C-65 and C-66. + +**The tripwire gates the release path, which this entry did not say when it was added — see C-110.** From 2026-10-18 it reddens `test`, which `protect_main` makes a required check with zero bypass actors, so no release can be tagged until the keys rotate or `ACKNOWLEDGED_UNTIL` is set. + +--- + +--- + +### C-110: The release path will block itself from 2026-10-18, and nothing tells the person it blocks — RESOLVED + +| Field | Value | +|-------|-------| +| ID | C-110 | +| Tier | 3 — the ability to ship is interrupted on a known date, but the escape is in-repo and reachable. That is what separates it from **C-86**'s Tier 2, where the only responses are console actions by one person. If the acknowledgement were ever removed, or the tripwire made unconditional, this becomes C-86's tier. | +| Source | `/falsify` release-readiness audit, 2026-08-21 (probe P5) | +| Trigger | A release is cut on or after **2026-10-18**, or anyone reports a red `test` check on `main` they cannot explain — read `tests/test_credential_expiry.py` before diagnosing anything else. | +| Owner | This repository for the documentation; the operator for the rotation that removes the cause. | +| Location | `tests/test_credential_expiry.py::test_the_platform_keys_are_not_about_to_expire`; the `protect_main` ruleset; `docs/operations/` (where the runbook that would say this does not exist). | + +Measured 2026-08-21 against the live ruleset: `protect_main` is **active**, the `test` job is a **required status check**, and `bypass_actors` is **empty** — C-86's finding, re-confirmed. `test_credential_expiry` fails from 30 days before the 2026-11-17 expiry, i.e. **2026-10-18**. From that date the required check is red, so no pull request merges to `main` and no release can be tagged. + +**RESOLVED 2026-10-29 — it happened exactly as written, and the prediction is the reason it cost little.** The check went red on 2026-10-18 and `main` was closed for eleven days. Nothing was queued behind it, so the only cost was the operator's confusion on discovering it — which is precisely the gap this entry named: *"nothing tells the person it blocks"*. + +It was cleared by **remediation 1, not the escape hatch**. `ACKNOWLEDGED_UNTIL` was never set. Both keys were rotated (**C-84**) and `KEY_EXPIRY` updated to 2027-10-29, so the check passes on the underlying fact rather than on an acknowledgement. That matters for this entry specifically: the tier rested on the escape being *reachable*, and the escape turned out not to be needed. + +**One correction to this entry's own reasoning, recorded because it nearly caused harm.** During the incident an attempt was made to shorten `LEAD_DAYS` from 30 to 7 to quiet the alarm. Measured, that does not clear the check — it reddens `test_the_lead_window_is_not_quietly_shrunk` and the firing-path probe instead, so buying a shorter window costs deleting the two guards that keep the tripwire honest for the next rotation. The guard did exactly the job its message describes (*"shortening the warning is how a dated guard gets neutered while still looking present"*), and the attempt was abandoned. Worth knowing that the obvious shortcut is closed, and closed deliberately. + +**What is left undone, honestly.** The Location field named `docs/operations/` as *"where the runbook that would say this does not exist"*, and it still does not exist. What exists now instead is the rewritten failure message in `tests/test_credential_expiry.py`, which carries the create-before-delete procedure, the independence of the two keys, and the live-round-trip rule. That is arguably the better home — the person who meets this meets it as a red check, not by opening a runbook — but it is a different remedy than the one this entry proposed, and saying it was "documented as planned" would be false. + +**This entry exists because it falls between two records and is in neither.** C-84 registers the expiry and the tripwire and does not mention the release path or required checks. C-86 registers that a red build cannot be bypassed and does not mention the tripwire. The interaction — *the guard we added will redden the check that cannot be bypassed, on a date we chose* — is the product of the two, and was not noticed when the tripwire was added four days earlier. The ruleset was never queried at the time; the audit queried it. + +**The escape is real and was verified, which is why this is Tier 3.** A pull request that sets `ACKNOWLEDGED_UNTIL` is green on its own branch (measured: `1 failed` → `6 passed`), so the block is not a trap and the fix is not gated behind the thing it fixes. What is missing is that **nothing tells a releaser any of this**. There is no release runbook in `docs/operations/`, and the failure message names `ACKNOWLEDGED_UNTIL` without saying that a release is what it is blocking. + +**Closing this is documentation, not code.** Either a release runbook that names the interaction, or a cross-reference in C-84 and C-86 so that whoever reads one meets the other. + +Cross-refs: **C-84** (the expiry and the tripwire), **C-86** (no way past a red build; zero bypass actors), **C-81** (what actually gates `main`). + +--- + +--- + ### C-102: A guard that has never run is unproven, however carefully it was written — RESOLVED | Field | Value | diff --git a/tests/test_credential_expiry.py b/tests/test_credential_expiry.py index 36f9392..5b3cb12 100644 --- a/tests/test_credential_expiry.py +++ b/tests/test_credential_expiry.py @@ -35,16 +35,34 @@ import pytest #: The platform keys this repository's deliveries authenticate with, and when they die. -#: Read from the operator console 2026-08-05 (þing-02 A3(i)); recorded in views-appwrite -#: coordinate registry v1.4.4 and in register C-84. Both of this repo's paths — the FAO -#: delivery and the CRAF'd delivery — run under `VIEWS Pipeline Core`, the earlier one. #: -#: Naive datetimes, deliberately: the console reports local time and the window below is -#: 30 days, so an hour either way changes nothing. Adding tzinfo would imply a precision -#: the source does not have. +#: **2026-10-29 — both keys rotated; this is remediation 1, the one the message below +#: prefers.** The pair this entry used to name, `VIEWS Pipeline Core` and the key then +#: called `UN FAO` (latterly `faoapi-legacy-shared-RETIRING`), were 36-scope keys dying +#: within 3h35m of each other on 2026-11-17 — C-84's finding, and the reason this file +#: exists. Both have been replaced by least-privilege successors: +#: +#: - `pipeline-core-writer` — the producer key, held by views-models, +#: views-pipeline-core and this repository. Six load-bearing scopes. +#: - `faoapi-server` — the views-faoapi server-side key, for its daily monitor and +#: the Hetzner box. Four, all reads. +#: +#: Both verified live before the old keys were stood down: `appwrite_store` STORE_ACTIVE, +#: `unfao_delivery` DELIVERING, and faoapi's own smoke test ALL PASS with `appwrite=True`. +#: +#: Dates, honestly sourced: each key was created on 2026-10-29 with a **one-year** +#: expiry, so the DATE is declared rather than read back from the console. The TIMES are +#: approximate — creation order, an hour apart — and are not a console reading. That is +#: fine here and would not be fine elsewhere: the window below is 30 days, so an hour +#: either way changes nothing, and inventing a precision the source does not have is the +#: failure this comment exists to avoid. Naive datetimes for the same reason; adding +#: tzinfo would imply more than is known. +#: +#: The successor keys are documented at views-pipeline-core#572 and views-faoapi#543, +#: which ask for the ADRs that should outlive this comment. KEY_EXPIRY: dict[str, datetime] = { - "VIEWS Pipeline Core": datetime(2026, 11, 17, 12, 35), - "UN FAO": datetime(2026, 11, 17, 16, 10), + "pipeline-core-writer": datetime(2027, 10, 29, 22, 0), + "faoapi-server": datetime(2027, 10, 29, 23, 0), } #: How long before the earliest expiry this starts failing. C-84's own window, and the @@ -85,14 +103,24 @@ def expiry_warning(name: str, days_left: int) -> str: return ( f"the platform Appwrite keys expire {when}:\n{listing}\n\n" f"Both of this repository's delivery paths — un_fao and crafd — authenticate with " - f"{name!r}, the earlier one. The two keys are {gap} apart, which is not a stagger: " - "neither can carry traffic while the other is replaced, so a rotation that assumes " - "a window has none. After the later time every identity on the seam is dead at " - "once — model and ensemble writes, both partner deliveries, and FAO's own read " - "access.\n\n" + f"{name!r}, the earlier one.\n\n" + f"These two keys are {gap} apart, and unlike their predecessors that is now " + "merely a coincidence of when they were issued rather than a trap. The pair this " + "file used to name were 36-scope keys SHARED across every identity on the seam, " + "so neither could carry traffic while the other was replaced and the later death " + "took model writes, both partner deliveries and FAO's read access at once. The " + "2026-10-29 split ended that: these two serve different consumers, hold different " + "scopes, and can be rotated one at a time without the other noticing. Rotate them " + "separately — the whole point of the split was that you no longer have to do both " + "at once.\n\n" "This repository cannot rotate anything and must not hold credentials " "(þing-01 D3). Issuing and installing keys is an operator console action " - "(views-appwrite#12); the key split is views-faoapi#338.\n\n" + "(views-appwrite#12). The procedure that worked, and is worth repeating: create " + "the new key BEFORE deleting the old, install it, probe it live, and only then " + "stand the old one down — the old key is the fallback for as long as it exists. " + "No local analysis establishes Appwrite's scope contract; only a live round trip " + "does, and both predictions made from source on 2026-10-29 were wrong " + "(views-pipeline-core#572, views-faoapi#543).\n\n" "THREE ways to make this pass, in order of preference:\n" " 1. rotate the keys, then update KEY_EXPIRY to the new expiries;\n" " 2. if a key was replaced early, update KEY_EXPIRY to match;\n" @@ -141,13 +169,16 @@ def test_the_tripwire_actually_fires_inside_the_lead_window(): probe = earliest - timedelta(days=LEAD_DAYS - 7) name, days_left = days_until_earliest_expiry(probe) - assert name == "VIEWS Pipeline Core", "the earlier key is the one both paths use" + assert name == "pipeline-core-writer", "the earlier key is the one both paths use" assert 0 < days_left <= LEAD_DAYS, "the probe must sit inside the window it tests" message = expiry_warning(name, days_left) assert f"expire in {days_left} days" in message - assert "3h35m apart" in message, "the gap is C-84's finding, not the dates" - assert "not a stagger" in message + assert "1h00m apart" in message, "the gap is derived from the dates, not restated" + assert "Rotate them separately" in message, ( + "the 2026-10-29 split's whole benefit is that the keys are now independent, and " + "the operator must be told that rather than left with the old all-at-once warning" + ) assert "views-appwrite#12" in message, "the reader must be told who can act" assert "ACKNOWLEDGED_UNTIL" in message, "and how to clear it without deleting it" assert "Deleting this test" in message, ( From 55722a54c6ae02814ba8ecfc408c225045328381 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Fri, 30 Oct 2026 00:21:26 +0100 Subject: [PATCH 3/4] chore(register): collapse the duplicate rules my move script introduced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving C-84 and C-110 into Resolved Concerns appended a separator each where one already followed, leaving `---` / blank / `---`. Adjacent separator pairs went 11 -> 13; now back to 11, matching development. Cosmetic, and caught by the review's own summary pass rather than by a test — `test_register_integrity.py` does not assert separator shape. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ANY1CCy9Xo7zjMY4XJ69v9 --- reports/technical_risk_register.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/reports/technical_risk_register.md b/reports/technical_risk_register.md index 6c81483..e89bec6 100644 --- a/reports/technical_risk_register.md +++ b/reports/technical_risk_register.md @@ -1451,8 +1451,6 @@ Cross-refs: **C-81** (the same operator session's other half — branch protecti --- ---- - ### C-110: The release path will block itself from 2026-10-18, and nothing tells the person it blocks — RESOLVED | Field | Value | @@ -1484,8 +1482,6 @@ Cross-refs: **C-84** (the expiry and the tripwire), **C-86** (no way past a red --- ---- - ### C-102: A guard that has never run is unproven, however carefully it was written — RESOLVED | Field | Value | From c14de3202cb99a11cfccd33529140107e091eb79 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Fri, 30 Oct 2026 00:26:32 +0100 Subject: [PATCH 4/4] =?UTF-8?q?fix(creds):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20a=20self-contradiction=20and=20two=20pins=20on=20invented=20?= =?UTF-8?q?values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the review of 7c4ca88, all real. 1. The diff contradicted itself about the old keys. The docstring said they "were stood down"; the register said one "remains available" as the fallback for the unproven write scopes. Both were in the same change and they cannot both be true. Corrected to what actually happened: the predecessors still EXIST, nothing authenticates with them, and they were kept rather than deleted. The sharper half of that finding: if the old key is the standing fallback, it expires 2026-11-17 and this commit had just removed that date from KEY_EXPIRY — creating exactly the unguarded date this file exists to prevent. Now stated outright in both places: the retained fallback has a deadline, the tripwire deliberately does not guard it (nothing authenticates with it, so it is not this guard's subject), and that is acceptable only because the fallback is a convenience rather than a dependency — a missing write scope is fixed by a console edit, which is what fixed the two read scopes on the day. 2. `assert name == "pipeline-core-writer"` pinned an ordering the same diff admits it invented. The old `name == "VIEWS Pipeline Core"` asserted a console fact: 12:35 vs 16:10, and the earlier key was the one both delivery paths used. After the rotation the ordering is an artefact of two approximate creation times, so the assertion would fail the day someone reads the real values and finds the other key came first — a guard refusing its own documented fix, which is the shape of the literal pin this file already removed once. Dropped, with the reasoning in the docstring. `name in KEY_EXPIRY` stood in its place briefly and was removed too: `days_until_earliest_expiry` returns `min(KEY_EXPIRY, ...)`, so it cannot fail. C-102's family. An assertion that cannot fire reads as cover. 3. `"1h00m apart"` had the same defect. Under the old constants 3h35m was C-84's actual finding and worth pinning; it is now an artefact of the same invented times. Replaced with a format assertion, `\d+h\d{2}m apart`, which proves the gap still renders without pinning a number this file says it invented. Mutation-proven: deleting {gap} from the message fails it. Suite: 526 passed, 3 skipped, 41 xfailed. ruff clean. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ANY1CCy9Xo7zjMY4XJ69v9 --- reports/technical_risk_register.md | 6 +++-- tests/test_credential_expiry.py | 38 ++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/reports/technical_risk_register.md b/reports/technical_risk_register.md index e89bec6..d9ef65e 100644 --- a/reports/technical_risk_register.md +++ b/reports/technical_risk_register.md @@ -1407,7 +1407,7 @@ See also C-40 (the inheritance/representation coupling this migration unwinds), **RESOLVED 2026-10-29 — both keys rotated, 19 days before the date.** The tripwire fired on 2026-10-18 exactly as predicted and held `main` closed for eleven days, which is the mechanism working rather than failing: nobody had to remember. -Both 36-scope keys were replaced with least-privilege successors, each verified live **before** its predecessor was stood down: +Both 36-scope keys were replaced with least-privilege successors, each verified live **before** its predecessor was taken out of service: | was | now | scopes | verified | |---|---|---|---| @@ -1427,7 +1427,9 @@ views-faoapi's own guard had already written the rule this cost us a round trip ADRs requested at **views-pipeline-core#572** and **views-faoapi#543**, because a scope table that records the names without recording *why source analysis cannot produce them* will be wrong again. -*Residual, deliberately not chased here:* `files.write` and `documents.write` on `pipeline-core-writer` are unproven — verifying them means writing to the production store. The old key remains available until a real producer run exercises them. Whether the FAO have adopted `faoapi-caller-fao`, or still hold only the retired shared key, is unconfirmed and is tracked at views-faoapi#543; the operator held a meeting and sent the key, and adoption is the partner's to complete. +*Residual, deliberately not chased here:* `files.write` and `documents.write` on `pipeline-core-writer` are unproven — verifying them means writing to the production store, so the proof waits for a real producer run. + +**The predecessors still exist, unused, and that retained fallback has its own deadline: 2026-11-17.** They were kept rather than deleted so the unproven write scopes had something to fall back to. They are deliberately **not** in `KEY_EXPIRY` — that dict tracks what this repository's deliveries actually authenticate with, and nothing authenticates with them any more — so the tripwire will not warn when they die. That is a date this entry is choosing not to guard, stated rather than left implicit. It is acceptable because the fallback is a convenience, not a dependency: if a write scope proves missing, the remedy is a console edit adding it, which is what fixed the two read scopes on the day. Reverting to a 36-scope key was never the plan. Whether the FAO have adopted `faoapi-caller-fao`, or still hold only the retired shared key, is unconfirmed and is tracked at views-faoapi#543; the operator held a meeting and sent the key, and adoption is the partner's to complete. Cross-refs: **C-110** (the self-inflicted release block this resolves), **C-86** (no way past a red build), views-pipeline-core#572, views-faoapi#543, views-faoapi C-282. diff --git a/tests/test_credential_expiry.py b/tests/test_credential_expiry.py index 5b3cb12..2e49036 100644 --- a/tests/test_credential_expiry.py +++ b/tests/test_credential_expiry.py @@ -30,6 +30,7 @@ from __future__ import annotations +import re from datetime import date, datetime, timedelta import pytest @@ -47,8 +48,20 @@ #: - `faoapi-server` — the views-faoapi server-side key, for its daily monitor and #: the Hetzner box. Four, all reads. #: -#: Both verified live before the old keys were stood down: `appwrite_store` STORE_ACTIVE, -#: `unfao_delivery` DELIVERING, and faoapi's own smoke test ALL PASS with `appwrite=True`. +#: Both verified live before the old keys were taken out of service: `appwrite_store` +#: STORE_ACTIVE, `unfao_delivery` DELIVERING, and faoapi's own smoke test ALL PASS with +#: `appwrite=True`. +#: +#: **The predecessors still EXIST and are deliberately not listed here.** Nothing +#: authenticates with them any more — that is why they are gone from this dict, which +#: tracks what this repository's deliveries actually use. They were kept rather than +#: deleted because `files.write` and `documents.write` on `pipeline-core-writer` are +#: still unproven: verifying them means writing to the production store, so the proof +#: waits for a real producer run. **That retained fallback dies on 2026-11-17** with the +#: keys themselves, and this tripwire will not tell you — by design, since a date on a +#: key nothing uses is not this guard's subject. If a write scope turns out to be +#: missing after that date the remedy is a console edit adding it, not a revert; the old +#: key is a convenience with a deadline, not a dependency. #: #: Dates, honestly sourced: each key was created on 2026-10-29 with a **one-year** #: expiry, so the DATE is declared rather than read back from the console. The TIMES are @@ -164,17 +177,34 @@ def test_the_tripwire_actually_fires_inside_the_lead_window(): The probe date is DERIVED from `KEY_EXPIRY`, so this keeps proving something after a rotation. Hardcoding it — as the first draft did — meant the proof quietly expired the moment the constant was legitimately updated. + + **Nothing here pins WHICH key is earliest, and that is the same lesson a second + time.** Until 2026-10-29 this asserted `name == "VIEWS Pipeline Core"`, which was a + console fact: the two expiries were 12:35 and 16:10 and the earlier one was the key + both delivery paths used. After the rotation the ordering is an artefact — the times + above are approximate, an hour apart by creation order — so asserting an identity + derived from `min()` over them would pin an invention, and would fail the day someone + reads the real values from the console and finds the other key came first. A guard + that refuses its own documented fix is worse than no guard. """ earliest = min(KEY_EXPIRY.values()).date() probe = earliest - timedelta(days=LEAD_DAYS - 7) name, days_left = days_until_earliest_expiry(probe) - assert name == "pipeline-core-writer", "the earlier key is the one both paths use" + # No assertion on WHICH key `name` is: see the docstring. `name in KEY_EXPIRY` stood + # here briefly and was removed as decoration — `days_until_earliest_expiry` returns + # `min(KEY_EXPIRY, ...)`, so it cannot fail (C-102's family). An assertion that + # cannot fire is worse than none: it reads as cover. assert 0 < days_left <= LEAD_DAYS, "the probe must sit inside the window it tests" message = expiry_warning(name, days_left) assert f"expire in {days_left} days" in message - assert "1h00m apart" in message, "the gap is derived from the dates, not restated" + # The FORMAT, not the value. Under the old constants the gap was 3h35m and that was + # C-84's actual finding — a console fact worth pinning. It is now an artefact of two + # approximate creation times, so asserting the literal would pin a number this file + # openly says it invented, and would fail the day someone legitimately corrects the + # times from the console. That is the shape the removed literal pin had. + assert re.search(r"\d+h\d{2}m apart", message), "the gap must still render" assert "Rotate them separately" in message, ( "the 2026-10-29 split's whole benefit is that the keys are now independent, and " "the operator must be told that rather than left with the old all-at-once warning"