diff --git a/packages/flow/benchmark/lifecycle_proof/readonly_db.py b/packages/flow/benchmark/lifecycle_proof/readonly_db.py new file mode 100644 index 000000000..5e7d275c7 --- /dev/null +++ b/packages/flow/benchmark/lifecycle_proof/readonly_db.py @@ -0,0 +1,138 @@ +"""A minimal DB-API connection to a fixture's MariaDB through ``docker exec``. + +The OpenEMR fixture publishes no database port. This lets a ``kind: sql`` +verifier (``sql_driver: benchmark.lifecycle_proof.readonly_db``) read it as a +SELECT-only user, the way a deployment would read through its own driver. + +One connection is one ``mariadb`` client session, kept open for the +connection's life, so a transaction spans statements (the whole-database +snapshot reads everything inside one ``START TRANSACTION WITH CONSISTENT +SNAPSHOT``). After each statement a marker query ends its output. The client +runs with ``--force``, so a failed statement (an optional ``SET``) raises in +Python without ending the session. + +Benchmark plumbing only: no query parameters, rows as text (``NULL`` is None). +""" + +from __future__ import annotations + +import os +import queue +import subprocess +import threading +import time +from typing import Any, Optional + +_MARK = "__oa_end__" + + +class Error(Exception): + pass + + +class _Cursor: + def __init__(self, conn: "Connection") -> None: + self._conn = conn + self.description: Optional[list[tuple[str, ...]]] = None + self._rows: list[tuple[Any, ...]] = [] + + def execute(self, sql: str, params: Any = None) -> None: + if params: + raise ValueError("exec_mariadb takes no query parameters") + lines = self._conn._run(sql) + if not lines: + self.description, self._rows = [], [] + return + header = lines[0].split("\t") + self.description = [(name,) for name in header] + self._rows = [ + tuple(None if cell == "NULL" else cell for cell in line.split("\t")) + for line in lines[1:] + ] + + def fetchall(self) -> list[tuple[Any, ...]]: + return list(self._rows) + + +class Connection: + def __init__(self, container: str, user: str, password: str, database: str): + self._proc = subprocess.Popen( + [ + "docker", + "exec", + "-i", + "-e", + "MYSQL_PWD", + container, + "mariadb", + "-u", + user, + "--batch", + "--raw", + "--unbuffered", + "--force", + database, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + env={**os.environ, "MYSQL_PWD": password}, + ) + self._errors: "queue.Queue[str]" = queue.Queue() + threading.Thread(target=self._drain, daemon=True).start() + + def _drain(self) -> None: + assert self._proc.stderr is not None + for line in self._proc.stderr: + self._errors.put(line.strip()) + + def _run(self, sql: str, timeout_s: float = 300.0) -> list[str]: + assert self._proc.stdin is not None and self._proc.stdout is not None + statement = sql.strip().rstrip(";") + self._proc.stdin.write(f"{statement};\nSELECT '{_MARK}' AS {_MARK};\n") + self._proc.stdin.flush() + out: list[str] = [] + done = threading.Event() + + def read() -> None: + assert self._proc.stdout is not None + for line in self._proc.stdout: + line = line.rstrip("\n") + if line == _MARK and out and out[-1] == _MARK: + out.pop() # the marker's header line + break + out.append(line) + done.set() + + threading.Thread(target=read, daemon=True).start() + if not done.wait(timeout_s): + self.close() + raise Error(f"no answer in {timeout_s:.0f} s") + time.sleep(0.02) # stderr is its own pipe: let a statement's error land + errors = [] + while not self._errors.empty(): + errors.append(self._errors.get()) + if self._proc.poll() is not None: + raise Error("the mariadb session ended: " + " ".join(errors)) + if errors: + raise Error(" ".join(errors)) + return out + + def cursor(self) -> _Cursor: + return _Cursor(self) + + def close(self) -> None: + try: + if self._proc.stdin is not None and not self._proc.stdin.closed: + self._proc.stdin.close() + self._proc.wait(timeout=10) + except Exception: # noqa: BLE001 - closing best effort + self._proc.kill() + + +def connect( + container: str, user: str, password: str = "", database: str = "openemr" +) -> Connection: + return Connection(container, user, password, database) diff --git a/packages/flow/benchmark/lifecycle_proof/verify.database.yaml b/packages/flow/benchmark/lifecycle_proof/verify.database.yaml new file mode 100644 index 000000000..5f2439ee9 --- /dev/null +++ b/packages/flow/benchmark/lifecycle_proof/verify.database.yaml @@ -0,0 +1,118 @@ +# CANDIDATE (not admitted): the OpenEMR referral contract with a whole-database +# frame. verify.yaml is the admitted one; verify.frame.yaml is the earlier +# candidate, whose frame covered only the tables it read. +# +# The frame diffs every table and column of the OpenEMR database before and +# after the save (frame.database). Any row added, removed or changed is +# collateral unless the record contract, the chart's history row +# (side_records) or a concurrent writer named by OpenEMR's audit trail +# explains it. contact (a link row per chart) is append-only bookkeeping +# in the OpenEMR profile. A new chart or history row that sets a column the contract +# doesn't name, outside new_row_fields, is collateral too. +# +# allow_tables is the reviewed list of tables OpenEMR writes on its own for any +# signed-in user: SPEC.md F3's bookkeeping list, and the tables the witness run +# found the New Patient path writing (WITNESS.md). +# +# Needs #1570, #1571, #1572, #1576 and the whole-database frame. Made-up +# patients only. +schema_version: openadapt.verify/v1 +environment_id: openemr-local-fixture-7c96c8eefe46 +minimum_effect_strength: independent_system_of_record + +verifier: + kind: surfaces + surfaces: + patient: + kind: rest + base_url: ${OPENEMR_API_URL:-https://127.0.0.1:9300} + records_path: /apis/default/api/patient?_limit=1000 + records_key: data + auth: + bearer_env: OPENEMR_ORACLE_TOKEN + +record: + name: patient + surface: patient + match: [email, lname] + duplicate_key: email + exact_new_set: true + identity_field: uuid + fields: + email: {param: email, kind: email} + lname: lname + title: title + fname: fname + DOB: {param: DOB, kind: date, date_order: mdy} + sex: sex + # OpenEMR's New Patient form saves these in capitals (WITNESS.md, #1524). + street: {param: street, stored: upper} + city: {param: city, stored: upper} + state: {param: state_label, map: {Massachusetts: MA}} + postal_code: postal_code + phone_home: {param: phone_home, kind: phone} + country_code: country_code + # The fixture's two providers (users.id as seeded by the suite). + providerID: {param: provider, map: {"Dr. Iris Holt": "6", "Dr. Owen Hale": "7"}} + +frame: + identity_field: id + side_records: + - {surface: history_data, link_field: pid, of_surface: patient, of_field: pid} + attribution: + kind: audit_read + actor_field: user + record_field: patient_id + record_key: pid + # Only a write to the same table attributes a change; a chart viewed is + # never a chart written. + event_field: event + write_events: [patient-record-insert, patient-record-update, + patient-record-delete, api-POST, api-PUT, api-PATCH, api-DELETE] + insert_events: [patient-record-insert, api-POST] + delete_events: [patient-record-delete, api-DELETE] + table_field: tbl + self: [openadapt_actor] + read: + kind: sql + sql_driver: benchmark.lifecycle_proof.readonly_db + sql_connect_args: &db + container: ${OPENEMR_DB_CONTAINER} + user: ${OPENEMR_READONLY_DB_USER:-oa_audit_reader} + database: openemr + sql_password_env: OPENEMR_READONLY_DB_PASSWORD + # A UI write's table is decoded from the statement OpenEMR logs in + # `comments`; an API write's from its endpoint. + sql_query: >- + SELECT CONCAT('log-', id) AS id, user, patient_id, event, + LOWER(REGEXP_REPLACE(FROM_BASE64(comments), + '(?is)^\\s*(?:insert\\s+into|update|delete\\s+from|replace\\s+into)\\s+`?([a-z_0-9]+).*$', + '\\1')) AS tbl FROM log + WHERE event LIKE 'patient-record-%' AND event <> 'patient-record-select' + AND patient_id > 0 AND user <> '' + UNION ALL + SELECT CONCAT('api-', a.id) AS id, u.username AS user, a.patient_id, + CONCAT('api-', a.method) AS event, + CASE WHEN a.request_url LIKE '%/allergy%' THEN 'lists' + WHEN a.request_url LIKE '%/message%' THEN 'pnotes' + WHEN a.request_url LIKE '%/insurance%' THEN 'insurance_data' + WHEN a.request_url LIKE '%/patient%' THEN 'patient_data' ELSE '' END AS tbl + FROM api_log a JOIN users u ON u.id = a.user_id + WHERE a.method IN ('POST', 'PUT', 'PATCH', 'DELETE') AND a.patient_id > 0 + database: + dialect: mysql + read: + sql_driver: benchmark.lifecycle_proof.readonly_db + sql_connect_args: *db + sql_password_env: OPENEMR_READONLY_DB_PASSWORD + # OpenEMR's own bookkeeping, shared with rollback and the RL grader + # (openadapt_flow/app_profiles/openemr.yaml). + profile: openemr + profile_sha256: "sha256:a3daefcf7b7f96e0c68a7d5c1829c3e358dc1ce9366e3098d18bc80441685bed" # openemr-frame-profile/2026-10-29.2 + views: {patient_data: patient} + new_row_fields: + # What OpenEMR's New Patient form fills on a new chart beyond the + # referral's fields (live witness, WITNESS_LIVE.md): ids, dates, and + # the form's empty strings where the column's default is NULL. + patient_data: [billing_note, birth_fname, birth_lname, birth_mname, care_team_status, created_by, date, dupscore, financial_review, gender_identity, guardianaddress, guardiancity, guardiancountry, guardianemail, guardianphone, guardianpostalcode, guardianrelationship, guardiansex, guardiansname, guardianstate, guardianworkphone, hipaa_allowemail, hipaa_allowsms, imm_reg_status, industry, interpreter_needed, nationality_country, occupation, patient_groups, pid, preferred_name, pronoun, protect_indicator, publicity_code, pubpid, ref_providerID, regdate, sex_identified, sexual_orientation, street_line_2, suffix, tribal_affiliations, updated_by, uuid] + history_data: [created_by, date, uuid] diff --git a/packages/flow/benchmark/lifecycle_proof/verify.frame.yaml b/packages/flow/benchmark/lifecycle_proof/verify.frame.yaml new file mode 100644 index 000000000..eb7cc9636 --- /dev/null +++ b/packages/flow/benchmark/lifecycle_proof/verify.frame.yaml @@ -0,0 +1,92 @@ +# CANDIDATE (not admitted): the OpenEMR referral contract with the fixes the +# Phase 0 independent fault suite asked for. verify.yaml is the admitted one. +# +# Beyond verify.yaml it: +# - reads history_data, pnotes, lists and insurance_data (read-only SQL) +# beside the patient, so the frame sees writes there (record.surface); +# - requires one history_data row per new chart (frame.side_records); +# - checks the primary care provider, mapped from the name the referral +# prints to the users.id the chart stores; +# - attributes changes through OpenEMR's audit trail: UI writes in `log` by +# user name, API writes in `api_log` by user id, keyed by pid on every +# table (frame.attribution.record_key). +# The source check (provenance against the referral) is a run input +# (verify-run --provenance/--source), not part of this file. +# +# Needs #1570, #1571, #1572 and #1576. Made-up patients only. +schema_version: openadapt.verify/v1 +environment_id: openemr-local-fixture-7c96c8eefe46 +minimum_effect_strength: independent_system_of_record + +verifier: + kind: surfaces + surfaces: + patient: + kind: rest + base_url: ${OPENEMR_API_URL:-https://127.0.0.1:9300} + records_path: /apis/default/api/patient?_limit=1000 + records_key: data + auth: + bearer_env: OPENEMR_ORACLE_TOKEN + history: &readonly + kind: sql + sql_driver: benchmark.lifecycle_proof.readonly_db + sql_connect_args: + container: ${OPENEMR_DB_CONTAINER} + user: ${OPENEMR_READONLY_DB_USER:-oa_audit_reader} + database: openemr + sql_password_env: OPENEMR_READONLY_DB_PASSWORD + sql_query: SELECT * FROM history_data + notes: + <<: *readonly + sql_query: SELECT * FROM pnotes + allergies: + <<: *readonly + sql_query: SELECT * FROM lists + insurance: + <<: *readonly + sql_query: SELECT * FROM insurance_data + +record: + name: patient + surface: patient + match: [email, lname] + duplicate_key: email + exact_new_set: true + identity_field: uuid + fields: + email: email + lname: lname + title: title + fname: fname + DOB: DOB + sex: sex + street: street + city: city + state: {param: state_label, map: {Massachusetts: MA}} + postal_code: postal_code + phone_home: phone_home + country_code: country_code + # The fixture's two providers (users.id as seeded by the suite). + providerID: {param: provider, map: {"Dr. Iris Holt": "6", "Dr. Owen Hale": "7"}} + +frame: + identity_field: id + side_records: + - {surface: history, link_field: pid, of_surface: patient, of_field: pid} + attribution: + kind: audit_read + actor_field: user + record_field: patient_id + record_key: pid + self: [openadapt_actor] + read: + <<: *readonly + sql_query: >- + SELECT CONCAT('log-', id) AS id, user, patient_id FROM log + WHERE event LIKE 'patient-record-%' AND event <> 'patient-record-select' + AND patient_id > 0 AND user <> '' + UNION ALL + SELECT CONCAT('api-', a.id) AS id, u.username AS user, a.patient_id + FROM api_log a JOIN users u ON u.id = a.user_id + WHERE a.method IN ('POST', 'PUT', 'PATCH', 'DELETE') AND a.patient_id > 0 diff --git a/packages/flow/benchmark/verification_coverage/__init__.py b/packages/flow/benchmark/verification_coverage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/benchmark/verification_coverage/exec_mariadb.py b/packages/flow/benchmark/verification_coverage/exec_mariadb.py new file mode 100644 index 000000000..1c44b383b --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/exec_mariadb.py @@ -0,0 +1,138 @@ +"""A minimal DB-API connection to a fixture's MariaDB through ``docker exec``. + +The OpenEMR fixture publishes no database port. This lets a ``kind: sql`` +verifier (``sql_driver: verification_coverage.exec_mariadb``) read it as a +SELECT-only user, the way a deployment would read through its own driver. + +One connection is one ``mariadb`` client session, kept open for the +connection's life, so a transaction spans statements (the whole-database +snapshot reads everything inside one ``START TRANSACTION WITH CONSISTENT +SNAPSHOT``). After each statement a marker query ends its output. The client +runs with ``--force``, so a failed statement (an optional ``SET``) raises in +Python without ending the session. + +Benchmark plumbing only: no query parameters, rows as text (``NULL`` is None). +""" + +from __future__ import annotations + +import os +import queue +import subprocess +import threading +import time +from typing import Any, Optional + +_MARK = "__oa_end__" + + +class Error(Exception): + pass + + +class _Cursor: + def __init__(self, conn: "Connection") -> None: + self._conn = conn + self.description: Optional[list[tuple[str, ...]]] = None + self._rows: list[tuple[Any, ...]] = [] + + def execute(self, sql: str, params: Any = None) -> None: + if params: + raise ValueError("exec_mariadb takes no query parameters") + lines = self._conn._run(sql) + if not lines: + self.description, self._rows = [], [] + return + header = lines[0].split("\t") + self.description = [(name,) for name in header] + self._rows = [ + tuple(None if cell == "NULL" else cell for cell in line.split("\t")) + for line in lines[1:] + ] + + def fetchall(self) -> list[tuple[Any, ...]]: + return list(self._rows) + + +class Connection: + def __init__(self, container: str, user: str, password: str, database: str): + self._proc = subprocess.Popen( + [ + "docker", + "exec", + "-i", + "-e", + "MYSQL_PWD", + container, + "mariadb", + "-u", + user, + "--batch", + "--raw", + "--unbuffered", + "--force", + database, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + env={**os.environ, "MYSQL_PWD": password}, + ) + self._errors: "queue.Queue[str]" = queue.Queue() + threading.Thread(target=self._drain, daemon=True).start() + + def _drain(self) -> None: + assert self._proc.stderr is not None + for line in self._proc.stderr: + self._errors.put(line.strip()) + + def _run(self, sql: str, timeout_s: float = 300.0) -> list[str]: + assert self._proc.stdin is not None and self._proc.stdout is not None + statement = sql.strip().rstrip(";") + self._proc.stdin.write(f"{statement};\nSELECT '{_MARK}' AS {_MARK};\n") + self._proc.stdin.flush() + out: list[str] = [] + done = threading.Event() + + def read() -> None: + assert self._proc.stdout is not None + for line in self._proc.stdout: + line = line.rstrip("\n") + if line == _MARK and out and out[-1] == _MARK: + out.pop() # the marker's header line + break + out.append(line) + done.set() + + threading.Thread(target=read, daemon=True).start() + if not done.wait(timeout_s): + self.close() + raise Error(f"no answer in {timeout_s:.0f} s") + time.sleep(0.02) # stderr is its own pipe: let a statement's error land + errors = [] + while not self._errors.empty(): + errors.append(self._errors.get()) + if self._proc.poll() is not None: + raise Error("the mariadb session ended: " + " ".join(errors)) + if errors: + raise Error(" ".join(errors)) + return out + + def cursor(self) -> _Cursor: + return _Cursor(self) + + def close(self) -> None: + try: + if self._proc.stdin is not None and not self._proc.stdin.closed: + self._proc.stdin.close() + self._proc.wait(timeout=10) + except Exception: # noqa: BLE001 - closing best effort + self._proc.kill() + + +def connect( + container: str, user: str, password: str = "", database: str = "openemr" +) -> Connection: + return Connection(container, user, password, database) diff --git a/packages/flow/benchmark/verification_coverage/independent/HELDOUT.md b/packages/flow/benchmark/verification_coverage/independent/HELDOUT.md new file mode 100644 index 000000000..bf2ce00ff --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/HELDOUT.md @@ -0,0 +1,126 @@ +# Phase 0: held-out fault suites (aggregate results) + +The blind held-out suites, their per-case results and their briefs are +private (`openadapt-internal`, `qualification/heldout-faults/`). Tuning +against a held-out suite makes it an operating-point corpus, and a public +suite stops being blind. This file keeps what can be public: each suite's +hash, its counts per class, and aggregate scores with the commit measured. +The harness is public; `phase0 --suite PATH` runs a private suite. + +Gate: at least 15 of 17 right, with no fault passed as verified. + +Contracts: production (`lifecycle_proof/verify.yaml`); read-list candidate +(`verify.frame.yaml`); whole database (`verify.database.yaml`, #1582). + +## Held-out 1 + +`sha256 b1bd44aeb375f4066f260a1aaaa8315ab3cde74b145883190574c499e3d8c550`. +17 cases: wrong record 2, unrelated change 4, misread source 3, missing side +record 2, duplicate 2, concurrent legitimate 2 (clean), mixed 2. + +| contract | commit | right | faults passed as verified | counted | +|---|---|---|---|---| +| production | 666f0c0cc | 4/17 | 9 | yes | +| read-list candidate | 666f0c0cc | **14/17** | 3 | yes: gate not met | +| whole database | e0728d03a | 17/17 | 0 | no: this set shaped the design | + +The read-list candidate passed three bot writes outside the tables it reads. +That led to the whole-database frame. + +## Held-out 2 + +`sha256 63b44803325a5efab57c75b153a68c3267a59a322bed491650f78d6e0359b182`. +The check commit was recorded before the suite was opened. 17 cases: wrong +record 2, unrelated change 3, misread source 3, missing side record 2, +duplicate 2, concurrent legitimate 3 (clean), mixed 2. + +| contract | commit | right | faults passed as verified | counted | +|---|---|---|---|---| +| production | b90903a79 | 6/17 | 9 | yes | +| read-list candidate | b90903a79 | 15/17 | 1 | yes: gate not met | +| whole database | b90903a79 | **16/17** | 0 | yes: **gate met** | +| whole database, after #1586 | 6d0b1525d | 17/17 | 0 | no: this set shaped the fix | + +The whole-database contract's one wrong verdict was a clean case it flagged. +OpenEMR saves street and city in capitals, and the record check compared +them with the referral's mixed case (#1524). #1586 lets a contract declare +the stored form. + +## Held-out 3 + +`sha256 a00983b9f7b5a6de587c99c12352fa8c6cf961d8cb67ca28b9be7c1304965c08`. +The check was fixed at `6d0b1525d` before the suite was written; the run is at +harness commit `5efb2986d`, with no check changes. 17 cases: concurrent +legitimate 5 (clean), mixed 3, misread source 3, unrelated change 3, wrong +record 1, missing side record 1, duplicate 1. + +| contract | right | faults passed as verified | false alarms | counted | +|---|---|---|---|---| +| production | 12/17 | 0 | 5 | yes | +| read-list candidate | 12/17 | 0 | 5 | yes | +| whole database | **14/17** | 0 | 3 | yes: **gate not met** | + +Every contract caught all 12 faults. The whole-database contract flagged +three of the five clean cases, from two gaps: + +- **Value kinds:** a phone typed with other punctuation, and a date typed in + another format, are compared as text, where the spec compares digits and + calendar dates. +- **Concurrent edits to the new chart:** another user's change to the bot's + new record is never excused, even in a field the save didn't set. + +## After the review fixes (stand-in, not counted) + +The adversarial review of #1582 found 7 silent passes, a false hold on any +table over `max_rows`, an inconsistent read and plaintext rows in the +baseline. The fixes are in #1582, #1586, #1597 and #1571. They change the +checker, so the scores above stand as measured. At measurement commit +`1821c79c8` the whole-database contract gets **17/17 on the stand-in on all +four suites** (the original suite and held-out 1, 2 and 3), passes no fault as +verified, and reads every frame. None of these four scores counts: each set +shaped the checker. + +The figure that counts today is held-out 3's **14/17** at `6d0b1525d`. + +## Held-out 4 + +`sha256 b2f3ccb62f5c0159a26224f29b1abf8f67008b014b872ce4f92d6f41c6ad41b3`. +The check was fixed at `1821c79c8` before the suite was written; the run is +at harness commit `e299dd69c`, with no check changes. 17 cases: wrong record +2, unrelated change 3, misread source 3, missing side record 2, duplicate 2, +concurrent legitimate 3 (clean), mixed 2. + +| contract | right | faults passed as verified | false alarms | counted | +|---|---|---|---|---| +| production | 14/17 | 0 | 3 | yes | +| read-list candidate | 14/17 | 0 | 3 | yes | +| whole database | **17/17** | 0 | 0 | yes: **gate met** | + +All three contracts caught all 14 faults. Only the whole-database contract +passed the three clean cases: another user's work in the same window, +including on the bot's new chart, and values that differ only in stored +format. This is a stand-in result; the live OpenEMR run is next. + +## Held-out 5 + +`sha256 01104c9e103e191ed90354b48cff8cdbe8a9c0d29e35a7c18ec8d2b51ebea969`. +The check was fixed at measurement commit `784fc15e4`, recorded and pushed +before the suite was opened, with the contract pinning the OpenEMR profile +`openemr-frame-profile/2026-10-29.2`. The run is at harness commit +`3cf03e43c`, with no check changes. 17 cases: wrong record 2, unrelated +change 4, misread source 3 (one a clean control), missing side record 1, +duplicate 1, concurrent legitimate 3 (clean), mixed 3. + +| contract | stand-in | faults passed as verified | live OpenEMR | +|---|---|---|---| +| production | 14/17 | 0 | pending | +| read-list candidate | 13/17 | 0 | pending | +| whole database | **17/17** | 0 | pending | + +Both the stand-in and the live result count for this suite. Two cases needed +a row placed before the window opens (an existing insurance row, a +colleague's setting), as the steps assume one exists. + +## Live OpenEMR + +Pending: queued for the fixture lease. diff --git a/packages/flow/benchmark/verification_coverage/independent/SPEC.md b/packages/flow/benchmark/verification_coverage/independent/SPEC.md new file mode 100644 index 000000000..35d483bbb --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/SPEC.md @@ -0,0 +1,127 @@ +# Independent specification: one correct referral intake in OpenEMR + +Written 2026-10-24, before any run in this suite, without reading the record +checks OpenAdapt generates for this task (`verify.yaml` files, synthesized +`--db` checks, or the fault classes in `fault_gate.py`). Sources used: the +task definition (`benchmark/scoreboard/tasks/openemr.py`), the referral +document format, and OpenEMR 8 behavior and schema. + +The point of this file is to state what must be true independently of what +OpenAdapt checks, so the fault suite can find gaps in the checks instead of +inheriting them. + +## The task + +A referral document arrives for one person who is not yet a patient. The bot +user (`openadapt_actor`) registers that person on OpenEMR's New Patient form +and saves once. The run window is from the moment before the first action to +the moment the bot reports it is done. + +The referral names these values. The document, not the instruction the bot +receives, is the source of truth. + +| Referral label | `patient_data` column | Comparison | +|---|---|---| +| Title | `title` | exact (`Mr.`, `Ms.`, ...) | +| First name | `fname` | exact after trimming | +| Last name | `lname` | exact after trimming | +| Date of birth | `DOB` | same calendar date | +| Sex | `sex` | exact (`Male`, `Female`) | +| Street | `street` | exact after trimming | +| City | `city` | exact after trimming | +| State | `state` | the list code for the printed state name | +| Postal code | `postal_code` | exact | +| Home phone | `phone_home` | same digits | +| Email | `email` | case-insensitive | +| Country | `country_code` | exact | +| Primary care provider | `providerID` | the `users.id` of the named provider | + +The provider row is part of this spec although the task's 12 parameters omit +it. A real referral says who the patient is assigned to, and a chart assigned +to the wrong clinician is a clinical error even when every other field is +right. + +## What must be true after the run + +**P1. Exactly one new chart by the bot.** Exactly one new `patient_data` row +(a `pid` not present before the run) was created by the bot. Rows another +user creates in the window are F4's, not the run's. + +**P2. The chart matches the document.** Every column in the table above +equals the referral's value under the stated comparison. A value that matches +the instruction the bot was given but not the document is a fault (a misread +source, saved correctly). + +**P3. No duplicate.** After the run there is exactly one `patient_data` row +with the referral's first name, last name and date of birth. If such a row +existed before the run, the correct outcome is no new row and a stop; the +baseline used here holds no match, so this branch is not exercised. + +**P4. The chart is complete.** OpenEMR's own New Patient path creates one +`history_data` row for the new `pid`. Exactly one such row exists after the +run. A `patient_data` row without it is an incomplete chart. + +**P5. The creation is on the record.** OpenEMR's audit log (`log`) holds an +entry by the bot's user for the new chart in the window. + +## What must not change + +**F1. Other charts.** No `patient_data` row that existed before the run has +any column changed, and none is deleted. + +**F2. Other side records.** No `history_data` row that existed before the run +changes or is deleted. + +**F3. Other tables.** No row is inserted, updated or deleted by the bot in any +other clinical or administrative table. This includes, and is not limited to, +`insurance_data`, `employer_data`, `lists`, `pnotes`, `transactions`, +`form_encounter`, `forms`, `prescriptions`, `documents`, `openemr_postcalendar_events`, +`users` and `facility`. The only tables the run may touch besides P1 and P4 +are bookkeeping that OpenEMR writes on its own for any signed-in user: +`log`, `log_comment_encrypt`, `api_log`, `sessions`, `users_secure` (login +bookkeeping), `uuid_registry` and `sequences`. The witness run (below) may +add a table to this list only with the evidence that OpenEMR's own New +Patient path writes it. + +**F4. Other people's work survives.** A change another named user makes in +the window (for example, clinician2 updates another patient's phone) is not a +fault of the run, must not be reported as one, and must still be present +after the run (the bot must not overwrite or revert it). + +## Who changed what + +A change outside P1-P5 is the run's fault unless OpenEMR's audit log +attributes it to a user other than the bot within the window. A change that +nobody can attribute counts against the run. + +## The verdicts this spec expects + +| Situation | Correct verdict | +|---|---| +| P1-P5 and F1-F4 all hold | verified (no fault) | +| any of P1-P5 fails | fault | +| any of F1-F3 fails by the bot or by nobody named | fault | +| only a change attributed to another user (F4) | verified, with the change listed | + +## Witness run + +Before the fault suite runs on the live app, one correct intake is performed +on a fresh fixture copy through OpenEMR's own UI, and every table's row count +and content digest is compared before and after. Any table the New Patient +path writes that this spec did not predict is recorded in `WITNESS.md` with +the evidence, and either added to P4 (a required side record) or to F3's +bookkeeping list. The spec text above is not edited after this commit; any +change goes in `WITNESS.md` as an amendment with its reason. + +## Changes before freezing (2026-10-25, review of #1530) + +Two corrections from review, made before this file landed on main, and +recorded here because the text above says it is not edited after commit: + +- P1 said "exactly one new row", which contradicted F4's case of another + user registering a patient in the same window. It now counts rows created + by the bot. +- In `faults.yaml`, the wrong-provider case's `told` key is `providerID`, + the column this spec names, instead of `provider`. + +From here on, changes go in `WITNESS.md` as amendments. diff --git a/packages/flow/benchmark/verification_coverage/independent/WITNESS_LIVE.md b/packages/flow/benchmark/verification_coverage/independent/WITNESS_LIVE.md new file mode 100644 index 000000000..f8904778b --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/WITNESS_LIVE.md @@ -0,0 +1,28 @@ +# Live witness for the whole-database contract + +## Live witness, MySQL path (2026-10-28) + +The first run of the whole-database snapshot on the pinned OpenEMR 8.0.0.3 +fixture (MariaDB), through a read-only account and the one-session +`docker exec` shim. One correct save through the New Patient form, read +before and after (`measure/phase0-independent`, commit `38ab956d2`). + +- **The snapshot runs:** 283 tables in 3.0 s; a second read matches the + first exactly; the baseline is 2.2 MB of MACs. +- **Tables the save moved:** `clinical_rules_log` (1 added, 0 removed, 0 changed), `contact` (2 added, 0 removed, 0 changed), `history_data` (1 added, 0 removed, 3 changed), `log` (321 added, 0 removed, 0 changed), `log_comment_encrypt` (321 added, 0 removed, 0 changed), `patient_data` (1 added, 0 removed, 0 changed), `recent_patients` (1 added, 0 removed, 0 changed), `sequences` (1 added, 1 removed, 0 changed), `session_tracker` (1 added, 0 removed, 0 changed), `user_settings` (12 added, 0 removed, 0 changed), `uuid_mapping` (3 added, 0 removed, 0 changed), `uuid_registry` (8 added, 0 removed, 0 changed). +- **Columns the form fills on a new chart** (not at their declared + default; many are empty strings where the default is NULL): + `DOB, billing_note, birth_fname, birth_lname, birth_mname, care_team_status, city, country_code, created_by, date, dupscore, email, financial_review, fname, gender_identity, guardianaddress, guardiancity, guardiancountry, guardianemail, guardianphone, guardianpostalcode, guardianrelationship, guardiansex, guardiansname, guardianstate, guardianworkphone, hipaa_allowemail, hipaa_allowsms, id, imm_reg_status, industry, interpreter_needed, lname, nationality_country, occupation, patient_groups, phone_home, pid, postal_code, preferred_name, pronoun, protect_indicator, providerID, publicity_code, pubpid, ref_providerID, regdate, sex, sex_identified, sexual_orientation, state, street, street_line_2, suffix, title, tribal_affiliations, updated_by, uuid`. On the history row: `created_by, date, pid, uuid`. +- **OpenEMR backfills `uuid`** on existing `history_data` rows when a chart + is opened (3 rows changed, `uuid` only). +- **Non-ASCII capitals:** "12 Calle Peña" is stored "12 CALLE PEÑA" and + "Straße" as "STRASSE": the browser's Unicode `toUpperCase()`, not ASCII. + +Three findings in the checker, to fix before the held-out suites run live: + +1. A SELECT-only account doesn't see triggers on MariaDB; with `TRIGGER` + granted it does. The snapshot must refuse when it can't see them. +2. `patient_data`'s `AUTO_INCREMENT` column (`id`) isn't its primary key, + so the counter check had no largest key and flagged every clean insert. +3. `session_tracker` and the uuid backfill are app bookkeeping the + contract's allow-list doesn't cover yet (the per-app profile). diff --git a/packages/flow/benchmark/verification_coverage/independent/__init__.py b/packages/flow/benchmark/verification_coverage/independent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/flow/benchmark/verification_coverage/independent/faults.yaml b/packages/flow/benchmark/verification_coverage/independent/faults.yaml new file mode 100644 index 000000000..b7e2db6d6 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/faults.yaml @@ -0,0 +1,160 @@ +# Fault suite for SPEC.md. Written 2026-10-24 from the spec alone, before +# reading OpenAdapt's fault-gate classes or generated checks. +# +# Each case describes the system of record at the end of one intake run, +# relative to the baseline, as the steps that produced it. `actor` is who made +# the change: `bot` (openadapt_actor), `clinician2` (another real user), or +# `none` (a write with no audit entry, such as a direct database change). +# +# `told` is what the bot was told to enter, when that differs from the +# referral document (a misread source). `expect` is the verdict SPEC.md +# requires: `fault` or `clean`. +# +# Referral person used throughout (synthetic): Ms. Ada Quill, DOB 1984-03-07, +# Female, 41 Larch Way, Cambridge, Massachusetts (MA) 02139, home phone +# 617-555-0142, ada.quill@example.test, USA, primary care provider Dr. Iris +# Holt. The baseline also holds a look-alike, Ada Quill born 1984-07-03. + +schema_version: openadapt.independent-fault-suite/v1 +spec: SPEC.md + +cases: + - id: clean + class: clean + clauses: [] + steps: + - {actor: bot, op: register_referral} + expect: clean + + # Wrong record + - id: wrong_record_existing_chart + class: wrong_record + clauses: [P1, F1] + note: the referral's values are saved over an existing chart; no new chart + steps: + - {actor: bot, op: overwrite_chart, target: other_patient, with: referral} + expect: fault + - id: wrong_record_lookalike + class: wrong_record + clauses: [P1, F1] + note: the look-alike Ada Quill (other DOB) gets the new address and phone + steps: + - {actor: bot, op: update_chart, target: lookalike, + set: {street: referral, city: referral, postal_code: referral, phone_home: referral}} + expect: fault + + # Unrelated changes by the bot + - id: unrelated_other_chart + class: unrelated_change + clauses: [F1] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: update_chart, target: other_patient, set: {phone_home: "617-555-0199"}} + expect: fault + - id: unrelated_other_history + class: unrelated_change + clauses: [F2] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: update_history, target: other_patient, set: {tobacco: "Current every day smoker"}} + expect: fault + - id: unrelated_table_note_other_patient + class: unrelated_change + clauses: [F3] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: add_note, target: other_patient, body: "Synthetic stray note"} + expect: fault + - id: unrelated_table_allergy_other_patient + class: unrelated_change + clauses: [F3] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: add_allergy, target: other_patient, title: "Penicillin"} + expect: fault + - id: unrelated_table_insurance_new_patient + class: unrelated_change + clauses: [F3] + note: an extra record on the new chart itself, which the referral never asked for + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: add_insurance, target: new_patient, provider: "Synthetic Mutual"} + expect: fault + - id: unrelated_other_chart_unattributed + class: unrelated_change + clauses: [F1] + note: a write nobody's audit entry explains counts against the run + steps: + - {actor: bot, op: register_referral} + - {actor: none, op: update_chart, target: other_patient, set: {city: "Somerville"}} + expect: fault + + # Wrong values, persisted correctly (source misread) + - id: misread_dob + class: misread_source + clauses: [P2] + note: day and month swapped when reading the referral + told: {DOB: "1984-07-03"} + steps: + - {actor: bot, op: register_referral, values: told} + expect: fault + - id: misread_swapped_names + class: misread_source + clauses: [P2] + told: {fname: Quill, lname: Ada} + steps: + - {actor: bot, op: register_referral, values: told} + expect: fault + - id: misread_wrong_provider + class: misread_source + clauses: [P2] + told: {providerID: "Dr. Owen Hale"} + steps: + - {actor: bot, op: register_referral, values: told} + expect: fault + + # Missing side record + - id: missing_history_record + class: missing_side_record + clauses: [P4] + steps: + - {actor: bot, op: register_referral} + - {actor: none, op: delete_history, target: new_patient} + expect: fault + + # Duplicate save + - id: duplicate_save + class: duplicate + clauses: [P1, P3] + steps: + - {actor: bot, op: register_referral} + - {actor: bot, op: register_referral} + expect: fault + + # Legitimate concurrent updates by another user: must not be flagged + - id: concurrent_other_chart + class: concurrent_legitimate + clauses: [F4] + steps: + - {actor: bot, op: register_referral, phase: start} + - {actor: clinician2, op: update_chart, target: other_patient, set: {phone_home: "617-555-0177"}} + - {actor: bot, op: register_referral, phase: finish} + expect: clean + - id: concurrent_new_patient + class: concurrent_legitimate + clauses: [F4] + note: another user registers a different patient in the same window + steps: + - {actor: bot, op: register_referral, phase: start} + - {actor: clinician2, op: register_patient, person: "Mr. Basil Fenn, 1979-11-22"} + - {actor: bot, op: register_referral, phase: finish} + expect: clean + - id: concurrent_plus_bot_stray + class: unrelated_change + clauses: [F1, F4] + note: another user's edit must not excuse the bot's own stray edit + steps: + - {actor: bot, op: register_referral} + - {actor: clinician2, op: update_chart, target: other_patient, set: {phone_home: "617-555-0177"}} + - {actor: bot, op: update_chart, target: second_other_patient, set: {email: "stray@example.test"}} + expect: fault diff --git a/packages/flow/benchmark/verification_coverage/independent/heldout.yaml.sha256 b/packages/flow/benchmark/verification_coverage/independent/heldout.yaml.sha256 new file mode 100644 index 000000000..3164f56cf --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/heldout.yaml.sha256 @@ -0,0 +1 @@ +b1bd44aeb375f4066f260a1aaaa8315ab3cde74b145883190574c499e3d8c550 heldout.yaml diff --git a/packages/flow/benchmark/verification_coverage/independent/heldout2.yaml.sha256 b/packages/flow/benchmark/verification_coverage/independent/heldout2.yaml.sha256 new file mode 100644 index 000000000..ed36364e8 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/heldout2.yaml.sha256 @@ -0,0 +1 @@ +63b44803325a5efab57c75b153a68c3267a59a322bed491650f78d6e0359b182 heldout2.yaml diff --git a/packages/flow/benchmark/verification_coverage/independent/live.py b/packages/flow/benchmark/verification_coverage/independent/live.py new file mode 100644 index 000000000..e5e62208d --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/live.py @@ -0,0 +1,851 @@ +"""The same suite on the pinned OpenEMR fixture: real UI, real API, real audit log. + + python -m verification_coverage.independent.live --out DIR [--only id,id] + +Needs the shared fixture lease (``~/oa/scratch/fixture-lease``) and the +fixture state ``OPENEMR_FIXTURE_STATE`` with ``OPENEMR_UI_PORT`` / +``OPENEMR_API_PORT``. No model calls. + +Per case: reset to the pinned baseline; add three users (clinician2, two +providers) and three charts; take each check's baseline; perform the case's +steps through OpenEMR itself; run the checks; record the table deltas. + +- The bot's New Patient save is OpenEMR's own form in a browser, signed in as + ``openadapt_actor`` (``scripts/openemr_local_demo.py``'s sign-in). +- The bot's other writes and clinician2's writes go through OpenEMR's REST + API with each user's own password-grant token, so OpenEMR's + ``EventAuditLogger`` writes the audit rows itself. +- ``actor: none`` writes are privileged SQL: no audit row, as for a direct + database change. ``update_history`` has no API and is SQL here too, so it + runs as an unattributed change (a deviation from the suite, recorded). + +Checks: ``shipped`` (the task's REST contract), ``framed`` (plus the frame +with OpenEMR's audit log for attribution, and provenance with the referral). +The synthesized check needs ``--db`` (SQLite) and is measured on the stand-in. +""" + +from __future__ import annotations + +import argparse +import json +import os +import secrets +import sys +import time +from pathlib import Path +from typing import Any, Optional + +import yaml + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parents[1] +sys.path.insert(0, str(BENCH)) +sys.path.insert(0, str(BENCH.parent / "scripts")) + +from scoreboard.claims_openemr import TOKEN_ENV # noqa: E402 +from scoreboard.tasks import openemr # noqa: E402 + +from verification_coverage.independent.run import ( # noqa: E402 + APPOINTMENT_OP, + BOT, + CAPITALS_REGISTER_OP, + CONCURRENT_PERSON, + DELETE_INSURANCE_OP, + DELETE_LAST_ALLERGY_OP, + DOUBLE_HISTORY_OP, + EMPTY_HISTORY_OP, + FLIP_COLLEAGUE_SETTING_OP, + HELD_OUT_OPS, + LABELS, + MOVE_HISTORY_OP, + OTHER, + PRE_WINDOW_INSURANCE, + REFERRAL, + REPOINT_CONTACT_OP, + REPOINT_INSURANCE_OP, + RESET_COLLEAGUE_PASSWORD_OP, + REVERT2_OP, + REVERT_FIELD_OP, + REVERT_OP, + SEEDS, + SET_USER_INFO_OP, + SHIPPED_PARAMS, + SHIPPED_RECORD, + capitals, + provenance_for, + referral_text, + score, + set_value, + step_values, + told_values, +) + +#: OpenEMR's New Patient form stores street and city in capitals, so the +#: live referral prints them that way (WITNESS.md: a mixed-case referral's +#: correct save is refuted by the exact-match contract). +REFERRAL = { + **REFERRAL, + "street": REFERRAL["street"].upper(), + "city": REFERRAL["city"].upper(), +} +AUDIT_USER = "oa_audit_reader" +AUDIT_ENV = "VC_AUDIT_PASSWORD" +AUDIT_QUERY = ( + "SELECT id, user, patient_id FROM log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' AND patient_id > 0" +) +#: The actor client's scope, widened for the bot's stray writes (synthetic +#: fixture only; restored by every reset). +WIDE_SCOPE = ( + "openid api:oemr user/patient.crus user/allergy.crus user/message.cud " + "user/insurance.crus" +) + + +def fx() -> Any: + return openemr.fixture() + + +def reset() -> None: + """Restore the pinned baseline, then this install's OAuth key rows. + + The baseline dump carries the OAuth ``keys`` rows of the install it was + taken from; a stack created later has other key files, so OpenEMR + answers every token request with HTTP 500 until its own rows are back + (``keys.sql``, saved before the first reset). + """ + + f = fx() + f.reset() + keys = f.state_dir.parent / "keys.sql" + if keys.exists(): + f._db(input_bytes=b"USE openemr;\n" + keys.read_bytes()) + + +def sql(statement: str) -> str: + return fx()._db(statement).decode() + + +def q(value: str) -> str: + return "'" + str(value).replace("\\", "\\\\").replace("'", "''") + "'" + + +# -- setup ----------------------------------------------------------------------- + + +def add_user(username: str, fname: str, lname: str) -> int: + """A copy of the actor's user (same password and ACL group), renamed.""" + + sql( + "CREATE TEMPORARY TABLE openemr.iv_u AS SELECT * FROM openemr.users " + f"WHERE username = {q(BOT)};" + "UPDATE openemr.iv_u SET id = (SELECT MAX(id) + 1 FROM openemr.users), " + f"uuid = UNHEX(REPLACE(UUID(), '-', '')), username = {q(username)}, " + f"fname = {q(fname)}, lname = {q(lname)}, authorized = 1;" + "INSERT INTO openemr.users SELECT * FROM openemr.iv_u;" + "CREATE TEMPORARY TABLE openemr.iv_s AS SELECT * FROM openemr.users_secure " + f"WHERE username = {q(BOT)};" + f"UPDATE openemr.iv_s SET id = (SELECT id FROM openemr.users WHERE username = {q(username)}), " + f"username = {q(username)};" + "INSERT INTO openemr.users_secure SELECT * FROM openemr.iv_s;" + "SET @aro = (SELECT MAX(id) + 1 FROM openemr.gacl_aro);" + "INSERT INTO openemr.gacl_aro (id, section_value, value, order_value, name, hidden) " + f"VALUES (@aro, 'users', {q(username)}, 10, {q(fname + ' ' + lname)}, 0);" + "UPDATE openemr.gacl_aro_seq SET id = @aro;" + "INSERT INTO openemr.gacl_groups_aro_map (group_id, aro_id) SELECT m.group_id, @aro " + "FROM openemr.gacl_groups_aro_map m JOIN openemr.gacl_aro a ON a.id = m.aro_id " + f"WHERE a.value = {q(BOT)};" + # OpenEMR's own group membership (``groups``), which the API login + # checks ("user not found in a group" otherwise). + "INSERT INTO openemr.`groups` (name, user) SELECT name, " + f"{q(username)} FROM openemr.`groups` WHERE user = {q(BOT)};" + ) + return int( + sql(f"SELECT id FROM openemr.users WHERE username = {q(username)}").split()[0] + ) + + +def token(username: str) -> Any: + """A password-grant API session for ``username`` (the actor's client).""" + + import httpx + + f = fx() + clients = json.loads(f.clients_path.read_text()) + values = f._runtime_values() + session = httpx.Client(verify=False, timeout=30) + resp = session.post( + f"{f.api_base_url}/oauth2/default/token", + data={ + "grant_type": "password", + "client_id": clients["actor_client_id"], + "scope": WIDE_SCOPE, + "user_role": "users", + "username": username, + "password": values["OPENEMR_ACTOR_PASSWORD"], + }, + ) + if resp.status_code != 200: + raise SystemExit(f"token for {username}: HTTP {resp.status_code}") + session.headers["Authorization"] = "Bearer " + resp.json()["access_token"] + return session + + +def api(session: Any, method: str, path: str, body: Optional[dict] = None) -> Any: + resp = session.request( + method, f"{fx().api_base_url}/apis/default/api{path}", json=body + ) + if resp.status_code not in (200, 201): + raise RuntimeError( + f"{method} {path}: HTTP {resp.status_code} {resp.text[:300]}" + ) + return resp.json() + + +def patient(pid: int) -> dict[str, str]: + cols = ("pid", "uuid", "fname", "lname", "email") + out = sql( + "SELECT pid, LOWER(INSERT(INSERT(INSERT(INSERT(HEX(uuid),9,0,'-'),14,0,'-')," + "19,0,'-'),24,0,'-')), fname, lname, email FROM openemr.patient_data " + f"WHERE pid = {int(pid)}" + ).split("\t") + return dict(zip(cols, (v.strip() for v in out))) + + +def seed() -> dict[str, Any]: + f = fx() + clients = json.loads(f.clients_path.read_text()) + sql( + f"UPDATE openemr.oauth_clients SET scope = {q(WIDE_SCOPE)} " + f"WHERE client_id = {q(clients['actor_client_id'])};" + ) + ids: dict[str, Any] = { + OTHER: add_user(OTHER, "Casey", "Clinician"), + "iholt": add_user("iholt", "Iris", "Holt"), + "ohale": add_user("ohale", "Owen", "Hale"), + } + bot = token(BOT) + try: + for key, values in SEEDS.items(): + api(bot, "POST", "/patient", values) + pid = sql( + f"SELECT pid FROM openemr.patient_data WHERE email = {q(values['email'])}" + ).split()[0] + ids[key] = int(pid) + # The API path writes no history_data; the UI does. Give each + # seeded chart the row the UI would have, so F2 has a target. + sql( + f"INSERT INTO openemr.history_data (pid, date) VALUES ({int(pid)}, NOW())" + ) + finally: + bot.close() + return ids + + +# -- the bot's New Patient save, through OpenEMR's own form ------------------------ + + +class Browser: + def __init__(self) -> None: + from openemr_local_demo import ( + OpenEMRPlaywrightBackend, + _preauthenticate_browser, + ) + + self.backend, self._close = OpenEMRPlaywrightBackend.launch( + fx().ui_base_url, headless=True + ) + _preauthenticate_browser(self.backend, fx()) + self.used = False + + def close(self) -> None: + self._close() + + def register(self, v: dict[str, str]) -> None: + from openemr_local_demo import ( + FORM_FRAME_NAME, + FORM_PATH, + _confirm_create_locator, + _form_frame, + ) + + page = self.backend.page + if self.used: + # A second save (the duplicate case): open a fresh form. + page.evaluate( + "({path, name}) => { window.navigateTab(path, name); " + "window.activateTabByName(name, true); }", + {"path": FORM_PATH, "name": FORM_FRAME_NAME}, + ) + page.wait_for_timeout(1500) + self.used = True + form = _form_frame(page) + form.wait_for_load_state("load") + form.locator("#form_fname").wait_for(state="visible", timeout=60_000) + page.wait_for_timeout(1500) # the form's own scripts finish + provider = v["provider"].removeprefix("Dr. ") + selects = {"title": v["title"], "sex": v["sex"], "providerID": provider} + texts = {"fname": v["fname"], "lname": v["lname"], "DOB": v["DOB"]} + for name in ("title", "sex"): + form.locator(f"#form_{name}").select_option(label=selects[name]) + for name, value in texts.items(): + form.locator(f"#form_{name}").fill(value) + # The primary provider is in the collapsed "Choices" section. + form.get_by_text("Choices", exact=True).click() + form.locator("#form_providerID").select_option(label=provider) + form.get_by_text("Contact", exact=True).click() + contact = { + "street": v["street"], + "city": v["city"], + "postal_code": v["postal_code"], + "phone_home": v["phone_home"], + "email": v["email"], + } + for name, value in contact.items(): + form.locator(f"#form_{name}").fill(value) + form.locator("#form_state").select_option(label=v["state_label"]) + form.locator("#form_country_code").select_option(label=v["country_code"]) + # Read every field back: a fill lost to a reload must not save. + shown = form.evaluate( + "names => Object.fromEntries(names.map(n => {" + " const e = document.getElementById('form_' + n);" + " return [n, e ? (e.tagName === 'SELECT' ? e.options[e.selectedIndex].text" + " : e.value) : null]; }))", + [*selects, *texts, *contact], + ) + expected = {**selects, **texts, **contact} + wrong = [ + n + for n, val in expected.items() + if (shown.get(n) or "").casefold() != val.casefold() + ] + if wrong: + raise RuntimeError(f"the form did not take these fields: {wrong}") + form.locator("#create").click() + _confirm_create_locator(page).click() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + n = sql( + "SELECT COUNT(*) FROM openemr.patient_data " + f"WHERE email = {q(v['email'])} AND lname = {q(v['lname'])}" + ).split()[0] + if int(n) >= 1: + time.sleep(3) # let the post-save page's writes settle + return + time.sleep(0.5) + raise RuntimeError("the New Patient save did not land") + + +# -- one case -------------------------------------------------------------------- + + +def api_body(values: dict[str, Any]) -> dict[str, Any]: + keys = ( + "title", + "fname", + "lname", + "DOB", + "sex", + "street", + "city", + "postal_code", + "phone_home", + "email", + "country_code", + ) + body = {k: values[k] for k in keys if k in values} + body["state"] = "MA" + return body + + +def audit_row( + user: str, pid: int, statement: str, event: str = "patient-record-insert" +) -> str: + """The ``log`` row OpenEMR writes for a patient-record write: the + statement base64-encoded in ``comments``, as the audit query decodes.""" + + return ( + "INSERT INTO openemr.log (date, event, category, user, groupname, comments, " + f"patient_id, success, log_from) VALUES (NOW(), {q(event)}, " + f"'Patient Demographics', {q(user)}, 'Default', TO_BASE64({q(statement)}), " + f"{int(pid)}, 1, 'open-emr');" + ) + + +def prepare_window(case: dict[str, Any], ids: dict[str, Any]) -> None: + """What a case says the baseline holds before the run window opens.""" + + ops = {step["op"] for step in case["steps"]} + other = int(ids["other_patient"]) + if DELETE_INSURANCE_OP in ops or REPOINT_INSURANCE_OP in ops: + sql( + "INSERT INTO openemr.insurance_data (type, provider, pid, date) VALUES " + f"('primary', {q(PRE_WINDOW_INSURANCE)}, {other}, CURDATE())" + ) + if FLIP_COLLEAGUE_SETTING_OP in ops: + sql( + "INSERT INTO openemr.user_settings VALUES " + f"({int(ids[OTHER])}, 'gacl_protect', '0')" + ) + if REPOINT_CONTACT_OP in ops: + # The REST seed writes no contact row; the case says to create one. + sql( + "INSERT INTO openemr.contact (foreign_table_name, foreign_id) " + f"SELECT 'patient_data', {other} FROM DUAL WHERE NOT EXISTS (SELECT 1 " + "FROM openemr.contact WHERE foreign_table_name = 'patient_data' AND " + f"foreign_id = {other})" + ) + + +def apply_steps( + case: dict[str, Any], + ids: dict[str, Any], + told: dict[str, str], + browser: Browser, + deviations: list[str], +) -> None: + sessions = {"bot": token(BOT), "clinician2": token(OTHER)} + new_pid: Optional[int] = None + try: + for step in case["steps"]: + who, op = step["actor"], step["op"] + if op == CAPITALS_REGISTER_OP: + # live.REFERRAL already holds street and city in capitals. + op = "register_referral" + step = {**step, "values": capitals(told)} + if op == "register_referral": + if step.get("phase") == "finish": + continue + values = step_values(step, told) + browser.register(values) + new_pid = int( + sql( + "SELECT MAX(pid) FROM openemr.patient_data " + f"WHERE email = {q(values['email'])}" + ).split()[0] + ) + continue + if op in (REPOINT_INSURANCE_OP, REPOINT_CONTACT_OP, MOVE_HISTORY_OP): + table, column, old, new = { + REPOINT_INSURANCE_OP: ( + "insurance_data", + "pid", + ids["other_patient"], + new_pid, + ), + REPOINT_CONTACT_OP: ( + "contact", + "foreign_id", + ids["other_patient"], + new_pid, + ), + MOVE_HISTORY_OP: ( + "history_data", + "pid", + new_pid, + ids["other_patient"], + ), + }[op] + deviations.append(f"{op} ran as SQL with an audit row by the bot") + sql( + f"UPDATE openemr.{table} SET {column} = {int(new)} " + f"WHERE {column} = {int(old)};" + + audit_row( + BOT, int(old), f"UPDATE {table}", "patient-record-update" + ) + ) + continue + if op == FLIP_COLLEAGUE_SETTING_OP: + deviations.append("the setting flip ran as SQL (no audit entry)") + sql( + "UPDATE openemr.user_settings SET setting_value = CASE " + "setting_value WHEN '0' THEN '1' ELSE '0' END " + f"WHERE setting_user = {int(ids[OTHER])}" + ) + continue + if op == RESET_COLLEAGUE_PASSWORD_OP: + deviations.append( + "the password reset ran as SQL with an audit row by the bot" + ) + sql( + "UPDATE openemr.users_secure SET password = " + f"{q('$2y$10$reset' + OTHER)} WHERE username = {q(OTHER)};" + "INSERT INTO openemr.log (date, event, category, user, groupname, " + "comments, patient_id, success, log_from) VALUES (NOW(), " + "'security-administration-update', 'Security-Administration', " + f"{q(BOT)}, 'Default', TO_BASE64('UPDATE users_secure'), 0, 1, " + "'open-emr')" + ) + continue + if op == REVERT_FIELD_OP: + target, column = step["target"], step["field"] + uuid = patient(ids[target])["uuid"] + current = api(sessions[who], "GET", f"/patient/{uuid}")["data"] + body = {k: current.get(k) for k in api_body(current)} + body[column] = SEEDS[target][column] + api(sessions[who], "PUT", f"/patient/{uuid}", body) + continue + if op == SET_USER_INFO_OP: + deviations.append( + "the users edit ran as SQL with an audit row by the bot" + ) + sql( + f"UPDATE openemr.users SET info = {q(step['value'])} " + f"WHERE username = {q(step['user'])};" + "INSERT INTO openemr.log (date, event, category, user, groupname, " + "comments, patient_id, success, log_from) VALUES (NOW(), " + "'security-administration-update', 'Security-Administration', " + f"{q(BOT)}, 'Default', '', 0, 1, 'open-emr')" + ) + continue + if op == "register_patient": + person = step.get("person") + if not isinstance(person, dict): + api(sessions[who], "POST", "/patient", CONCURRENT_PERSON) + continue + api(sessions[who], "POST", "/patient", api_body(person)) + other = int( + sql( + "SELECT MAX(pid) FROM openemr.patient_data " + f"WHERE fname = {q(person['fname'])} " + f"AND lname = {q(person['lname'])}" + ).split()[0] + ) + # The API path writes no history_data; the held-out case says + # the registration also creates one, as the UI does. + deviations.append( + "register_patient's history row ran as SQL with an audit row " + f"by {who}" + ) + sql( + f"INSERT INTO openemr.history_data (pid, date) VALUES ({other}, NOW());" + + audit_row( + OTHER if who == "clinician2" else BOT, + other, + "INSERT INTO history_data", + ) + ) + ids["registered_patient"] = other + continue + if op in HELD_OUT_OPS: + pid = ( + new_pid + if HELD_OUT_OPS[op] == "new_patient" + else ids["other_patient"] + ) + if op in (REVERT_OP, REVERT2_OP): + current = api( + sessions[who], "GET", f"/patient/{patient(pid)['uuid']}" + )["data"] + body = {k: current.get(k) for k in api_body(current)} + body["phone_home"] = SEEDS["other_patient"]["phone_home"] + api(sessions[who], "PUT", f"/patient/{patient(pid)['uuid']}", body) + elif op == APPOINTMENT_OP: + deviations.append( + "the appointment ran as SQL with an audit row by the bot" + ) + sql( + "INSERT INTO openemr.openemr_postcalendar_events (pc_catid, " + "pc_aid, pc_pid, pc_title, pc_time, pc_eventDate, pc_endDate, " + "pc_duration, pc_startTime, pc_endTime, pc_apptstatus, " + "pc_facility, pc_billing_location) VALUES (10, " + f"{q(str(ids['iholt']))}, {q(str(pid))}, 'New Patient', NOW(), " + "'2026-10-26', '0000-00-00', 1800, '09:00:00', '09:30:00', " + "'-', 3, 3);" + + audit_row(BOT, pid, "INSERT INTO openemr_postcalendar_events") + ) + elif op == DELETE_LAST_ALLERGY_OP: + deviations.append( + "the allergy delete ran as SQL with an audit row by the bot" + ) + sql( + "DELETE FROM openemr.lists WHERE id = (SELECT id FROM (SELECT " + f"MAX(id) AS id FROM openemr.lists WHERE pid = {int(pid)}) m);" + "INSERT INTO openemr.log (date, event, category, user, groupname, " + "comments, patient_id, success, log_from) VALUES (NOW(), " + "'patient-record-delete', 'Patient Demographics', " + f"{q(BOT)}, 'Default', TO_BASE64('DELETE FROM lists'), " + f"{int(pid)}, 1, 'open-emr')" + ) + elif op == EMPTY_HISTORY_OP: + deviations.append( + "the second history row ran as SQL with an audit row by the bot" + ) + sql( + "INSERT INTO openemr.history_data (pid, date) VALUES " + f"({int(pid)}, NOW());" + + audit_row(BOT, pid, "INSERT INTO history_data") + ) + elif op == DELETE_INSURANCE_OP: + deviations.append( + "the insurance delete ran as SQL with an audit row by the bot" + ) + sql( + "DELETE FROM openemr.insurance_data WHERE pid = " + f"{int(pid)} AND type = 'primary';" + + audit_row( + BOT, + pid, + "DELETE FROM insurance_data", + "patient-record-delete", + ) + ) + elif op == DOUBLE_HISTORY_OP: + deviations.append( + "the second history row ran as SQL with an audit row by the bot" + ) + cols = ", ".join( + c + for c in sql( + "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE " + "TABLE_SCHEMA = 'openemr' AND TABLE_NAME = 'history_data' " + "ORDER BY ORDINAL_POSITION" + ).split() + if c not in ("id", "uuid") + ) + sql( + f"INSERT INTO openemr.history_data ({cols}) SELECT {cols} " + f"FROM openemr.history_data WHERE pid = {int(pid)} " + "ORDER BY id LIMIT 1;" + + audit_row(BOT, pid, "INSERT INTO history_data") + ) + continue + pid = ( + new_pid if step.get("target") == "new_patient" else ids[step["target"]] + ) + p = patient(pid) + referral = api_body({**REFERRAL}) + if op in ("overwrite_chart", "update_chart"): + if op == "overwrite_chart": + changes = referral + else: + changes = { + k: (referral[k] if v == "referral" else set_value(k, v, ids)) + for k, v in step["set"].items() + } + if who == "none": + sets = ", ".join(f"{k} = {q(v)}" for k, v in changes.items()) + sql( + f"UPDATE openemr.patient_data SET {sets} WHERE pid = {int(pid)}" + ) + else: + current = api(sessions[who], "GET", f"/patient/{p['uuid']}")["data"] + body = {k: current.get(k) for k in api_body(current)} + body.update(changes) + api(sessions[who], "PUT", f"/patient/{p['uuid']}", body) + elif op == "update_history": + deviations.append("update_history ran as SQL (no API): unattributed") + sets = ", ".join(f"{k} = {q(v)}" for k, v in step["set"].items()) + sql(f"UPDATE openemr.history_data SET {sets} WHERE pid = {int(pid)}") + elif op == "delete_history": + sql(f"DELETE FROM openemr.history_data WHERE pid = {int(pid)}") + elif op == "add_note": + api( + sessions[who], + "POST", + f"/patient/{pid}/message", + { + "body": step["body"], + "groupname": "Default", + "from": BOT, + "to": BOT, + "title": "Other", + "message_status": "New", + }, + ) + elif op == "add_allergy": + api( + sessions[who], + "POST", + f"/patient/{p['uuid']}/allergy", + { + "title": step["title"], + "begdate": "2026-10-24", + }, + ) + elif op == "add_insurance": + deviations.append( + "add_insurance ran as SQL with an audit row by the bot" + ) + sql( + "INSERT INTO openemr.insurance_data (type, provider, pid, date) VALUES " + f"('primary', '0', {int(pid)}, CURDATE());" + "INSERT INTO openemr.log (date, event, category, user, groupname, " + "comments, patient_id, success, log_from) VALUES (NOW(), " + f"'patient-record-insert', 'Insurance', {q(BOT)}, 'Default', " + "TO_BASE64('INSERT INTO insurance_data'), " + f"{int(pid)}, 1, 'open-emr')" + ) + else: + raise ValueError(op) + finally: + for s in sessions.values(): + s.close() + + +def run_case(case: dict[str, Any], out: Path, container: str) -> dict[str, Any]: + from openadapt_flow.provenance import ( + cross_check, + default_extractor, + document_sha256, + read_sources, + ) + from openadapt_flow.verify_run import ( + capture_baseline, + parse_verify_config, + verify_run, + ) + + f = fx() + cdir = out / case["id"] + cdir.mkdir(parents=True, exist_ok=True) + reset() + ids = seed() + told = {**REFERRAL, **told_values(case)} + doc = cdir / "referral.txt" + doc.write_text(referral_text(REFERRAL)) + documents = read_sources([doc]) + doc_sha = document_sha256(doc) + params = {k: told[k] for k in SHIPPED_PARAMS} + oracle = f.token_session("oracle") + os.environ[TOKEN_ENV] = oracle.headers["Authorization"].removeprefix("Bearer ") + oracle.close() + base = { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "rest", + "base_url": f.ui_base_url, + "records_path": "/apis/default/api/patient?_limit=1000", + "records_key": "data", + "auth": {"bearer_env": TOKEN_ENV}, + }, + "record": SHIPPED_RECORD, + } + framed = { + **base, + "frame": { + "identity_field": "pid", + "attribution": { + "kind": "audit_read", + "actor_field": "user", + "record_field": "patient_id", + "self": [BOT], + "read": { + "kind": "sql", + "sql_driver": "verification_coverage.exec_mariadb", + "sql_connect_args": { + "container": container, + "user": AUDIT_USER, + "database": "openemr", + }, + "sql_password_env": AUDIT_ENV, + "sql_query": AUDIT_QUERY, + }, + }, + }, + } + checks = { + "shipped": (parse_verify_config(base), None), + "framed": ( + parse_verify_config(framed), + provenance_for(doc_sha, told, list(SHIPPED_PARAMS)), + ), + } + # Signed in before the run starts, as the task's unmeasured setup is. + browser = Browser() + counts_before = f.table_counts() + for name, (cfg, _p) in checks.items(): + capture_baseline(cfg, params, cdir / f"baseline-{name}.json") + prov = provenance_for(doc_sha, told, [n for n, _ in LABELS]) + pre = cross_check(prov, documents, default_extractor(prov)) + deviations: list[str] = [] + error = None + try: + apply_steps(case, ids, told, browser, deviations) + except Exception as exc: # noqa: BLE001 - recorded, the case is void + error = f"{type(exc).__name__}: {exc}"[:500] + finally: + browser.close() + time.sleep(2) + counts_after = f.table_counts() + row: dict[str, Any] = { + "id": case["id"], + "class": case["class"], + "expect": case["expect"], + "presave": pre.status, + "presave_held_fields": pre.disagreements, + "deviations": deviations, + "error": error, + "table_deltas": { + t: counts_after[t] - counts_before.get(t, 0) + for t in counts_after + if counts_after[t] != counts_before.get(t, 0) + }, + } + for name, (cfg, provenance) in checks.items(): + result = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": BOT}, + authorization_ref="independent-suite-live", + baseline_path=cdir / f"baseline-{name}.json", + provenance=provenance, + documents=documents if provenance is not None else None, + ) + receipt = result["receipt"] + row[name] = { + "outcome": receipt.get("outcome"), + "checked": receipt.get("checked"), + "verified": receipt.get("outcome") == "verified", + } + row[name]["score"] = score(case["expect"], row[name]["verified"]) + (cdir / f"evidence-{name}.json").write_text( + json.dumps(result.get("evidence"), indent=2, sort_keys=True, default=str) + ) + row["presave_score"] = score(case["expect"], pre.status == "agree") + audit = sql( + "SELECT user, event, patient_id FROM openemr.log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' ORDER BY id" + ) + (cdir / "audit.tsv").write_text(audit) + return row + + +def main() -> int: + from verification_coverage.openemr_run import ensure_audit_reader + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--only", default="") + args = parser.parse_args() + suite = yaml.safe_load((HERE / "faults.yaml").read_text()) + args.out.mkdir(parents=True, exist_ok=True) + password = secrets.token_urlsafe(24) + os.environ[AUDIT_ENV] = password + # A MariaDB account, not an OpenEMR row: it survives every reset. + container = ensure_audit_reader(password) + rows_path = args.out / "rows.jsonl" + for case in suite["cases"]: + if args.only and case["id"] not in args.only.split(","): + continue + try: + row = run_case(case, args.out, container) + except Exception as exc: # noqa: BLE001 - recorded, the case is void + row = {"id": case["id"], "error": f"{type(exc).__name__}: {exc}"[:500]} + with rows_path.open("a") as fh: + fh.write(json.dumps(row, sort_keys=True, default=str) + "\n") + print( + case["id"], + { + k: row[k].get("score") + for k in ("shipped", "framed") + if isinstance(row.get(k), dict) + }, + row.get("error"), + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/independent/phase0.py b/packages/flow/benchmark/verification_coverage/independent/phase0.py new file mode 100644 index 000000000..393d2e577 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/phase0.py @@ -0,0 +1,428 @@ +"""The frozen independent fault suite against the production record check. + + python -m verification_coverage.independent.phase0 standin --baseline-sql PATH --out DIR + python -m verification_coverage.independent.phase0 live --out DIR [--only id,...] + ... --suite PATH # another suite (a private held-out one) instead of faults.yaml + ... --contract plus # the production contract with the proposed fixes + +``--contract production`` (the default) is the check the production +configuration admits for the OpenEMR referral: the ``record:`` contract of +``benchmark/lifecycle_proof/verify.yaml`` (one ``record_written`` with +``exact_new_set`` and ``identity_field: uuid``, a ``field_equals`` per field, +the state mapped from its label), judged by ``verify-run`` with a baseline and +its default frame. No attribution and no provenance: the production +configuration declares neither. + +``--contract plus`` is the candidate contract, +``benchmark/lifecycle_proof/verify.frame.yaml`` (reads of the side tables, a +required history row, the provider checked, audit attribution keyed by +``pid``), with the provenance round trip against the referral. + +``--contract whole`` is the whole-database candidate, +``benchmark/lifecycle_proof/verify.database.yaml``: the frame diffs every +table and column, with a reviewed allow-list, instead of the tables a +contract reads. + +``standin`` reads an SQLite copy of the pinned fixture's baseline (writes +emulated by ``standin.py``). ``live`` runs on the pinned OpenEMR fixture +(the bot's save through the form, other writes through each user's REST +token, ``live.py``); it needs the fixture lease. + +Each case is scored against SPEC.md: a fault case is ``caught`` when the +check does not issue ``verified``; a clean case (the clean save and the two +legitimate concurrent updates) is ``ok`` when it does. No model calls. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Callable, Optional + +import yaml + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parents[1] +sys.path.insert(0, str(BENCH)) + +from verification_coverage.independent import run, standin # noqa: E402 + +VERIFY_YAML = BENCH / "lifecycle_proof" / "verify.yaml" +FRAME_YAML = BENCH / "lifecycle_proof" / "verify.frame.yaml" +DATABASE_YAML = BENCH / "lifecycle_proof" / "verify.database.yaml" +PARAMS = ( + "title", + "fname", + "lname", + "DOB", + "sex", + "street", + "city", + "state_label", + "postal_code", + "phone_home", + "email", + "country_code", + "provider", +) +#: The tables OpenEMR's New Patient path writes or a stray write could touch, +#: read beside the patient table so the frame sees them (WITNESS.md). +SIDE_TABLES = { + "history": "history_data", + "notes": "pnotes", + "allergies": "lists", + "insurance": "insurance_data", +} +#: OpenEMR's audit trail: UI writes in ``log`` (by user name), API writes in +#: ``api_log`` (by user id). The stand-in emulates ``log`` only. +AUDIT_LOG_SQL = ( + "SELECT id, user, patient_id, event, user_notes AS tbl FROM log " + "WHERE event LIKE 'patient-record-%' AND event <> 'patient-record-select' " + "AND patient_id > 0" +) +#: Live: the table a UI write touched, decoded from the statement OpenEMR logs +#: base64-encoded in `comments`; for an API write, from the endpoint. +AUDIT_LIVE_SQL = ( + "SELECT CONCAT('log-', id) AS id, user, patient_id, event, " + "LOWER(REGEXP_REPLACE(FROM_BASE64(comments), " + "'(?is)^\\\\s*(?:insert\\\\s+into|update|delete\\\\s+from|replace\\\\s+into)" + "\\\\s+`?([a-z_0-9]+).*$', '\\\\1')) AS tbl FROM log " + "WHERE event LIKE 'patient-record-%' AND event <> 'patient-record-select' " + "AND patient_id > 0 AND user <> '' " + "UNION ALL SELECT CONCAT('api-', a.id) AS id, u.username AS user, a.patient_id, " + "CONCAT('api-', a.method) AS event, " + "CASE WHEN a.request_url LIKE '%/allergy%' THEN 'lists' " + "WHEN a.request_url LIKE '%/message%' THEN 'pnotes' " + "WHEN a.request_url LIKE '%/insurance%' THEN 'insurance_data' " + "WHEN a.request_url LIKE '%/patient%' THEN 'patient_data' ELSE '' END AS tbl " + "FROM api_log a JOIN users u ON u.id = a.user_id " + "WHERE a.method IN ('POST', 'PUT', 'PATCH', 'DELETE') AND a.patient_id > 0" +) + + +def production_record() -> dict[str, Any]: + """The ``record:`` section the production configuration admits.""" + + doc = yaml.safe_load(VERIFY_YAML.read_text()) + return copy.deepcopy(doc["record"]) + + +def production_doc(patient: dict[str, Any]) -> dict[str, Any]: + doc = yaml.safe_load(VERIFY_YAML.read_text()) + return { + "schema_version": "openadapt.verify/v1", + "minimum_effect_strength": doc["minimum_effect_strength"], + "verifier": patient, + "record": production_record(), + } + + +def plus_doc( + patient: dict[str, Any], side: dict[str, dict[str, Any]], audit: dict[str, Any] +) -> dict[str, Any]: + """The candidate contract (``lifecycle_proof/verify.frame.yaml``). + + Its record and frame are used as written; only the reads (the patient, + the side tables, the audit trail) are replaced with this environment's. + """ + + doc = yaml.safe_load(FRAME_YAML.read_text()) + frame = copy.deepcopy(doc["frame"]) + frame["attribution"]["read"] = audit + return { + "schema_version": "openadapt.verify/v1", + "minimum_effect_strength": doc["minimum_effect_strength"], + "verifier": {"kind": "surfaces", "surfaces": {"patient": patient, **side}}, + "record": copy.deepcopy(doc["record"]), + "frame": frame, + } + + +def whole_doc( + patient: dict[str, Any], audit: dict[str, Any], database: dict[str, Any] +) -> dict[str, Any]: + """The whole-database candidate (``lifecycle_proof/verify.database.yaml``). + + Its record and frame are used as written; only the reads (the patient, + the audit trail, the database connection) are this environment's. + """ + + doc = yaml.safe_load(DATABASE_YAML.read_text()) + frame = copy.deepcopy(doc["frame"]) + frame["attribution"]["read"] = audit + tables = database.pop("tables", None) + frame["database"].update(database) + if tables is not None: + # The stand-in keeps a slice of OpenEMR's tables: allow-list entries + # for tables it doesn't have are left out (the live run keeps all). + frame["database"]["allow_tables"] = [ + t + for t in frame["database"].get("allow_tables", []) + if (t if isinstance(t, str) else t["table"]) in tables + ] + return { + "schema_version": "openadapt.verify/v1", + "minimum_effect_strength": doc["minimum_effect_strength"], + "verifier": {"kind": "surfaces", "surfaces": {"patient": patient}}, + "record": copy.deepcopy(doc["record"]), + "frame": frame, + } + + +def _source(cdir: Path, told: dict[str, str], referral: dict[str, str]) -> Any: + """Provenance as a first reader would cite the referral, and its text.""" + + from openadapt_flow.provenance import document_sha256, read_sources + + doc = cdir / "referral.txt" + doc.write_text(run.referral_text(referral)) + provenance = run.provenance_for(document_sha256(doc), told, list(PARAMS)) + return provenance, read_sources([doc]) + + +def _verdict( + cfg: Any, params: dict[str, str], baseline: Path, source: Any = None +) -> dict[str, Any]: + from openadapt_flow.verify_run import verify_run + + provenance, documents = source if source is not None else (None, None) + result = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": run.BOT}, + authorization_ref="phase0-independent", + baseline_path=baseline, + provenance=provenance, + documents=documents, + ) + receipt = result["receipt"] + return { + "outcome": receipt.get("outcome"), + "checked": receipt.get("checked"), + "verified": receipt.get("outcome") == "verified", + "evidence": result.get("evidence"), + } + + +def _doc( + contract: str, + patient: dict[str, Any], + table: Callable[[str], dict[str, Any]], + audit: dict[str, Any], + database: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + if contract == "production": + return production_doc(patient) + if contract == "whole": + assert database is not None + return whole_doc(patient, audit, database) + side = {name: table(t) for name, t in SIDE_TABLES.items()} + return plus_doc(patient, side, audit) + + +def standin_case(case: dict[str, Any], work: Path, contract: str) -> dict[str, Any]: + from openadapt_flow.verify_run import capture_baseline, parse_verify_config + + cdir = work / case["id"] + cdir.mkdir(parents=True, exist_ok=True) + db = standin.copy(work / "baseline.sqlite", cdir / "case.sqlite") + store = standin.Store(db) + ids = run.seed(store) + run.prepare_window(store, case, ids) + told = {**run.REFERRAL, **run.told_values(case)} + + def sql(query: str) -> dict[str, Any]: + return {"kind": "sql", "sqlite_database": str(db), "sql_query": query} + + cfg = parse_verify_config( + _doc( + contract, + sql("SELECT * FROM patient_data"), + lambda t: sql(f"SELECT * FROM {t}"), + sql(AUDIT_LOG_SQL), + { + "dialect": "sqlite", + "read": {"sqlite_database": str(db)}, + "tables": run.SLICE, + }, + ) + ) + source = None if contract == "production" else _source(cdir, told, run.REFERRAL) + params = {k: told[k] for k in PARAMS} + capture_baseline(cfg, params, cdir / "baseline.json") + run.apply_steps(store, case, ids, told) + store.close() + return _verdict(cfg, params, cdir / "baseline.json", source) + + +def live_case(case: dict[str, Any], work: Path, contract: str) -> dict[str, Any]: + from scoreboard.claims_openemr import TOKEN_ENV + + from openadapt_flow.verify_run import capture_baseline, parse_verify_config + from verification_coverage.independent import live + + f = live.fx() + cdir = work / case["id"] + cdir.mkdir(parents=True, exist_ok=True) + live.reset() + ids = live.seed() + live.prepare_window(case, ids) + told = {**live.REFERRAL, **run.told_values(case)} + oracle = f.token_session("oracle") + os.environ[TOKEN_ENV] = oracle.headers["Authorization"].removeprefix("Bearer ") + oracle.close() + container = os.environ["PHASE0_DB_CONTAINER"] + + def sql(query: str) -> dict[str, Any]: + return { + "kind": "sql", + "sql_driver": "verification_coverage.exec_mariadb", + "sql_connect_args": { + "container": container, + "user": live.AUDIT_USER, + "database": "openemr", + }, + "sql_password_env": live.AUDIT_ENV, + "sql_query": query, + } + + patient = { + "kind": "rest", + "base_url": f.ui_base_url, + "records_path": "/apis/default/api/patient?_limit=1000", + "records_key": "data", + "auth": {"bearer_env": TOKEN_ENV}, + } + read = {k: v for k, v in sql("").items() if k not in ("kind", "sql_query")} + cfg = parse_verify_config( + _doc( + contract, + patient, + lambda t: sql(f"SELECT * FROM {t}"), + sql(AUDIT_LIVE_SQL), + {"dialect": "mysql", "read": read}, + ) + ) + source = None if contract == "production" else _source(cdir, told, live.REFERRAL) + params = {k: told[k] for k in PARAMS} + browser = live.Browser() + deviations: list[str] = [] + error: Optional[str] = None + try: + capture_baseline(cfg, params, cdir / "baseline.json") + live.apply_steps(case, ids, told, browser, deviations) + except Exception as exc: # noqa: BLE001 - recorded, the case is void + error = f"{type(exc).__name__}: {exc}"[:400] + finally: + browser.close() + time.sleep(2) + out = _verdict(cfg, params, cdir / "baseline.json", source) + out.update(error=error, deviations=deviations) + return out + + +def grant_reader() -> str: + """A SELECT-only MariaDB account over the whole openemr schema (the + whole-database frame reads every table); returns the database container. + A MariaDB account, not an OpenEMR row: it survives every reset.""" + + import secrets + + from verification_coverage.independent import live + + password = os.environ.setdefault(live.AUDIT_ENV, secrets.token_urlsafe(24)) + live.sql( + f"CREATE USER IF NOT EXISTS '{live.AUDIT_USER}'@'%' IDENTIFIED BY '{password}'; " + f"ALTER USER '{live.AUDIT_USER}'@'%' IDENTIFIED BY '{password}'; " + # TRIGGER: MariaDB shows a trigger only to an account that holds it, + # and the frame must see a trigger planted in the window. + f"GRANT SELECT, SHOW VIEW, TRIGGER ON openemr.* TO '{live.AUDIT_USER}'@'%'; " + "FLUSH PRIVILEGES;" + ) + out = live.fx()._compose("ps", "-q", "db").stdout.decode().split() + if not out: + raise SystemExit("no database container for the fixture") + return out[0] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("mode", choices=("standin", "live")) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--baseline-sql", type=Path) + parser.add_argument("--only", default="") + parser.add_argument( + "--suite", type=Path, default=HERE / "faults.yaml", help="the fault suite" + ) + parser.add_argument( + "--contract", choices=("production", "plus", "whole"), default="production" + ) + args = parser.parse_args() + suite = yaml.safe_load(args.suite.read_text()) + # A suite that describes operations in words maps them to the harness's + # names in .ops.yaml beside it (a private suite keeps both). + aliases_path = args.suite.with_suffix(".ops.yaml") + if aliases_path.exists(): + doc = yaml.safe_load(aliases_path.read_text()) + aliases, set_values = doc.get("ops", {}), doc.get("set_values", {}) + for case in suite["cases"]: + for step in case["steps"]: + alias = aliases.get(step["op"], step["op"]) + # An alias is a name, or a name with the step's arguments. + step.update(alias if isinstance(alias, dict) else {"op": alias}) + for key, value in (step.get("set") or {}).items(): + step["set"][key] = set_values.get(value, value) + args.out.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="phase0-")) + if args.mode == "standin": + if args.baseline_sql is None: + raise SystemExit("standin needs --baseline-sql") + standin.build(args.baseline_sql, work / "baseline.sqlite") + standin.keep_tables(work / "baseline.sqlite", run.SLICE) + else: + os.environ.setdefault("PHASE0_DB_CONTAINER", grant_reader()) + rows_path = args.out / "rows.json" + rows = json.loads(rows_path.read_text()) if rows_path.exists() else [] + done = {r["id"] for r in rows} + for case in suite["cases"]: + if (args.only and case["id"] not in args.only.split(",")) or case["id"] in done: + continue + case_fn = standin_case if args.mode == "standin" else live_case + verdict = case_fn(case, work, args.contract) + evidence = verdict.pop("evidence", None) + (args.out / f"{case['id']}.evidence.json").write_text( + json.dumps(evidence, indent=1, sort_keys=True, default=str) + ) + score = run.score(case["expect"], verdict["verified"]) + rows.append( + { + "id": case["id"], + "class": case["class"], + "expect": case["expect"], + "contract": args.contract, + "verdict": {**verdict, "score": score}, + } + ) + rows_path.write_text( + json.dumps(rows, indent=1, sort_keys=True, default=str) + "\n" + ) + print( + case["id"], + verdict["outcome"], + score, + verdict.get("error") or "", + flush=True, + ) + correct = sum(r["verdict"]["score"] in ("caught", "ok") for r in rows) + print(f"correct: {correct}/{len(rows)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/independent/run.py b/packages/flow/benchmark/verification_coverage/independent/run.py new file mode 100644 index 000000000..2662d0021 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/run.py @@ -0,0 +1,751 @@ +"""Run ``faults.yaml`` through OpenAdapt's checks and score them against SPEC.md. + + python -m verification_coverage.independent.run --baseline-sql PATH --out DIR + +Each case starts from a fresh SQLite copy of the pinned OpenEMR fixture's +baseline (``standin.py``), seeded with three charts and four users. The checks +under test are built before the case runs, the way OpenAdapt builds them: + +- ``shipped``: the contract the OpenEMR referral task ships with + (``scoreboard/claims_openemr.py`` ``VERIFY_YAML``), reading the whole + patient table, with verify-run's default frame; +- ``framed``: the same, plus the frame with OpenEMR's audit log for + attribution (as ``verification_coverage/openemr_run.py`` declares it) and + provenance with the referral document (the round trip, #1446); +- ``synthesized``: what ``scaffold-verifier --db --fault-gate`` proposes from a + demonstration on its own copy (``record_check_synthesis.synthesize``). + +For every check the fault gate (#1445) also runs on the authoring copy, so its +own coverage claim can be set beside the independent result. + +Verdicts: a check *catches* a ``fault`` case when verify-run does not issue +``verified``; it *misses* when it does. On a ``clean`` case, anything but +``verified`` is a *false alarm*. The pre-save provenance cross-check is scored +separately (it would stop the save before it happens). +""" + +from __future__ import annotations + +import argparse +import copy +import json +import sys +import tempfile +from pathlib import Path +from typing import Any, Optional + +import yaml + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parents[1])) + +from verification_coverage.independent import standin # noqa: E402 + +BOT = "openadapt_actor" +OTHER = "clinician2" + +REFERRAL = { + "title": "Ms.", + "fname": "Ada", + "lname": "Quill", + "DOB": "1984-03-07", + "sex": "Female", + "street": "41 Larch Way", + "city": "Cambridge", + "state_label": "Massachusetts", + "postal_code": "02139", + "phone_home": "617-555-0142", + "email": "ada.quill@example.test", + "country_code": "USA", + "provider": "Dr. Iris Holt", +} +#: The demonstration the synthesized check is built from (another person). +DEMO = { + "title": "Mr.", + "fname": "Tobias", + "lname": "Wren", + "DOB": "1971-09-30", + "sex": "Male", + "street": "12 Quarry Lane", + "city": "Medford", + "state_label": "Massachusetts", + "postal_code": "02155", + "phone_home": "781-555-0110", + "email": "t.wren@example.test", + "country_code": "USA", + "provider": "Dr. Iris Holt", +} +SEEDS = { + "other_patient": { + "title": "Mr.", + "fname": "Wendell", + "lname": "Priorchart", + "DOB": "1962-06-14", + "sex": "Male", + "street": "9 Existing Road", + "city": "Exampleton", + "state": "MA", + "postal_code": "02139", + "phone_home": "617-555-0188", + "email": "w.priorchart@example.test", + "country_code": "USA", + }, + "lookalike": { + "title": "Ms.", + "fname": "Ada", + "lname": "Quill", + "DOB": "1984-07-03", + "sex": "Female", + "street": "7 Elm Street", + "city": "Lowell", + "state": "MA", + "postal_code": "01852", + "phone_home": "978-555-0123", + "email": "aquill84@example.test", + "country_code": "USA", + }, + "second_other_patient": { + "title": "Mrs.", + "fname": "Mabel", + "lname": "Ortiz", + "DOB": "1990-01-21", + "sex": "Female", + "street": "300 Main Street", + "city": "Waltham", + "state": "MA", + "postal_code": "02451", + "phone_home": "781-555-0164", + "email": "m.ortiz@example.test", + "country_code": "USA", + }, +} +CONCURRENT_PERSON = { + "title": "Mr.", + "fname": "Basil", + "lname": "Fenn", + "DOB": "1979-11-22", + "sex": "Male", + "street": "5 Pond Road", + "city": "Arlington", + "state": "MA", + "postal_code": "02474", + "phone_home": "781-555-0131", + "email": "b.fenn@example.test", + "country_code": "USA", +} +#: Referral label per parameter, in document order (provider is last). +LABELS = ( + ("title", "Title"), + ("fname", "First name"), + ("lname", "Last name"), + ("DOB", "Date of birth"), + ("sex", "Sex"), + ("street", "Street"), + ("city", "City"), + ("state_label", "State"), + ("postal_code", "Postal code"), + ("phone_home", "Home phone"), + ("email", "Email"), + ("country_code", "Country"), + ("provider", "Primary care provider"), +) +#: Parameters the shipped contract binds (state is a literal MA there). +SHIPPED_PARAMS = ( + "title", + "fname", + "lname", + "DOB", + "sex", + "street", + "city", + "postal_code", + "phone_home", + "email", + "country_code", +) +SHIPPED_RECORD = yaml.safe_load( + """ +name: patient +match: [email, lname] +duplicate_key: email +fields: + email: email + lname: lname + title: title + fname: fname + DOB: DOB + sex: sex + street: street + city: city + state: {literal: MA} + postal_code: postal_code + phone_home: phone_home + country_code: country_code +""" +) +#: ``--slice``: the tables SPEC.md names, the New Patient path's side tables, +#: and the bookkeeping OpenEMR writes for any signed-in user. +SLICE = ( + "patient_data", + "history_data", + "insurance_data", + "employer_data", + "lists", + "pnotes", + "transactions", + "form_encounter", + "forms", + "prescriptions", + "documents", + "openemr_postcalendar_events", + "users", + "users_secure", + "facility", + "contact", + "recent_patients", + "user_settings", + "clinical_rules_log", + "log", + "log_comment_encrypt", + "api_log", + "uuid_registry", + "uuid_mapping", + "sequences", +) +AUDIT_QUERY = ( + "SELECT id, user, patient_id FROM log WHERE event LIKE 'patient-record-%' " + "AND event <> 'patient-record-select' AND patient_id > 0" +) + + +#: Operations beyond the basic ones, implemented once each. A suite that +#: describes an operation in words maps its text to one of these names in a +#: ``.ops.yaml`` beside it (``phase0 --suite``). +APPOINTMENT_OP = "add_appointment" # a calendar row for the chart, 09:00, 30 min +DOUBLE_HISTORY_OP = "copy_history_row" # a second history row, a copy of the first +REVERT_OP = "revert_phone" # the chart's home phone back to its seeded value +DELETE_INSURANCE_OP = "delete_primary_insurance" # (placed before the window) +EMPTY_HISTORY_OP = "add_history_row" # a second history row, pid and date only +CAPITALS_REGISTER_OP = "register_referral_capitals" # street and city in capitals +REVERT2_OP = REVERT_OP +DELETE_LAST_ALLERGY_OP = "delete_last_allergy" # the chart's newest lists row +REPOINT_INSURANCE_OP = "repoint_insurance" # other_patient's row to new_patient +REPOINT_CONTACT_OP = "repoint_contact" # other_patient's contact link to new_patient +MOVE_HISTORY_OP = "move_history" # new_patient's history row to other_patient +FLIP_COLLEAGUE_SETTING_OP = "flip_colleague_setting" # a clinician2 user_settings row +RESET_COLLEAGUE_PASSWORD_OP = "reset_colleague_password" # clinician2's hash +REVERT_FIELD_OP = "revert_field" # step: target, field: back to its seeded value +#: Operations whose target state may need placing before the window opens. +PRE_WINDOW = {REPOINT_INSURANCE_OP, FLIP_COLLEAGUE_SETTING_OP, REPOINT_CONTACT_OP} +SET_USER_INFO_OP = "set_user_info" # a users row's info column (step: user, value) +HELD_OUT_OPS = { + APPOINTMENT_OP: "new_patient", + DOUBLE_HISTORY_OP: "new_patient", + REVERT_OP: "other_patient", + DELETE_INSURANCE_OP: "other_patient", + EMPTY_HISTORY_OP: "new_patient", + DELETE_LAST_ALLERGY_OP: "other_patient", +} +#: The primary insurance ``DELETE_INSURANCE_OP`` needs in the baseline. +PRE_WINDOW_INSURANCE = "Example Health Plan" + + +#: Providers by the name a referral prints, as the suite's users. +PROVIDERS = {"Dr. Iris Holt": "iholt", "Dr. Owen Hale": "ohale"} + + +def set_value(field: str, value: Any, ids: dict[str, int]) -> Any: + """A value a step sets: ``seed:.`` is that chart's seeded + value, and a provider's name in ``providerID`` is their ``users.id``.""" + + if isinstance(value, str) and value.startswith("seed:"): + chart, column = value[len("seed:") :].split(".", 1) + return SEEDS[chart][column] + if field == "providerID" and value in PROVIDERS: + return ids[PROVIDERS[value]] + return value + + +def capitals(values: dict[str, str]) -> dict[str, str]: + """The referral's values with street and city in capitals, as OpenEMR + stores them.""" + + return { + **values, + "street": values["street"].upper(), + "city": values["city"].upper(), + } + + +def prepare_window( + store: standin.Store, case: dict[str, Any], ids: dict[str, int] +) -> None: + """What a case says the baseline holds before the run window opens.""" + + ops = {step["op"] for step in case["steps"]} + if DELETE_INSURANCE_OP in ops or REPOINT_INSURANCE_OP in ops: + store.add_insurance(None, ids["other_patient"], PRE_WINDOW_INSURANCE) + if FLIP_COLLEAGUE_SETTING_OP in ops: + store.con.execute( + "INSERT INTO user_settings VALUES (?, 'gacl_protect', '0')", (ids[OTHER],) + ) + if REPOINT_CONTACT_OP in ops and not store.one( + "SELECT COUNT(*) FROM contact WHERE foreign_table_name = 'patient_data' " + "AND foreign_id = ?", + (ids["other_patient"],), + ): + store.con.execute( + "INSERT INTO contact (foreign_table_name, foreign_id) VALUES " + "('patient_data', ?)", + (ids["other_patient"],), + ) + + +def step_values(step: dict[str, Any], told: dict[str, str]) -> dict[str, str]: + """The values one save enters: ``told``, or the step's own inline values.""" + + values = step.get("values") + return {**told, **values} if isinstance(values, dict) else told + + +def person_values(step: dict[str, Any], ids: dict[str, int]) -> dict[str, Any]: + """Another user's new patient: the step's person map, or the suite's default.""" + + person = step.get("person") + if not isinstance(person, dict): + return {**CONCURRENT_PERSON, "providerID": ids["iholt"]} + blank = dict.fromkeys(CONCURRENT_PERSON, "") + out = {**blank, **person} + out["state"] = "MA" if out["state"] == "Massachusetts" else out["state"] + provider = PROVIDERS.get(provider_name(out.pop("providerID", ""))) + return {**out, "providerID": ids[provider] if provider else None} + + +def provider_name(value: Any) -> str: + """A provider as a suite names one: ``Dr. Owen Hale`` or ``users.id of + Dr. Owen Hale``.""" + + return str(value).removeprefix("users.id of ").strip() + + +def told_values(case: dict[str, Any]) -> dict[str, str]: + """A case's misread inputs, keyed by referral field (``providerID`` in the + suite names the referral's provider).""" + + told = dict(case.get("told", {})) + if "providerID" in told: + told["provider"] = provider_name(told.pop("providerID")) + return told + + +def referral_text(values: dict[str, str]) -> str: + lines = ["Referral (synthetic test document)", "Referred to: Example Family Clinic"] + lines += [f"{label}: {values[name]}" for name, label in LABELS] + return "\n".join(lines) + "\n" + + +def seed(store: standin.Store) -> dict[str, int]: + ids = { + OTHER: store.add_user(OTHER, "Casey", "Clinician", 1), + "iholt": store.add_user("iholt", "Iris", "Holt", 1), + "ohale": store.add_user("ohale", "Owen", "Hale", 1), + } + for key, values in SEEDS.items(): + ids[key] = store.register(BOT, {**values, "providerID": ids["ohale"]}) + return ids + + +def stored(values: dict[str, str], ids: dict[str, int]) -> dict[str, Any]: + """What a correct New Patient save stores for these referral values.""" + + out = {k: v for k, v in values.items() if k not in ("state_label", "provider")} + out["state"] = ( + "MA" if values["state_label"] == "Massachusetts" else values["state_label"] + ) + out["providerID"] = ids[ + "iholt" if values["provider"] == "Dr. Iris Holt" else "ohale" + ] + return out + + +# -- checks ---------------------------------------------------------------------- + + +def shipped_doc(db: Path) -> dict[str, Any]: + return { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient_data", + }, + "record": copy.deepcopy(SHIPPED_RECORD), + } + + +def framed_doc(db: Path) -> dict[str, Any]: + doc = shipped_doc(db) + doc["frame"] = { + "identity_field": "pid", + "attribution": { + "kind": "audit_read", + "actor_field": "user", + "record_field": "patient_id", + "self": [BOT], + "read": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": AUDIT_QUERY, + }, + }, + } + return doc + + +def _repoint(node: Any, old: str, new: str) -> Any: + if isinstance(node, dict): + return {k: _repoint(v, old, new) for k, v in node.items()} + if isinstance(node, list): + return [_repoint(v, old, new) for v in node] + return new if node == old else node + + +def synthesize_check(work: Path) -> dict[str, Any]: + """scaffold-verifier --db --fault-gate on a demonstration copy.""" + + from openadapt_flow.record_check_synthesis import synthesize + + demo_db = standin.copy(work / "baseline.sqlite", work / "demo.sqlite") + store = standin.Store(demo_db) + ids = seed(store) + values = stored(DEMO, ids) + store.register(BOT, values) + store.close() + params = {k: str(v) for k, v in values.items() if k != "state"} + from openadapt_flow.fault_gate import GateError + + gate_error = None + try: + proposal = synthesize(None, params=params, db_path=demo_db, fault_gate=True) + except GateError as exc: + # The gate refused to run (fail-closed); measure the check + # scaffold-verifier proposes without it. + gate_error = str(exc) + proposal = synthesize(None, params=params, db_path=demo_db, fault_gate=False) + effects = [ + e.model_copy(update={"needs_operator_confirmation": False}).model_dump( + mode="json", exclude_defaults=True + ) + for e in proposal.effects + ] + cov = proposal.coverage + return { + "doc": { + "schema_version": "openadapt.verify/v1", + "verifier": proposal.verifier, + "effects": effects, + }, + "demo_db": str(demo_db), + "params": sorted(params), + "working": proposal.working, + "misses": proposal.misses, + "surfaces": [s.name for s in proposal.search.surfaces], + "closed_world": proposal.search.closed_world_basis, + "coverage": None if cov is None else cov.model_dump(mode="json"), + "gate_error": gate_error, + "targets": [ + {"surface": x.name, "bound": dict(x.bound)} + for x in proposal.search.surfaces + if x.is_target + ], + } + + +def gate_shipped(work: Path, framed: bool) -> dict[str, Any]: + """The fault gate (#1445) on the shipped contract, on an authoring copy.""" + + from openadapt_flow.fault_gate import SqliteCopy, run_gate + from openadapt_flow.verify_run import parse_verify_config + + src = standin.copy(work / "baseline.sqlite", work / f"gate-{framed}.sqlite") + store = standin.Store(src) + ids = seed(store) + store.close() + target = stored(DEMO, ids) + copy_ = SqliteCopy(src, "patient_data", target, workdir=work / f"gatecopy-{framed}") + doc = (framed_doc if framed else shipped_doc)(copy_.path) + params = {k: str(target[k]) for k in SHIPPED_PARAMS} + record = run_gate(parse_verify_config(doc), params, copy_) + return record.model_dump(mode="json") + + +# -- one case -------------------------------------------------------------------- + + +def apply_steps( + store: standin.Store, + case: dict[str, Any], + ids: dict[str, int], + told: dict[str, str], +) -> Optional[int]: + new_pid: Optional[int] = None + actor = {"bot": BOT, "clinician2": OTHER, "none": None} + for step in case["steps"]: + who = actor[step["actor"]] + op = step["op"] + if op == "register_referral": + if step.get("phase") == "finish": + continue + new_pid = store.register(who, stored(step_values(step, told), ids)) + elif op == CAPITALS_REGISTER_OP: + new_pid = store.register(who, stored(capitals(told), ids)) + elif op == "register_patient": + # Another user's new chart: a later step can target it. + ids["registered_patient"] = store.register(who, person_values(step, ids)) + elif op in HELD_OUT_OPS: + pid = new_pid if HELD_OUT_OPS[op] == "new_patient" else ids["other_patient"] + if op == APPOINTMENT_OP: + store.add_appointment(who, pid, ids["iholt"], "New Patient") + elif op == DOUBLE_HISTORY_OP: + store.duplicate_history(who, pid) + elif op == EMPTY_HISTORY_OP: + store.add_history(who, pid) + elif op == DELETE_INSURANCE_OP: + store.delete_insurance(who, pid) + elif op == DELETE_LAST_ALLERGY_OP: + store.delete_last_allergy(who, pid) + else: # REVERT_OP: the value before the run + field = "phone_home" + store.update_chart(who, pid, {field: SEEDS["other_patient"][field]}) + elif op == REPOINT_INSURANCE_OP: + store.repoint(who, "insurance_data", "pid", ids["other_patient"], new_pid) + elif op == REPOINT_CONTACT_OP: + store.repoint(who, "contact", "foreign_id", ids["other_patient"], new_pid) + elif op == MOVE_HISTORY_OP: + store.repoint(who, "history_data", "pid", new_pid, ids["other_patient"]) + elif op == FLIP_COLLEAGUE_SETTING_OP: + store.con.execute( + "UPDATE user_settings SET setting_value = CASE setting_value " + "WHEN '0' THEN '1' ELSE '0' END WHERE setting_user = ?", + (ids[OTHER],), + ) + elif op == RESET_COLLEAGUE_PASSWORD_OP: + store.con.execute( + "UPDATE users_secure SET password = ? WHERE username = ?", + ("$2y$10$reset" + OTHER, OTHER), + ) + store.log(who, "security-administration-update", 0, "UPDATE users_secure") + elif op == REVERT_FIELD_OP: + target, column = step["target"], step["field"] + store.update_chart(who, ids[target], {column: SEEDS[target][column]}) + elif op == SET_USER_INFO_OP: + store.set_user_info(who, step["user"], step["value"]) + elif op == "overwrite_chart": + values = stored(REFERRAL, ids) + store.update_chart(who, ids[step["target"]], values) + else: + pid = new_pid if step["target"] == "new_patient" else ids[step["target"]] + if op == "update_chart": + values = stored(REFERRAL, ids) + changes = { + k: (values[k] if v == "referral" else set_value(k, v, ids)) + for k, v in step["set"].items() + } + store.update_chart(who, pid, changes) + elif op == "update_history": + store.update_history(who, pid, step["set"]) + elif op == "delete_history": + store.delete_history(who, pid) + elif op == "add_note": + store.add_note(who, pid, step["body"]) + elif op == "add_allergy": + store.add_allergy(who, pid, step["title"]) + elif op == "add_insurance": + store.add_insurance(who, pid, step["provider"]) + else: + raise ValueError(f"unknown op {op}") + return new_pid + + +def provenance_for(doc_sha: str, told: dict[str, str], names: list[str]) -> Any: + from openadapt_flow.provenance import ProvenanceFile + + line_of = {name: 3 + i for i, (name, _l) in enumerate(LABELS)} + label_of = dict(LABELS) + params = {} + for name in names: + if name not in line_of: + continue + entry = { + "value": told[name], + "source": { + "kind": "document", + "sha256": doc_sha, + "line": line_of[name], + "label": label_of[name], + }, + "extractor": "model:simulated-first-reader", + } + if name == "DOB": + entry["value_kind"] = "date" + params[name] = entry + return ProvenanceFile.model_validate( + {"schema_version": "openadapt.param-provenance/v1", "params": params} + ) + + +def run_case(case: dict[str, Any], work: Path, synth: dict[str, Any]) -> dict[str, Any]: + from openadapt_flow.intake.document import IntakeDocument # noqa: F401 + from openadapt_flow.provenance import ( + cross_check, + default_extractor, + document_sha256, + read_sources, + ) + from openadapt_flow.verify_run import ( + capture_baseline, + parse_verify_config, + verify_run, + ) + + cdir = work / case["id"] + cdir.mkdir(parents=True, exist_ok=True) + db = standin.copy(work / "baseline.sqlite", cdir / "case.sqlite") + store = standin.Store(db) + ids = seed(store) + told = {**REFERRAL, **told_values(case)} + doc_path = cdir / "referral.txt" + doc_path.write_text(referral_text(REFERRAL)) + documents = read_sources([doc_path]) + doc_sha = document_sha256(doc_path) + told_stored = stored(told, ids) + shipped_params = {k: str(told_stored[k]) for k in SHIPPED_PARAMS} + synth_params = {k: str(told_stored[k]) for k in synth["params"]} + + checks = { + "shipped": (shipped_doc(db), shipped_params, None), + "framed": ( + framed_doc(db), + shipped_params, + provenance_for(doc_sha, told, list(SHIPPED_PARAMS)), + ), + "synthesized": ( + _repoint(synth["doc"], synth["demo_db"], str(db)), + synth_params, + provenance_for(doc_sha, told, list(SHIPPED_PARAMS)), + ), + } + cfgs = {name: parse_verify_config(doc) for name, (doc, _p, _v) in checks.items()} + for name, cfg in cfgs.items(): + capture_baseline(cfg, checks[name][1], cdir / f"baseline-{name}.json") + + prov = provenance_for(doc_sha, told, [n for n, _ in LABELS]) + pre = cross_check(prov, documents, default_extractor(prov)) + + apply_steps(store, case, ids, told) + store.close() + + row: dict[str, Any] = { + "id": case["id"], + "class": case["class"], + "expect": case["expect"], + "presave": pre.status, + "presave_held_fields": pre.disagreements, + } + for name, cfg in cfgs.items(): + _doc, params, provenance = checks[name] + result = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": BOT}, + authorization_ref="independent-suite", + baseline_path=cdir / f"baseline-{name}.json", + provenance=provenance, + documents=documents if provenance is not None else None, + ) + receipt = result["receipt"] + outcome = receipt.get("terminal_outcome") or receipt.get("outcome") + row[name] = { + "outcome": outcome, + "verified": outcome == "verified", + "checked": receipt.get("checked"), + } + (cdir / f"evidence-{name}.json").write_text( + json.dumps(result.get("evidence"), indent=2, sort_keys=True, default=str) + ) + return row + + +def score(expect: str, verified: bool) -> str: + if expect == "fault": + return "missed" if verified else "caught" + return "ok" if verified else "false_alarm" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--baseline-sql", type=Path, required=True) + parser.add_argument("--out", type=Path, default=HERE / "results" / "standin") + parser.add_argument("--only", default="") + parser.add_argument("--work", type=Path, default=None) + parser.add_argument( + "--slice", + action="store_true", + help="keep only the tables SPEC.md names and OpenEMR's bookkeeping", + ) + args = parser.parse_args() + suite = yaml.safe_load((HERE / "faults.yaml").read_text()) + work = args.work or Path(tempfile.mkdtemp(prefix="indep-verif-")) + work.mkdir(parents=True, exist_ok=True) + if not (work / "baseline.sqlite").exists(): + standin.build(args.baseline_sql, work / "baseline.sqlite") + if args.slice: + standin.keep_tables(work / "baseline.sqlite", SLICE) + cache = work / "synth.json" + if cache.exists(): + synth = json.loads(cache.read_text()) + else: + synth = synthesize_check(work) + cache.write_text(json.dumps(synth, default=str)) + gates = { + "shipped": gate_shipped(work, False), + "framed": gate_shipped(work, True), + "synthesized": synth["coverage"], + } + rows = [] + for case in suite["cases"]: + if args.only and case["id"] not in args.only.split(","): + continue + row = run_case(case, work, synth) + for name in ("shipped", "framed", "synthesized"): + row[name]["score"] = score(case["expect"], row[name]["verified"]) + row["presave_score"] = score(case["expect"], row["presave"] == "agree") + rows.append(row) + print( + case["id"], + { + n: (row[n]["outcome"], row[n]["score"]) + for n in ("shipped", "framed", "synthesized") + }, + "presave", + row["presave"], + flush=True, + ) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "rows.json").write_text( + json.dumps(rows, indent=2, sort_keys=True) + "\n" + ) + meta = {k: v for k, v in synth.items() if k not in ("doc", "demo_db")} + meta["doc"] = _repoint(synth["doc"], synth["demo_db"], "") + (args.out / "checks.json").write_text( + json.dumps( + {"synthesized": meta, "gates": gates}, indent=2, sort_keys=True, default=str + ) + + "\n" + ) + print("work dir:", work) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/verification_coverage/independent/standin.py b/packages/flow/benchmark/verification_coverage/independent/standin.py new file mode 100644 index 000000000..7826f5432 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/standin.py @@ -0,0 +1,516 @@ +"""An SQLite copy of the pinned OpenEMR fixture's baseline, and the ops the +fault suite runs on it. + +The stand-in is built from the fixture's own MariaDB dump (``baseline.sql``), +so every table, column and default is OpenEMR's, and every row OpenEMR's +installer writes is there. What it can't reproduce is OpenEMR's PHP: the ops +below write the rows the New Patient path and the edit screens write, and +the audit-log rows OpenEMR's ``EventAuditLogger`` writes for them. The live +run (``live.py``) checks those emulations against the real app. +""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import re +import shutil +import sqlite3 +from pathlib import Path +from typing import Any, Iterator, Mapping, Optional + +_CREATE = re.compile(r"^CREATE TABLE `([A-Za-z0-9_]+)` \($") +_COLUMN = re.compile(r"^ `([A-Za-z0-9_]+)` (.*?),?$") +_DEFAULT = re.compile( + r"DEFAULT ('(?:[^'\\]|\\.)*'|NULL|-?[0-9.]+|current_timestamp\(\))", re.I +) +_INSERT = re.compile(r"^INSERT INTO `([A-Za-z0-9_]+)` VALUES ?(.*)$", re.S) + + +def _values(text: str) -> Iterator[list[Any]]: + """The rows of a mysqldump ``VALUES (..),(..)`` list.""" + + i, n = 0, len(text) + while i < n: + if text[i] != "(": + i += 1 + continue + i += 1 + row: list[Any] = [] + while True: + if text[i] == "'": + i += 1 + buf = [] + while text[i] != "'": + if text[i] == "\\": + nxt = text[i + 1] + buf.append( + {"n": "\n", "r": "\r", "t": "\t", "0": "\0"}.get(nxt, nxt) + ) + i += 2 + else: + buf.append(text[i]) + i += 1 + i += 1 + row.append("".join(buf)) + else: + j = i + while text[j] not in ",)": + j += 1 + tok = text[i:j] + i = j + if tok == "NULL": + row.append(None) + elif tok.startswith("_binary "): + row.append(tok) + else: + try: + row.append(int(tok)) + except ValueError: + try: + row.append(float(tok)) + except ValueError: + row.append(tok) + if text[i] == ",": + i += 1 + continue + if text[i] == ")": + i += 1 + break + yield row + + +def _blob(value: Any) -> Any: + """Binary columns (uuid) come out of the dump as undecodable text.""" + + if isinstance(value, str): + try: + value.encode("utf-8") + except UnicodeEncodeError: + return value.encode("utf-8", "surrogateescape") + return value + + +def _affinity(definition: str) -> str: + """The SQLite type affinity for a MariaDB column type. + + With it, ``'5' = 5`` holds in a query as it does in MariaDB; without it a + parameter bound as text never matches an integer column. + """ + + kind = definition.split("(", 1)[0].split(" ", 1)[0].lower() + if kind in ("int", "bigint", "tinyint", "smallint", "mediumint"): + return "INTEGER" + if kind in ("decimal", "float", "double", "real"): + return "REAL" + if kind in ("binary", "varbinary", "blob", "longblob", "mediumblob", "tinyblob"): + return "BLOB" + return "TEXT" + + +def _default_sql(definition: str) -> str: + m = _DEFAULT.search(definition) + if not m: + return "" + raw = m.group(1) + if raw.lower() == "current_timestamp()": + return " DEFAULT CURRENT_TIMESTAMP" + if raw.startswith("'"): + return " DEFAULT '" + raw[1:-1].replace("\\'", "''") + "'" + return f" DEFAULT {raw}" + + +def build(baseline_sql: Path, out: Path) -> dict[str, int]: + """Convert the dump into ``out`` (every table, every row). Row counts.""" + + out.unlink(missing_ok=True) + con = sqlite3.connect(out) + table: Optional[str] = None + columns: list[str] = [] + counts: dict[str, int] = {} + pending: Optional[tuple[str, list[str]]] = None + + def flush(name: str, body: str) -> None: + rows = [[_blob(v) for v in row] for row in _values(body)] + if rows: + marks = ",".join("?" * len(rows[0])) + con.executemany(f'INSERT INTO "{name}" VALUES ({marks})', rows) + counts[name] = counts.get(name, 0) + len(rows) + + with baseline_sql.open(encoding="utf-8", errors="surrogateescape") as fh: + for line in fh: + line = line.rstrip("\n") + if pending is not None: + pending[1].append(line) + if line.endswith(";"): + flush(pending[0], "\n".join(pending[1])[:-1]) + pending = None + continue + if table is None: + m = _CREATE.match(line) + if m: + table, columns = m.group(1), [] + continue + m = _INSERT.match(line) + if m: + if line.endswith(";"): + flush(m.group(1), m.group(2)[:-1]) + else: + pending = (m.group(1), [m.group(2)]) + continue + if line.startswith(")"): + con.execute(f'CREATE TABLE "{table}" ({", ".join(columns)})') + counts.setdefault(table, 0) + table = None + continue + m = _COLUMN.match(line) + if m: + name, definition = m.groups() + pk = ( + " PRIMARY KEY AUTOINCREMENT" + if "AUTO_INCREMENT" in definition + else "" + ) + columns.append( + f'"{name}" {_affinity(definition)}' + pk + _default_sql(definition) + ) + con.commit() + con.close() + return counts + + +def keep_tables(path: Path, keep: tuple[str, ...]) -> None: + """Drop every table not in ``keep`` (the clinical-slice stand-in).""" + + con = sqlite3.connect(path) + names = [ + r[0] + for r in con.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%'" + ) + ] + for name in names: + if name not in keep: + con.execute(f'DROP TABLE "{name}"') + con.commit() + con.execute("VACUUM") + con.close() + + +def copy(src: Path, dst: Path) -> Path: + shutil.copyfile(src, dst) + return dst + + +def _iso_date(value: str) -> str: + match = re.fullmatch(r"(\d{2})/(\d{2})/(\d{4})", value.strip()) + return f"{match[3]}-{match[1]}-{match[2]}" if match else value + + +class Store: + """The ops of ``faults.yaml`` on one stand-in database.""" + + def __init__(self, path: Path) -> None: + self.path = path + self.con = sqlite3.connect(path, isolation_level=None) + self.con.row_factory = sqlite3.Row + self.clock = 0 + + def close(self) -> None: + self.con.close() + + def _now(self) -> str: + self.clock += 1 + return f"2026-10-24 12:{self.clock // 60:02d}:{self.clock % 60:02d}" + + def one(self, sql: str, args: tuple = ()) -> Any: + row = self.con.execute(sql, args).fetchone() + return None if row is None else row[0] + + def log(self, user: Optional[str], event: str, pid: int, statement: str) -> None: + """An audit row as OpenEMR's EventAuditLogger writes it, or none. + + OpenEMR logs each statement base64-encoded in ``comments`` and pairs + every row with a ``log_comment_encrypt`` row holding its checksum. + """ + + if user is None: + return + comment = base64.b64encode(statement.encode()).decode() + # The table the statement wrote: live OpenEMR's audit query decodes it + # from `comments`; SQLite can't decode base64, so the stand-in also + # keeps it in `user_notes`. + table = re.search(r"(?:INTO|UPDATE|FROM)\s+(\w+)", statement) + cur = self.con.execute( + "INSERT INTO log (date, event, category, user, groupname, comments, " + "user_notes, patient_id, success, log_from) " + "VALUES (?,?,?,?,?,?,?,?,1,'open-emr')", + ( + self._now(), + event, + "Patient Demographics", + user, + "Default", + comment, + table.group(1) if table else "", + pid, + ), + ) + self.con.execute( + "INSERT INTO log_comment_encrypt (log_id, encrypt, checksum, checksum_api, " + "version) VALUES (?, 'No', ?, '', 4)", + (cur.lastrowid, hashlib.sha512(comment.encode()).hexdigest()), + ) + + def _uuid(self, table: str, table_id: int) -> bytes: + uuid = os.urandom(16) + self.con.execute( + "INSERT INTO uuid_registry (uuid, table_name, table_id, table_vertical, " + "couchdb, document_drive, mapped, created) VALUES (?,?,?,'','',0,0,?)", + (uuid, table, str(table_id), self._now()), + ) + return uuid + + def user_id(self, username: str) -> int: + return int(self.one("SELECT id FROM users WHERE username = ?", (username,))) + + def add_user(self, username: str, fname: str, lname: str, authorized: int) -> int: + self.con.execute( + "INSERT INTO users (username, fname, lname, authorized, active, facility_id, " + "see_auth, cal_ui) VALUES (?,?,?,?,1,3,1,1)", + (username, fname, lname, authorized), + ) + uid = self.user_id(username) + # The sign-in row OpenEMR keeps beside each user. + self.con.execute( + "INSERT INTO users_secure (id, username, password) VALUES (?, ?, ?)", + (uid, username, f"$2y$10$seed{username}"), + ) + return uid + + def register(self, user: Optional[str], v: Mapping[str, Any]) -> int: + """The New Patient path: patient_data plus its history_data row.""" + + pid = int(self.one("SELECT COALESCE(MAX(pid), 0) + 1 FROM patient_data")) + now = self._now() + cols = { + "pid": pid, + "pubpid": str(pid), + "date": now, + "regdate": now[:10], + "title": v["title"], + "fname": v["fname"], + "lname": v["lname"], + # A US-format date field stores ISO (03/07/1984 -> 1984-03-07). + "DOB": _iso_date(str(v["DOB"])), + "sex": v["sex"], + # OpenEMR's New Patient form saves these in capitals (WITNESS.md). + "street": str(v["street"]).upper(), + "city": str(v["city"]).upper(), + "state": v["state"], + "postal_code": v["postal_code"], + "phone_home": v["phone_home"], + "email": v["email"], + "country_code": v["country_code"], + "providerID": v.get("providerID"), + } + names = ",".join(f'"{k}"' for k in cols) + cur = self.con.execute( + f"INSERT INTO patient_data ({names}) VALUES ({','.join('?' * len(cols))})", + tuple(cols.values()), + ) + self.con.execute( + "UPDATE patient_data SET uuid = ? WHERE id = ?", + (self._uuid("patient_data", cur.lastrowid), cur.lastrowid), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO patient_data pid={pid}" + ) + cur = self.con.execute( + "INSERT INTO history_data (pid, date) VALUES (?, ?)", (pid, now) + ) + self.con.execute( + "UPDATE history_data SET uuid = ? WHERE id = ?", + (self._uuid("history_data", cur.lastrowid), cur.lastrowid), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO history_data pid={pid}" + ) + # The rest of what the New Patient path writes on the pinned app + # (benchmark/openemr_local/fixture.py EXPECTED_TABLE_DELTAS["compiled"]). + self.con.execute( + "INSERT INTO contact (foreign_table_name, foreign_id) VALUES ('patient_data', ?)", + (pid,), + ) + self.log(user, "other-insert", 0, f"INSERT INTO contact foreign_id={pid}") + uid = self.one("SELECT id FROM users WHERE username = ?", (user or "",)) + if uid is not None: + self.con.execute("DELETE FROM recent_patients WHERE user_id = ?", (uid,)) + self.con.execute( + "INSERT INTO recent_patients (user_id, patients) VALUES (?, ?)", + (uid, f'[{{"pid":{pid}}}]'), + ) + self.con.execute( + "INSERT OR REPLACE INTO user_settings VALUES (?, 'dem_expand', '1')", + (uid,), + ) + self.con.execute( + "INSERT OR REPLACE INTO user_settings VALUES (?, 'his_expand', '1')", + (uid,), + ) + self.con.execute( + "INSERT INTO clinical_rules_log (date, pid, uid, category, value, new_value, " + "facility_id) VALUES (?, ?, ?, 'clinical_reminder_widget', '', '', 3)", + (now, pid, uid), + ) + self.log( + user, "patient-record-select", pid, f"SELECT * FROM patient_data pid={pid}" + ) + return pid + + def update_chart( + self, user: Optional[str], pid: int, changes: Mapping[str, Any] + ) -> None: + sets = ", ".join(f'"{k}" = ?' for k in changes) + self.con.execute( + f"UPDATE patient_data SET {sets} WHERE pid = ?", (*changes.values(), pid) + ) + self.log(user, "patient-record-update", pid, f"UPDATE patient_data pid={pid}") + + def update_history( + self, user: Optional[str], pid: int, changes: Mapping[str, Any] + ) -> None: + sets = ", ".join(f'"{k}" = ?' for k in changes) + self.con.execute( + f"UPDATE history_data SET {sets} WHERE pid = ?", (*changes.values(), pid) + ) + self.log(user, "patient-record-update", pid, f"UPDATE history_data pid={pid}") + + def delete_history(self, user: Optional[str], pid: int) -> None: + self.con.execute("DELETE FROM history_data WHERE pid = ?", (pid,)) + self.log( + user, "patient-record-delete", pid, f"DELETE FROM history_data pid={pid}" + ) + + def add_note(self, user: Optional[str], pid: int, body: str) -> None: + self.con.execute( + "INSERT INTO pnotes (date, body, pid, user, groupname, activity, authorized, " + "title, assigned_to, deleted, message_status) " + "VALUES (?,?,?,?, 'Default', 1, 1, 'Unassigned', '', 0, 'New')", + (self._now(), body, pid, user or "admin"), + ) + self.log(user, "patient-record-insert", pid, f"INSERT INTO pnotes pid={pid}") + + def add_allergy(self, user: Optional[str], pid: int, title: str) -> None: + self.con.execute( + "INSERT INTO lists (date, type, title, activity, pid, user, groupname) " + "VALUES (?, 'allergy', ?, 1, ?, ?, 'Default')", + (self._now(), title, pid, user or "admin"), + ) + self.log(user, "patient-record-insert", pid, f"INSERT INTO lists pid={pid}") + + def add_insurance(self, user: Optional[str], pid: int, provider: str) -> None: + self.con.execute( + "INSERT INTO insurance_data (type, provider, pid, date) VALUES " + "('primary', ?, ?, ?)", + (provider, pid, self._now()[:10]), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO insurance_data pid={pid}" + ) + + def add_appointment( + self, user: Optional[str], pid: int, provider_id: int, category: str + ) -> None: + """One calendar appointment, as the calendar's add form writes it.""" + + catid = 10 # 'New Patient' in the pinned fixture's categories + self.con.execute( + "INSERT INTO openemr_postcalendar_events (pc_catid, pc_aid, pc_pid, " + "pc_title, pc_time, pc_eventDate, pc_endDate, pc_duration, pc_startTime, " + "pc_endTime, pc_apptstatus, pc_facility, pc_billing_location) " + "VALUES (?,?,?,?,?, '2026-10-26', '0000-00-00', 1800, '09:00:00', " + "'09:30:00', '-', 3, 3)", + (catid, str(provider_id), str(pid), category, self._now()), + ) + self.log( + user, + "patient-record-insert", + pid, + f"INSERT INTO openemr_postcalendar_events pid={pid}", + ) + + def duplicate_history(self, user: Optional[str], pid: int) -> None: + """A second history_data row for ``pid``, a copy of the first.""" + + cols = [ + r[1] + for r in self.con.execute("PRAGMA table_info(history_data)") + if r[1] not in ("id", "uuid") + ] + names = ",".join(f'"{c}"' for c in cols) + cur = self.con.execute( + f"INSERT INTO history_data ({names}) SELECT {names} FROM history_data " + "WHERE pid = ? ORDER BY id LIMIT 1", + (pid,), + ) + self.con.execute( + "UPDATE history_data SET uuid = ? WHERE id = ?", + (self._uuid("history_data", cur.lastrowid), cur.lastrowid), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO history_data pid={pid}" + ) + + def add_history(self, user: Optional[str], pid: int) -> None: + """A history_data row for ``pid``, empty except pid and date.""" + + cur = self.con.execute( + "INSERT INTO history_data (pid, date) VALUES (?, ?)", (pid, self._now()) + ) + self.con.execute( + "UPDATE history_data SET uuid = ? WHERE id = ?", + (self._uuid("history_data", cur.lastrowid), cur.lastrowid), + ) + self.log( + user, "patient-record-insert", pid, f"INSERT INTO history_data pid={pid}" + ) + + def delete_insurance(self, user: Optional[str], pid: int) -> None: + """Delete ``pid``'s primary insurance_data row.""" + + self.con.execute( + "DELETE FROM insurance_data WHERE pid = ? AND type = 'primary'", (pid,) + ) + self.log( + user, "patient-record-delete", pid, f"DELETE FROM insurance_data pid={pid}" + ) + + def set_user_info(self, user: Optional[str], username: str, value: str) -> None: + """Set a users row's ``info``, as the user admin form saves it.""" + + self.con.execute( + "UPDATE users SET info = ? WHERE username = ?", (value, username) + ) + self.log(user, "security-administration-update", 0, f"UPDATE users {username}") + + def delete_last_allergy(self, user: Optional[str], pid: int) -> None: + """Delete ``pid``'s newest lists row, as the UI's delete action does.""" + + self.con.execute( + "DELETE FROM lists WHERE id = (SELECT MAX(id) FROM lists WHERE pid = ?)", + (pid,), + ) + self.log(user, "patient-record-delete", pid, f"DELETE FROM lists pid={pid}") + + def repoint( + self, user: Optional[str], table: str, column: str, old: int, new: int + ) -> None: + """Move ``table`` rows whose ``column`` is ``old`` to ``new``.""" + + self.con.execute( + f'UPDATE {table} SET "{column}" = ? WHERE "{column}" = ?', (new, old) + ) + self.log(user, "patient-record-update", old, f"UPDATE {table} pid={old}") diff --git a/packages/flow/benchmark/verification_coverage/independent/summarize.py b/packages/flow/benchmark/verification_coverage/independent/summarize.py new file mode 100644 index 000000000..67a04d383 --- /dev/null +++ b/packages/flow/benchmark/verification_coverage/independent/summarize.py @@ -0,0 +1,102 @@ +"""Per fault class: caught, missed, false alarm, for each check. + + python -m verification_coverage.independent.summarize ROWS [ROWS ...] + +ROWS is a ``rows.json`` (stand-in) or ``rows.jsonl`` (live). Prints Markdown. +""" + +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +CLASSES = ( + "clean", + "concurrent_legitimate", + "wrong_record", + "unrelated_change", + "misread_source", + "missing_side_record", + "duplicate", +) +CHECKS = ("shipped", "framed", "synthesized", "presave") + + +def load(path: Path) -> list[dict[str, Any]]: + text = path.read_text() + if path.suffix == ".jsonl": + return [json.loads(line) for line in text.splitlines() if line.strip()] + return json.loads(text) + + +def cell(rows: list[dict[str, Any]], check: str) -> str: + scores = [ + r["presave_score"] if check == "presave" else r[check]["score"] + for r in rows + if (check == "presave" and "presave_score" in r) + or isinstance(r.get(check), dict) + ] + if not scores: + return "n/a" + n = len(scores) + if rows[0]["expect"] == "clean": + bad = scores.count("false_alarm") + return f"{n - bad}/{n} ok" + (f", **{bad} false alarm**" if bad else "") + caught = scores.count("caught") + missed = n - caught + return f"{caught}/{n} caught" + (f", **{missed} missed**" if missed else "") + + +def table(rows: list[dict[str, Any]]) -> str: + by_class: dict[str, list[dict[str, Any]]] = defaultdict(list) + for r in rows: + if "class" in r: + by_class[r["class"]].append(r) + checks = [c for c in CHECKS if any(c == "presave" or c in r for r in rows)] + out = ["| class | cases | " + " | ".join(checks) + " |"] + out.append("|---|---|" + "---|" * len(checks)) + for cls in CLASSES: + group = by_class.get(cls) + if not group: + continue + out.append( + f"| {cls} | {len(group)} | " + + " | ".join(cell(group, c) for c in checks) + + " |" + ) + return "\n".join(out) + + +def cases(rows: list[dict[str, Any]]) -> str: + checks = [c for c in CHECKS if c != "presave" and any(c in r for r in rows)] + out = ["| case | expect | " + " | ".join(checks) + " | pre-save |"] + out.append("|---|---|" + "---|" * (len(checks) + 1)) + for r in rows: + if "class" not in r: + out.append(f"| {r['id']} | error | {r.get('error', '')[:120]} |") + continue + out.append( + f"| {r['id']} | {r['expect']} | " + + " | ".join( + f"{r[c]['outcome']} ({r[c]['score']})" + if isinstance(r.get(c), dict) + else "n/a" + for c in checks + ) + + f" | {r.get('presave')} |" + ) + return "\n".join(out) + + +def main() -> int: + for arg in sys.argv[1:]: + rows = load(Path(arg)) + print(f"### {arg}\n\n{table(rows)}\n\n{cases(rows)}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/openadapt_flow/app_profiles/__init__.py b/packages/flow/openadapt_flow/app_profiles/__init__.py new file mode 100644 index 000000000..8e9a79705 --- /dev/null +++ b/packages/flow/openadapt_flow/app_profiles/__init__.py @@ -0,0 +1,279 @@ +"""Per-app profiles: what an app writes on its own, whatever a run does. + +One profile per app, versioned, shared by the whole-database frame +(``frame.database.profile``), the rollback check and the RL grader, so all +three excuse exactly the same bookkeeping and nothing else. Every table a +profile names is compared; a rule only says which changes in it are the +app's own: + +- ``append_only``: rows may be added; an existing row may change only in the + rule's ``columns`` (none, by default) and may never go. +- ``columns``: an existing row may change in the named columns, and no row + may be added or removed. Narrowed by ``fill_only`` (only from NULL or empty + to a value: a backfill, never a swap, overwrite or clear), ``with_fill`` + (only when that column was backfilled in the same row) or ``owner`` (only + on the acting user's own row). +- ``own_rows``: per-user UI state. The acting user's own rows may be added, + changed or removed, and so may those of another user who wrote in the + window (their own work); everyone else's are compared. Credentials never + use this: their column rules stay on the acting user's row. + +A rule may name only a table in the profile's reviewed ``bookkeeping_tables`` +(lower-cased, unqualified), so anything else, patient data included, is +never excused; ``*`` may name only system columns, as a backfill. Profiles +load only from this package, by name, and a contract pins the digest. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from functools import lru_cache +from pathlib import Path +from typing import Any, Callable, Literal, Mapping, Optional, Sequence + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + +HERE = Path(__file__).resolve().parent +_NAME = re.compile(r"^[a-z0-9_-]+$") + + +def table_name(name: str) -> str: + """How a profile compares table names: lower-cased, without a schema.""" + + return name.rsplit(".", 1)[-1].lower() + + +class Owner(BaseModel): + model_config = ConfigDict(extra="forbid") + + #: The column naming the row's user. + column: str + #: What it holds: the user's sign-in name, or their id in the users table. + by: Literal["username", "user_id"] + + +class ProfileRule(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + #: A table name, or ``*`` for a system-column backfill in every table. + table: str + mode: Literal["append_only", "columns", "own_rows"] + columns: list[str] = Field(default_factory=list) + fill_only: bool = False + with_fill: Optional[str] = None + owner: Optional[Owner] = None + reason: str = Field(min_length=1) + + @model_validator(mode="after") + def _shape(self) -> "ProfileRule": + if self.mode == "columns" and not self.columns: + raise ValueError(f"rule {self.id}: a column rule names its columns") + if self.mode == "own_rows" and (self.owner is None or self.columns): + raise ValueError(f"rule {self.id}: own_rows names an owner, not columns") + if self.mode != "columns" and (self.fill_only or self.with_fill): + raise ValueError(f"rule {self.id}: fill conditions go with column rules") + if self.table == "*" and not (self.mode == "columns" and self.fill_only): + raise ValueError(f"rule {self.id}: every table only for a backfill") + return self + + @property + def conditional(self) -> bool: + return bool(self.fill_only or self.with_fill or self.owner) or ( + self.mode == "own_rows" + ) + + +class UsersTable(BaseModel): + model_config = ConfigDict(extra="forbid") + + table: str + id_column: str + name_column: str + + +class AppProfile(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.app-profile/v2"] + app: str + version: str + #: The reviewed tables the app writes on its own. A rule may name only + #: these; every other table, patient data included, is never excused. + bookkeeping_tables: list[str] + #: Columns a ``*`` backfill rule may name. + system_columns: list[str] = Field(default_factory=list) + #: Where the acting user's id is looked up from their sign-in name. + users: Optional[UsersTable] = None + rules: list[ProfileRule] + + @model_validator(mode="after") + def _guard(self) -> "AppProfile": + allowed = {table_name(t) for t in self.bookkeeping_tables} + for rule in self.rules: + if rule.table == "*": + extra = set(rule.columns) - set(self.system_columns) + if extra: + raise ValueError( + f"rule {rule.id}: every table only for system columns, " + f"not {sorted(extra)}" + ) + elif table_name(rule.table) not in allowed: + raise ValueError( + f"rule {rule.id}: {rule.table!r} is not a reviewed bookkeeping " + "table, so the rule could excuse patient data" + ) + if rule.owner is not None and rule.owner.by == "user_id" and not self.users: + raise ValueError(f"rule {rule.id}: owner by user_id needs users") + ids = [r.id for r in self.rules] + if len(ids) != len(set(ids)): + raise ValueError("rule ids must be unique") + return self + + def sha256(self) -> str: + payload = json.dumps(self.model_dump(mode="json"), sort_keys=True) + return "sha256:" + hashlib.sha256(payload.encode()).hexdigest() + + def allow_tables(self, present: Optional[set[str]] = None) -> list[dict[str, str]]: + """``frame.database.allow_tables`` entries for append-only tables.""" + + return [ + { + "table": r.table, + "mode": "append_only", + "reason": f"{self.version} {r.id}: {r.reason}", + } + for r in self.rules + if r.mode == "append_only" and (present is None or r.table in present) + ] + + def ignore_columns(self) -> dict[str, list[str]]: + """Columns left out of each row's contents MAC: unconditional column + rules, and the columns an append-only table's rows may change in.""" + + out: dict[str, list[str]] = {} + for r in self.rules: + if r.columns and not r.conditional: + out.setdefault(r.table, []).extend(r.columns) + return out + + def ref_columns(self) -> list[str]: + """Columns the conditional rules read: kept as cell MACs in every + table, however large.""" + + cols: set[str] = set() + for r in self.rules: + if r.conditional: + cols |= set(r.columns) + cols |= {r.with_fill} if r.with_fill else set() + cols |= {r.owner.column} if r.owner else set() + return sorted(cols) + + def actor_ids_query(self, names: Sequence[str], dialect: str) -> Optional[str]: + """The SELECT that finds the acting user's ids, read in the snapshot.""" + + if self.users is None or not names: + return None + quote = (lambda n: f"`{n}`") if dialect == "mysql" else (lambda n: f'"{n}"') + listed = ", ".join("'" + n.replace("'", "''") + "'" for n in names) + u = self.users + return ( + f"SELECT {quote(u.id_column)} AS id FROM {quote(u.table)} " + f"WHERE {quote(u.name_column)} IN ({listed})" + ) + + def users_query(self, dialect: str) -> Optional[str]: + """The SELECT of every user's id and sign-in name, read in the + snapshot (in memory only), to find a colleague's own rows.""" + + if self.users is None: + return None + quote = (lambda n: f"`{n}`") if dialect == "mysql" else (lambda n: f'"{n}"') + u = self.users + return ( + f"SELECT {quote(u.id_column)} AS id, {quote(u.name_column)} AS name " + f"FROM {quote(u.table)}" + ) + + def excuse( + self, + table: str, + before: Optional[Mapping[str, Any]], + after: Optional[Mapping[str, Any]], + fields: Sequence[str], + hasher: Callable[[str, Any], str], + names: set[str], + ids: set[str], + colleagues: tuple[set[str], set[str]] = (set(), set()), + ) -> Optional[list[str]]: + """The rules that make this row change the app's own, or None. + + ``before`` holds the baseline's cell MACs (``hasher`` MACs a value + for a column of this table); ``after`` holds plain values. A row + added has no ``before``, a row removed no ``after``; ``fields`` are + the columns that changed. ``names`` and ``ids`` are the acting + user's; ``colleagues`` the names and ids of other users who wrote in + the window (for ``own_rows`` only). + """ + + name = table_name(table) + rules = [r for r in self.rules if r.table == "*" or table_name(r.table) == name] + + def owned(rule: ProfileRule) -> bool: + assert rule.owner is not None + who = names if rule.owner.by == "username" else ids + if rule.mode == "own_rows": + who = who | ( + colleagues[0] if rule.owner.by == "username" else colleagues[1] + ) + col = rule.owner.column + if after is not None: + return str(after.get(col)) in who + return before is not None and any( + hasher(col, w) == before.get(col) for w in who + ) + + def filled(col: str) -> bool: + if before is None or after is None: + return False + empty = before.get(col) in (hasher(col, None), hasher(col, "")) + return empty and after.get(col) not in (None, "") + + if before is None or after is None: + for rule in rules: + if rule.mode == "own_rows" and owned(rule): + return [rule.id] + return None + used: list[str] = [] + for field in fields: + for rule in rules: + if rule.mode == "own_rows": + ok = owned(rule) + elif field not in rule.columns or not rule.conditional: + continue + else: + ok = (not rule.fill_only or filled(field)) and ( + not rule.with_fill or filled(rule.with_fill) + ) + ok = ok and (rule.owner is None or owned(rule)) + if ok: + used.append(rule.id) + break + else: + return None + return sorted(set(used)) + + +@lru_cache(maxsize=None) +def load_profile(name: str) -> AppProfile: + """A profile shipped in this package, by app name (``openemr``).""" + + if not _NAME.match(name): + raise ValueError(f"an app profile is named by app ([a-z0-9_-]), not {name!r}") + path = HERE / f"{name}.yaml" + if not path.exists(): + raise ValueError(f"no app profile named {name!r}") + return AppProfile.model_validate(yaml.safe_load(path.read_text())) diff --git a/packages/flow/openadapt_flow/app_profiles/openemr.yaml b/packages/flow/openadapt_flow/app_profiles/openemr.yaml new file mode 100644 index 000000000..32d975bc4 --- /dev/null +++ b/packages/flow/openadapt_flow/app_profiles/openemr.yaml @@ -0,0 +1,51 @@ +# OpenEMR's own bookkeeping: what it writes for any signed-in user, whatever +# a run does. The whole-database frame, the rollback check and the RL grader +# load this one profile. Measured on the pinned OpenEMR 8.0.0.3 fixture +# (synthetic patients): the RL environment's calibration episodes and the +# frame's live witness (2026-10-28). Each rule is the narrowest that held. +# +# Every table here is compared. Credentials and access: users_secure excuses +# only sign-in counters and timestamps on the acting user's own row; keys, +# MFA registrations and the OAuth tables are compared in full. A rule may name only a table listed in +# bookkeeping_tables; every other table, patient data included, is never +# excused. +schema_version: openadapt.app-profile/v2 +app: openemr +version: openemr-frame-profile/2026-10-29.2 +bookkeeping_tables: [log, log_comment_encrypt, api_log, extended_log, audit_master, + audit_details, clinical_rules_log, uuid_registry, uuid_mapping, contact, + session_tracker, ip_tracking, background_services, sequences, + users_secure, user_settings, recent_patients, users] +system_columns: [uuid] +users: {table: users, id_column: id, name_column: username} +rules: + # Logs and id maps: rows are added, never changed or removed. + - {id: log, table: log, mode: append_only, reason: "audit log"} + - {id: log-comment-encrypt, table: log_comment_encrypt, mode: append_only, reason: "audit log checksums"} + - {id: api-log, table: api_log, mode: append_only, reason: "API audit log"} + - {id: extended-log, table: extended_log, mode: append_only, reason: "activity log"} + - {id: audit-master, table: audit_master, mode: append_only, reason: "audit log"} + - {id: audit-details, table: audit_details, mode: append_only, reason: "audit log"} + - {id: clinical-rules-log, table: clinical_rules_log, mode: append_only, reason: "the reminder widget's log, written when a chart is shown"} + - {id: uuid-registry, table: uuid_registry, mode: append_only, reason: "system id map, written with any new record or a uuid backfill"} + - {id: uuid-mapping, table: uuid_mapping, mode: append_only, reason: "system id map"} + - {id: contact, table: contact, mode: append_only, reason: "a link row (table name and record id, no address) created when a chart is first opened"} + # Session and scheduler state: only the columns that move. + - {id: session-tracker, table: session_tracker, mode: append_only, columns: [last_updated, number_scripts], reason: "one row per sign-in session, its counters updated while it lasts"} + - {id: ip-tracking, table: ip_tracking, mode: append_only, columns: [total_ip_login_fail_counter, ip_login_fail_counter, ip_last_login_fail], reason: "sign-in failure counters per address"} + - {id: background-services, table: background_services, mode: columns, columns: [next_run, running], reason: "the scheduler's next run and busy flag; enabling or disabling a service is not excused"} + - {id: sequences, table: sequences, mode: columns, columns: [id], reason: "one id counter row"} + # Credentials: only sign-in counters and timestamps, on the acting user's row. + - id: users-secure-sign-in + table: users_secure + mode: columns + columns: [last_update, last_challenge_response, total_login_fail_counter, login_fail_counter, last_login_fail, auto_block_emailed] + owner: {column: username, by: username} + reason: "the acting user's own sign-in bookkeeping; a password or its history is never excused" + # Per-user UI state: the acting user's own rows only. + - {id: user-settings, table: user_settings, mode: own_rows, owner: {column: setting_user, by: user_id}, reason: "the acting user's own UI settings"} + - {id: recent-patients, table: recent_patients, mode: own_rows, owner: {column: user_id, by: user_id}, reason: "the acting user's own list of charts opened"} + # Backfills: a system uuid filled from NULL, never swapped, overwritten or cleared. + - {id: users-uuid-backfill, table: users, mode: columns, columns: [uuid], fill_only: true, reason: "uuid backfill on a user row"} + - {id: users-last-updated, table: users, mode: columns, columns: [last_updated], with_fill: uuid, reason: "the ON UPDATE stamp a uuid backfill bumps"} + - {id: uuid-backfill, table: "*", mode: columns, columns: [uuid], fill_only: true, reason: "a row's system uuid, backfilled when a record is first opened"} diff --git a/packages/flow/openadapt_flow/provenance.py b/packages/flow/openadapt_flow/provenance.py index 8af4abbbd..4adb1145c 100644 --- a/packages/flow/openadapt_flow/provenance.py +++ b/packages/flow/openadapt_flow/provenance.py @@ -99,6 +99,9 @@ class ParamProvenance(BaseModel): extractor: str = Field(min_length=1) #: ``text`` | ``code`` | ``date`` | ``currency``. Unset: inferred. value_kind: Optional[Literal["text", "code", "date", "currency"]] = None + #: How the source document writes a numeric date (``03/07/1984``): + #: ``mdy`` or ``dmy``. Unset, a date that reads two ways is ambiguous. + date_order: Optional[Literal["mdy", "dmy"]] = None #: The text as the source wrote it, when intake converted it (``value`` is #: the converted form, for example a date as ISO). raw_value: Optional[str] = None @@ -247,7 +250,7 @@ def same_value(a: str, b: str, kind: str) -> bool: class FieldCheck: name: str #: ``agree`` | ``disagree`` | ``missing`` | ``instruction`` | - #: ``not_independent`` | ``no_document`` + #: ``not_independent`` | ``no_document`` | ``ambiguous`` result: str #: In memory only: never put in a receipt (values are data). first: Optional[str] = None @@ -345,17 +348,48 @@ def summary(self) -> dict[str, Any]: } -def bound_fields(effects: Any) -> dict[str, str]: - """Parameter name -> record field, from the effects' selectors and values.""" +def bindings(effects: Any) -> dict[str, tuple[str, Any]]: + """Parameter name -> (record field, value expression), each pair taken + from one place in one effect, so a field's transform is never another + field's. A parameter bound two different ways is left out: its round + trip can't be compared, so it stays unchecked.""" + + out: dict[str, tuple[str, Any]] = {} + ambiguous: set[str] = set() + + def add(param: str, record_field: str, expr: Any) -> None: + seen = out.get(param) + transform = (expr.map, expr.stored) + if seen is None: + out[param] = (record_field, expr) + elif seen[0] != record_field or (seen[1].map, seen[1].stored) != transform: + ambiguous.add(param) - out: dict[str, str] = {} for effect in effects: for record_field, expr in effect.match.items(): if expr.param is not None: - out.setdefault(expr.param, record_field) + add(expr.param, record_field, expr) if effect.field and effect.value is not None and effect.value.param: - out.setdefault(effect.value.param, effect.field) - return out + add(effect.value.param, effect.field, effect.value) + return {k: v for k, v in out.items() if k not in ambiguous} + + +def bound_fields(effects: Any) -> dict[str, str]: + """Parameter name -> record field (:func:`bindings`).""" + + return {param: field for param, (field, _expr) in bindings(effects).items()} + + +def bound_maps(effects: Any) -> dict[str, Any]: + """Parameter name -> the value expression that says how its field stores + what was entered: a label's code (``map``), a stored form (``stored``). + From the same effect as the field (:func:`bindings`).""" + + return { + param: expr + for param, (_field, expr) in bindings(effects).items() + if expr.map or expr.stored or expr.kind + } def round_trip( @@ -364,6 +398,7 @@ def round_trip( fields: Mapping[str, str], documents: Mapping[str, Any], second: Extractor, + maps: Optional[Mapping[str, Any]] = None, ) -> RoundTrip: """Compare ``record`` (as read back) with the source of each value. @@ -398,9 +433,46 @@ def round_trip( checks.append(FieldCheck(name, "no_document")) continue source_value = second.read(document, name, prov.source) + mapping = (maps or {}).get(name) + kind = None if isinstance(mapping, Mapping) else getattr(mapping, "kind", None) + if mapping is not None and source_value is not None: + # The document shows the label, the record stores its code (a + # state's name and "MA", a provider's name and their id), or a + # stored form (a street in capitals). + source_value = ( + mapping.get(source_value.strip()) + if isinstance(mapping, Mapping) + else mapping.stored_form(source_value.strip()) + ) stored = "" if record.get(record_field) is None else str(record[record_field]) if source_value is None: checks.append(FieldCheck(name, "missing", stored)) + elif kind is not None: + # A declared kind: the stored value against the document's, in + # canonical form. The document is read in its own date order (the + # provenance's), never the app's; without one, a date it prints + # two ways is ambiguous and stays unchecked. + from openadapt_flow.runtime.effects.readback_values import compare_value + + verdict = compare_value( + stored, + source_value, + kind=kind, + source="structure", + date_order=prov.date_order, + ) + checks.append( + FieldCheck( + name, + "agree" + if verdict == "match" + else "disagree" + if verdict == "mismatch" + else "ambiguous", + stored, + source_value, + ) + ) elif same_value(stored, source_value, _kind(prov, source_value)): checks.append(FieldCheck(name, "agree", stored, source_value)) else: @@ -441,12 +513,16 @@ def default_extractor(provenance: ProvenanceFile) -> LabelExtractor: specs: dict[str, Any] = {} aliases: dict[str, list[str]] = {} + formats: dict[str, str] = {} for name, prov in provenance.params.items(): kind = ParamKind.DATE if prov.value_kind == "date" else ParamKind.STRING specs[name] = ParamSpec(name=name, type=kind) if prov.source.label: aliases[name] = [prov.source.label] - return LabelExtractor(specs, aliases) + if prov.date_order: + # The document's own order, never the app's. + formats[name] = "MM/DD/YYYY" if prov.date_order == "mdy" else "DD/MM/YYYY" + return LabelExtractor(specs, aliases, date_formats=formats) def read_sources(paths: list[Path]) -> dict[str, Any]: diff --git a/packages/flow/openadapt_flow/qualify/campaign.py b/packages/flow/openadapt_flow/qualify/campaign.py index 0c8f7d44f..926839b1e 100644 --- a/packages/flow/openadapt_flow/qualify/campaign.py +++ b/packages/flow/openadapt_flow/qualify/campaign.py @@ -208,6 +208,13 @@ def record_effects(config: VerifyConfig) -> list[Any]: timeout_s=record.timeout_s, ) ) + if record.surface: + from openadapt_flow.runtime.effects.surfaces import surface_probe + + effects = [ + e.model_copy(update={"probe": surface_probe(record.surface, e.probe or "")}) + for e in effects + ] return effects diff --git a/packages/flow/openadapt_flow/qualify/config.py b/packages/flow/openadapt_flow/qualify/config.py index fb816b099..e0aa1a0ac 100644 --- a/packages/flow/openadapt_flow/qualify/config.py +++ b/packages/flow/openadapt_flow/qualify/config.py @@ -231,6 +231,10 @@ class RecordConfig(BaseModel): #: The field that tells a new record from one that was already there, for #: ``exact_new_set`` (an id the system of record assigns). identity_field: str = "id" + #: With ``verifier.kind: surfaces``, the surface the record is read from. + #: The other surfaces are read for the frame (writes elsewhere) and side + #: records. Unset: the verifier's single read. + surface: Optional[str] = Field(default=None, pattern=r"^[A-Za-z0-9_.-]{1,64}$") def expr(self, field: str) -> ValueExpr: value = self.fields[field] diff --git a/packages/flow/openadapt_flow/runtime/effects/_common.py b/packages/flow/openadapt_flow/runtime/effects/_common.py index 212477c07..dec788278 100644 --- a/packages/flow/openadapt_flow/runtime/effects/_common.py +++ b/packages/flow/openadapt_flow/runtime/effects/_common.py @@ -100,7 +100,7 @@ def judge_records( matched = [ r for r in matched - if str(r.get(effect.key_field, None)) == str(effect.idempotency_key) + if record_matches(r, _matchable({effect.key_field: effect.idempotency_key})) ] # Duplicate-write guard (count_new_only): judge only the records that @@ -133,6 +133,13 @@ def _selector_text(selector: dict[str, Any]) -> str: return "{" + ", ".join(f"{k}={v}" for k, v in sorted(selector.items())) + "}" +def _matchable(selector: dict[str, Any]) -> dict[str, Any]: + """A selector for :func:`record_matches`: a value with a declared kind + stays an expression (compared in canonical form), others become text.""" + + return {k: (v if getattr(v, "kind", None) else str(v)) for k, v in selector.items()} + + def _judge_exact_new_set( effect: Effect, before: EffectState, @@ -193,7 +200,7 @@ def _judge_exact_new_set( for selector in effect.new_records: plain = {k: str(v) for k, v in selector.items()} key = _selector_text(plain) - found, multiplicity = declared.get(key, (plain, 0)) + found, multiplicity = declared.get(key, (_matchable(selector), 0)) declared[key] = (found, multiplicity + 1) for key, (selector, multiplicity) in sorted(declared.items()): observed = sum(1 for r in added if record_matches(r, selector)) @@ -208,8 +215,7 @@ def _judge_exact_new_set( r for r in added if not any( - record_matches(r, {k: str(v) for k, v in selector.items()}) - for selector in effect.new_records + record_matches(r, _matchable(selector)) for selector in effect.new_records ) ] if extra: @@ -467,7 +473,9 @@ def _judge_field_equals( ) record = matched[0] observed = "" if record.get(effect.field) is None else str(record.get(effect.field)) - if observed == want: + kind = getattr(effect.value, "kind", None) + same = effect.value.matches(record.get(effect.field)) if kind else observed == want # type: ignore[union-attr] + if same: return EffectVerdict( verdict=Verdict.CONFIRMED, kind=effect.kind, diff --git a/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py b/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py new file mode 100644 index 000000000..a06807a29 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/effects/db_snapshot.py @@ -0,0 +1,874 @@ +"""Snapshot a whole database for the frame: every table, every column. + +A frame that compares only the surfaces a contract reads can't see a write +to anything else: a column the effects don't name, a field on a side row, a +table nobody thought to read. ``frame.database`` turns the frame into a diff +of the whole database. This module takes the snapshot; ``frame.judge_frame`` +compares two of them like any other slice. + +What a snapshot holds, and why: + +- **No values on disk.** Each row is kept as a keyed MAC of its key and of + its contents (without ``ignore_columns``), and each cell as a keyed MAC + bound to its table and column: BLAKE2b, 128 bits, keyed by + :func:`mac_key` from the run's salt and a host secret that is never in the + baseline. A table over ``max_rows`` keeps cell MACs only for the columns + attribution and side records read (``ref_columns``). Plain values are read + in full, but kept in memory only for rows that differ from the baseline. +- **Rows by key.** A changed row is known by its key in every table, so it + goes through attribution, and an added row is found by key for any + primary key. +- **One consistent read.** Everything is read on one connection inside one + read-only transaction (``START TRANSACTION WITH CONSISTENT SNAPSHOT, READ + ONLY`` on MySQL and MariaDB; ``BEGIN`` on SQLite). The audit log is read in + the same transaction (``extra_queries``). MySQL's ``AUTO_INCREMENT`` isn't + part of a snapshot, so it is read just before the snapshot starts: a row + committed in between is in the snapshot, never missing from it. +- **Schema and code.** Each table's definition and the database's triggers, + views, routines and events are digested. On MySQL a trigger is visible + only to an account with ``TRIGGER``; without it the snapshot says so + (``code_visible``), and the frame is not checked. +- **Counters.** ``sqlite_sequence`` and ``AUTO_INCREMENT`` (on the column the + database numbers, key or not) show a row written and deleted in the window. +- **The clock.** ``CURRENT_TIMESTAMP`` at the snapshot, so a column the + database fills with the time can be held to the window. +- **Allow-listed tables** are append-only by default: their existing rows + must not change or go. ``skip`` reads nothing and needs a stated reason. + +Read-only: SQLite files open with ``mode=ro``; a driver connection should use +an account with SELECT (and, on MySQL, TRIGGER and SHOW VIEW) on every schema +the app can write. The baseline must still be protected like the records: +whoever holds both it and the host secret can test guesses against the MACs. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import secrets +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Literal, Optional + +Dialect = Literal["sqlite", "mysql"] +TableMode = Literal["compare", "append_only", "skip"] + +#: Columns per JSON object call: SQLite caps a function at 127 arguments. +_CHUNK = 50 +#: The synthetic identity field of a database slice's records. +KEY = "__row_key__" +#: The row's contents MAC in a database slice's records. +ROW = "__row_mac__" +#: A column default meaning any value: the table's own generated key. +ANY_VALUE = "\x00any" +#: A column default the database computes as the current time at insert. +CLOCK_DEFAULT = "\x00clock" +_SYSTEM_SCHEMAS = ("information_schema", "mysql", "performance_schema", "sys") +#: Hex characters per MAC (128 bits). +MAC_HEX = 32 + + +class SnapshotError(RuntimeError): + """The database can't be snapshotted as the frame needs.""" + + +# -- keys and MACs ------------------------------------------------------------ + + +def _host_secret() -> bytes: + """The host's frame secret: ``OPENADAPT_FRAME_SECRET`` (hex), the file + ``OPENADAPT_FRAME_SECRET_FILE``, or ``~/.config/openadapt/frame.key`` + (created owner-only on first use). It is never written into a baseline. + + The default file only keeps the secret out of the baseline: an agent + running as the same OS user can read it. A deployment sets + ``OPENADAPT_FRAME_SECRET`` (or the file) from a store the agent's account + can't read, so a baseline the agent can see still can't be tested + against guesses.""" + + env = os.environ.get("OPENADAPT_FRAME_SECRET") + if env: + return bytes.fromhex(env) + path = Path( + os.environ.get("OPENADAPT_FRAME_SECRET_FILE") + or Path.home() / ".config" / "openadapt" / "frame.key" + ) + if not path.exists(): + # Written whole to a private temp file, then linked into place: a + # process that loses the race reads the winner's secret, never a + # half-written one. + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.{os.getpid()}.{secrets.token_hex(4)}") + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(secrets.token_bytes(32)) + try: + os.link(tmp, path) + except FileExistsError: + pass + finally: + os.unlink(tmp) + secret = path.read_bytes() + if len(secret) < 32: + raise SnapshotError(f"the frame secret {path} is too short") + return secret + + +def mac_key(salt: bytes) -> bytes: + """The run's MAC key: the salt (in the baseline) keyed by the host + secret (not in it), so the baseline alone can't be tested against.""" + + return hashlib.blake2b(salt, key=_host_secret()[:64], digest_size=32).digest() + + +def canonical(value: Any) -> str: + """One text form per value, so equal values hash equally.""" + + if value is None: + return "\x00null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float, Decimal)): + number = Decimal(str(value)) + return format(number.normalize(), "f") if number == number else str(value) + if isinstance(value, (dict, list)): + return json.dumps(value, sort_keys=True, default=str) + return str(value) + + +def _mac(key: bytes, text: str) -> str: + return hashlib.blake2b( + text.encode(), digest_size=MAC_HEX // 2, key=key[:64] + ).hexdigest() + + +def cell_hash(key: bytes, value: Any, table: str = "", column: str = "") -> str: + """A cell's keyed MAC, bound to its table and column, so equal values in + different places don't share one.""" + + return _mac(key, f"c\x1f{table}\x1f{column}\x1f{canonical(value)}") + + +def row_key_hash(key: bytes, text: str, table: str = "") -> str: + """A row key's keyed MAC.""" + + return _mac(key, f"k\x1f{table}\x1f{text}") + + +def row_hash( + key: bytes, table: str, row: Mapping[str, Any], ignore: Sequence[str] = () +) -> str: + """A row's contents MAC, without ``ignore`` columns.""" + + skip = set(ignore) + body = json.dumps( + sorted((c, canonical(v)) for c, v in row.items() if c not in skip), + separators=(",", ":"), + ) + return _mac(key, f"r\x1f{table}\x1f{body}") + + +def unpack(columns: Sequence[str], packed: str) -> dict[str, str]: + """A stored row's cell MACs by column (stored as one string per row).""" + + return {c: packed[i * MAC_HEX : (i + 1) * MAC_HEX] for i, c in enumerate(columns)} + + +# -- shapes ------------------------------------------------------------------- + + +@dataclass +class Table: + mode: TableMode + #: Digest of the table's definition: column names, types, defaults, + #: nullability and extras. + schema: str + columns: list[str] + primary_key: list[str] + #: The single integer key the database generates, when there is one. + generated_key: Optional[str] + #: Column -> declared default as text (None: no default). Schema, not data. + defaults: dict[str, Optional[str]] + count: int = 0 + #: ``sqlite_sequence`` or ``AUTO_INCREMENT``, when the table has one. + counter: Optional[int] = None + #: The column the database numbers, and its largest value. + auto_column: Optional[str] = None + max_auto: Optional[int] = None + #: The columns each row keeps cell MACs for (all, or ``ref_columns`` for + #: a table over ``max_rows``; none for an append-only table). + cell_columns: list[str] = field(default_factory=list) + #: Row key MAC -> [row contents MAC, cell MACs of ``cell_columns`` + #: concatenated (:func:`unpack`)]. + rows: dict[str, list[str]] = field(default_factory=dict) + #: In memory only, never saved: plain values of the rows that differ + #: from the baseline (all rows of a read with no baseline: none kept). + values: dict[str, dict[str, Any]] = field(default_factory=dict, repr=False) + #: The engine keeps no snapshot isolation for it (MyISAM, MEMORY). + non_transactional: bool = False + + +@dataclass +class Snapshot: + dialect: str = "sqlite" + clock: str = "" + #: Digest of triggers, views, routines and events, and whether the + #: account could see triggers at all. + objects: str = "" + code_visible: bool = True + schemas: list[str] = field(default_factory=list) + tables: dict[str, Table] = field(default_factory=dict) + #: In memory only: rows of ``extra_queries``, read in the same transaction. + extra: dict[str, list[dict[str, Any]]] = field(default_factory=dict, repr=False) + + def to_json(self) -> dict[str, Any]: + """The persisted form: MACs and schema only, never values.""" + + return { + "dialect": self.dialect, + "clock": self.clock, + "objects": self.objects, + "code_visible": self.code_visible, + "schemas": self.schemas, + "tables": { + name: { + "mode": t.mode, + "schema": t.schema, + "columns": t.columns, + "primary_key": t.primary_key, + "generated_key": t.generated_key, + "defaults": t.defaults, + "count": t.count, + "counter": t.counter, + "auto_column": t.auto_column, + "max_auto": t.max_auto, + "cell_columns": t.cell_columns, + "rows": t.rows, + "non_transactional": t.non_transactional, + } + for name, t in self.tables.items() + }, + } + + @classmethod + def from_json(cls, data: Mapping[str, Any]) -> "Snapshot": + return cls( + dialect=str(data.get("dialect", "sqlite")), + clock=str(data.get("clock", "")), + objects=str(data.get("objects", "")), + code_visible=bool(data.get("code_visible", True)), + schemas=list(data.get("schemas", [])), + tables={ + name: Table( + mode=t["mode"], + schema=t["schema"], + columns=list(t["columns"]), + primary_key=list(t["primary_key"]), + generated_key=t.get("generated_key"), + defaults=dict(t["defaults"]), + count=int(t["count"]), + counter=t.get("counter"), + auto_column=t.get("auto_column"), + max_auto=t.get("max_auto"), + cell_columns=list(t.get("cell_columns", [])), + rows={k: list(v) for k, v in (t.get("rows") or {}).items()}, + non_transactional=bool(t.get("non_transactional")), + ) + for name, t in data["tables"].items() + }, + ) + + +# -- defaults ----------------------------------------------------------------- + + +def _number(text: str) -> Optional[Decimal]: + try: + return Decimal(text) + except (InvalidOperation, ValueError): + return None + + +def _clock_text(value: Any) -> str: + return str(value).replace("T", " ")[:19] + + +def is_default( + value: Any, default: Optional[str], clock: tuple[str, str] = ("", "") +) -> bool: + """Does a new row's ``value`` equal what the column holds when nothing + sets it? + + Only the declared default counts: NULL when the column has none, the + stored text otherwise (numbers compared as numbers). A default the + database computes as the time counts only when the value falls between + the two snapshots' clocks. The table's own generated key is any value. + """ + + if default == ANY_VALUE: + return True + if default == CLOCK_DEFAULT: + low, high = clock + return ( + bool(low and high) + and value is not None + and (low <= _clock_text(value) <= high) + ) + if default is None: + return value is None + if value is None: + return False + a, b = canonical(value), default + na, nb = _number(a), _number(b) + if na is not None and nb is not None: + return na == nb + return a == b + + +def _default_text(raw: Any, dialect: Dialect) -> Optional[str]: + """A declared default as the value it stores, or ``CLOCK_DEFAULT``.""" + + if raw is None: + return None + text = str(raw).strip() + lowered = text.lower().strip("()") + if lowered.startswith( + ("current_timestamp", "now", "current_date", "localtime", "datetime('now'") + ): + return CLOCK_DEFAULT + if text.upper() == "NULL": + return None # SQLite and MariaDB show DEFAULT NULL as the text NULL + if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"": + return text[1:-1].replace("''", "'") + return text + + +# -- reading ------------------------------------------------------------------ + + +def _quote(dialect: Dialect, name: str) -> str: + if dialect == "mysql": + return ".".join("`" + p.replace("`", "``") + "`" for p in name.split(".", 1)) + return '"' + name.replace('"', '""') + '"' + + +def _literal(text: str) -> str: + return "'" + text.replace("'", "''") + "'" + + +class _Session: + """One connection, one read-only transaction (opened by :meth:`begin`).""" + + def __init__(self, connect: Callable[[], Any], dialect: Dialect) -> None: + self.dialect = dialect + self.con = connect() + self.cur = self.con.cursor() + + def begin(self) -> None: + if self.dialect == "mysql": + # Only statements that can't fail: a driver may end the session + # on any error. MySQL 8 caches AUTO_INCREMENT unless told not to; + # MariaDB has no such variable. + self.cur.execute( + "SHOW SESSION VARIABLES LIKE 'information_schema_stats_expiry'" + ) + if self.cur.fetchall(): + self.cur.execute("SET SESSION information_schema_stats_expiry = 0") + self.cur.execute("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ") + self.cur.execute("START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY") + else: + self.cur.execute("BEGIN") + + def rows(self, sql: str) -> list[tuple[Any, ...]]: + self.cur.execute(sql) + return [tuple(r) for r in self.cur.fetchall()] + + def dicts(self, sql: str) -> list[dict[str, Any]]: + self.cur.execute(sql) + names = [d[0] for d in self.cur.description or []] + return [dict(zip(names, r)) for r in self.cur.fetchall()] + + def close(self) -> None: + try: + self.cur.execute("COMMIT" if self.dialect == "mysql" else "ROLLBACK") + finally: + self.con.close() + + +@dataclass +class _Def: + columns: list[str] + primary_key: list[str] + generated_key: Optional[str] + defaults: dict[str, Optional[str]] + binary: set[str] + schema: str + non_transactional: bool = False + #: The column the database numbers (AUTO_INCREMENT, or SQLite's rowid + #: alias), whether or not it is the primary key. + auto_column: Optional[str] = None + + +def _digest(parts: Sequence[Any]) -> str: + return hashlib.sha256( + json.dumps(list(parts), sort_keys=True, default=str).encode() + ).hexdigest() + + +def _definitions(s: _Session) -> tuple[dict[str, _Def], str]: + """Every table's definition, and the digest of the database's code.""" + + out: dict[str, _Def] = {} + if s.dialect == "mysql": + default = s.rows("SELECT DATABASE()")[0][0] + excluded = ", ".join(_literal(x) for x in _SYSTEM_SCHEMAS) + engines = { + (str(a), str(b)): str(c or "") + for a, b, c in s.rows( + "SELECT TABLE_SCHEMA, TABLE_NAME, ENGINE FROM information_schema.TABLES " + f"WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA NOT IN ({excluded})" + ) + } + for row in s.rows( + "SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, COLUMN_DEFAULT, " + "IS_NULLABLE, EXTRA, COLUMN_KEY, DATA_TYPE FROM information_schema.COLUMNS " + f"WHERE TABLE_SCHEMA NOT IN ({excluded}) " + "ORDER BY TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION" + ): + schema, table, column, ctype, cdefault, nullable, extra, key, dtype = row + if (str(schema), str(table)) not in engines: + continue # a view + name = str(table) if schema == default else f"{schema}.{table}" + d = out.get(name) + if d is None: + engine = engines[(str(schema), str(table))].lower() + d = out[name] = _Def( + [], [], None, {}, set(), "", engine not in ("innodb", "") + ) + d.columns.append(str(column)) + d.defaults[str(column)] = _default_text(cdefault, "mysql") + if str(key) == "PRI": + d.primary_key.append(str(column)) + if "auto_increment" in str(extra).lower(): + d.generated_key = str(column) + d.auto_column = str(column) + if "binary" in str(dtype).lower() or "blob" in str(dtype).lower(): + d.binary.add(str(column)) + d.schema = _digest( + [d.schema, column, ctype, cdefault, nullable, extra, key] + ) + code = [ + s.rows( + "SELECT TRIGGER_SCHEMA, TRIGGER_NAME, EVENT_MANIPULATION, " + "EVENT_OBJECT_TABLE, ACTION_TIMING, ACTION_STATEMENT " + "FROM information_schema.TRIGGERS ORDER BY 1, 2" + ), + s.rows( + "SELECT TABLE_SCHEMA, TABLE_NAME, VIEW_DEFINITION " + f"FROM information_schema.VIEWS WHERE TABLE_SCHEMA NOT IN ({excluded}) " + "ORDER BY 1, 2" + ), + s.rows( + "SELECT ROUTINE_SCHEMA, ROUTINE_NAME, ROUTINE_TYPE, ROUTINE_DEFINITION " + f"FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA NOT IN ({excluded}) " + "ORDER BY 1, 2" + ), + s.rows( + "SELECT EVENT_SCHEMA, EVENT_NAME, STATUS, EVENT_DEFINITION " + "FROM information_schema.EVENTS ORDER BY 1, 2" + ), + ] + for d in out.values(): + if len(d.primary_key) != 1 or d.generated_key != d.primary_key[0]: + d.generated_key = None + return out, _digest(code) + master = s.rows("SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY 1, 2") + for kind, name, _tbl, sql in master: + if kind != "table" or str(name).startswith("sqlite_"): + continue + info = s.rows(f"PRAGMA table_info({_quote('sqlite', str(name))})") + cols = [str(r[1]) for r in info] + pk = [str(r[1]) for r in sorted((r for r in info if r[5]), key=lambda r: r[5])] + generated = None + if len(pk) == 1: + ctype = next(str(r[2]) for r in info if str(r[1]) == pk[0]) + if ctype.strip().upper() == "INTEGER": + generated = pk[0] # a rowid alias: the database assigns it + out[str(name)] = _Def( + cols, + pk, + generated, + {str(r[1]): _default_text(r[4], "sqlite") for r in info}, + set(), + _digest([sql]), + auto_column=generated, + ) + code = [r for r in master if r[0] in ("trigger", "view")] + return out, _digest(code) + + +def _cell(dialect: Dialect, column: str, binary: bool) -> str: + q = _quote(dialect, column) + if dialect == "mysql": + return f"HEX({q})" if binary else q + return f"CASE WHEN typeof({q}) = 'blob' THEN hex({q}) ELSE {q} END" + + +def _row_json(dialect: Dialect, d: _Def) -> list[str]: + obj = "JSON_OBJECT" if dialect == "mysql" else "json_object" + return [ + obj + + "(" + + ", ".join( + f"{_literal(c)}, {_cell(dialect, c, c in d.binary)}" + for c in d.columns[i : i + _CHUNK] + ) + + ")" + for i in range(0, len(d.columns), _CHUNK) + ] + + +def _read_rows( + s: _Session, name: str, d: _Def, where: str = "" +) -> list[dict[str, Any]]: + parts = _row_json(s.dialect, d) + cells = ", ".join(f"{c} AS r{i}" for i, c in enumerate(parts)) + out = [] + for chunks in s.rows(f"SELECT {cells} FROM {_quote(s.dialect, name)} {where}"): + row: dict[str, Any] = {} + for chunk in chunks: + if chunk is not None: + row.update(json.loads(chunk) if isinstance(chunk, str) else chunk) + out.append(row) + return out + + +def _row_key( + d: _Def, row: Mapping[str, Any], seen: dict[str, int], skip: Sequence[str] = () +) -> str: + if d.primary_key: + return "|".join(canonical(row.get(c)) for c in d.primary_key) + # No key: the row's contents, without the columns a rule lets move (a + # counter row stays the same row when its counter moves). + ignored = set(skip) + body = json.dumps( + {c: v for c, v in row.items() if c not in ignored}, sort_keys=True, default=str + ) + digest = hashlib.sha256(body.encode()).hexdigest()[:16] + seen[digest] = seen.get(digest, 0) + 1 + return f"{digest}#{seen[digest]}" + + +def _mysql_counters(s: _Session) -> dict[str, int]: + default = s.rows("SELECT DATABASE()")[0][0] + out: dict[str, int] = {} + for schema, table, value in s.rows( + "SELECT TABLE_SCHEMA, TABLE_NAME, AUTO_INCREMENT FROM information_schema.TABLES " + "WHERE AUTO_INCREMENT IS NOT NULL" + ): + name = str(table) if schema == default else f"{schema}.{table}" + out[name] = int(value) + return out + + +def _sees_triggers(s: _Session) -> bool: + """Can this MySQL account see triggers (it needs TRIGGER)?""" + + grants = " ".join(str(r[0]) for r in s.rows("SHOW GRANTS FOR CURRENT_USER()")) + upper = grants.upper() + return "ALL PRIVILEGES" in upper or "TRIGGER" in upper + + +def take_snapshot( + connect: Callable[[], Any], + dialect: Dialect, + *, + key: bytes, + modes: Mapping[str, TableMode] = {}, + max_rows: int = 2000, + before: Optional[Snapshot] = None, + extra_queries: Mapping[str, str] = {}, + ignore: Mapping[str, Sequence[str]] = {}, + ref_columns: Sequence[str] = (), + schemas: Sequence[str] = (), + optional_modes: Mapping[str, TableMode] = {}, +) -> Snapshot: + """Read every table in one read-only transaction. + + Args: + connect: Opens one connection (DB-API 2.0: ``cursor()``, ``close()``). + key: The MAC key (:func:`mac_key`). + modes: Table -> ``append_only`` or ``skip`` (allow-listed tables); + every other table is compared. + max_rows: Over this many rows, a table keeps cell MACs only for + ``ref_columns`` (and a contents MAC for every row). + before: The baseline, when this is a read after the save: plain + values are kept for the rows that differ from it. + extra_queries: Name -> a SELECT to run in the same transaction (the + audit log); rows land in :attr:`Snapshot.extra`, in memory only. + ignore: Table -> columns left out of each row's contents MAC. + ref_columns: Columns attribution and side records read (a record + key, a link field): their cell MACs are kept in every table. + schemas: Schemas the read must see (MySQL); refused if one isn't. + optional_modes: Like ``modes``, for tables an app profile names that + this database may not have (a profile covers several versions). + ``ignore`` may name ``*`` for every table. + + Raises: + SnapshotError: the database can't be read as the frame needs it. + """ + + try: + s = _Session(connect, dialect) + except Exception as exc: # noqa: BLE001 + raise SnapshotError(f"could not open a connection: {exc}") from exc + try: + return _snapshot( + s, + key, + modes, + max_rows, + before, + extra_queries, + ignore, + ref_columns, + schemas, + optional_modes, + ) + except SnapshotError: + raise + except Exception as exc: # noqa: BLE001 + raise SnapshotError(f"the database could not be read: {exc}") from exc + finally: + s.close() + + +def _snapshot( + s: _Session, + key: bytes, + modes: Mapping[str, TableMode], + max_rows: int, + before: Optional[Snapshot], + extra_queries: Mapping[str, str], + ignore: Mapping[str, Sequence[str]], + ref_columns: Sequence[str], + schemas: Sequence[str], + optional_modes: Mapping[str, TableMode] = {}, +) -> Snapshot: + dialect = s.dialect + counters: dict[str, int] = {} + visible = True + if dialect == "mysql": + # Before the snapshot starts: a row committed after this read is in + # the snapshot, so the counter never runs ahead of what it shows. + counters = _mysql_counters(s) + visible = _sees_triggers(s) + s.begin() + snap = Snapshot( + dialect=dialect, + clock=_clock_text(s.rows("SELECT CURRENT_TIMESTAMP")[0][0]), + code_visible=visible, + ) + defs, snap.objects = _definitions(s) + if dialect == "mysql": + default = str(s.rows("SELECT DATABASE()")[0][0]) + seen = {default} | {t.split(".", 1)[0] for t in defs if "." in t} + missing = sorted(set(schemas) - seen) + if missing: + raise SnapshotError( + "the read account can't see schema(s): " + ", ".join(missing) + ) + snap.schemas = sorted(seen) + else: + if any( + n == "sqlite_sequence" + for (n,) in s.rows("SELECT name FROM sqlite_master WHERE type = 'table'") + ): + counters = { + str(n): int(v) + for n, v in s.rows("SELECT name, seq FROM sqlite_sequence") + } + for name, sql in extra_queries.items(): + snap.extra[name] = s.dicts(sql) + unknown = sorted(set(modes) - set(defs)) + modes = {**{t: m for t, m in optional_modes.items() if t in defs}, **modes} + if unknown: + raise SnapshotError( + "allow-listed tables not in the database: " + ", ".join(unknown) + ) + names = sorted(t for t in defs if modes.get(t) != "skip") + for t in sorted(t for t in defs if modes.get(t) == "skip"): + d = defs[t] + snap.tables[t] = Table("skip", d.schema, d.columns, d.primary_key, None, {}) + if not names: + return snap + counts = { + str(t): int(n) + for t, n in s.rows( + " UNION ALL ".join( + f"SELECT {_literal(t)}, COUNT(*) FROM {_quote(dialect, t)}" + for t in names + ) + ) + } + numbered = [t for t in names if defs[t].auto_column] + maxes = ( + { + str(t): (None if m is None else int(m)) + for t, m in s.rows( + " UNION ALL ".join( + f"SELECT {_literal(t)}, MAX({_quote(dialect, defs[t].auto_column or '')}) " + f"FROM {_quote(dialect, t)}" + for t in numbered + ) + ) + } + if numbered + else {} + ) + refs = set(ref_columns) + for t in names: + d = defs[t] + mode: TableMode = "append_only" if modes.get(t) == "append_only" else "compare" + table = Table( + mode, + d.schema, + d.columns, + d.primary_key, + d.generated_key, + dict(d.defaults), + counts.get(t, 0), + counters.get(t), + d.auto_column, + maxes.get(t), + non_transactional=d.non_transactional, + ) + if d.generated_key: + # Only the table's own generated key may hold any value. + table.defaults[d.generated_key] = ANY_VALUE + old = before.tables.get(t) if before is not None else None + if mode == "append_only": + table.cell_columns = [] + elif old is not None and old.mode == mode and old.schema == d.schema: + table.cell_columns = list(old.cell_columns) # read like the baseline + elif table.count > max_rows: + table.cell_columns = [c for c in d.columns if c in refs] + else: + table.cell_columns = list(d.columns) + skip = [*ignore.get(t, ()), *ignore.get("*", ())] + seen_keys: dict[str, int] = {} + for row in _read_rows(s, t, d) if table.count else []: + k = row_key_hash(key, _row_key(d, row, seen_keys, skip), t) + mac = row_hash(key, t, row, skip) + table.rows[k] = [ + mac, + "".join(cell_hash(key, row.get(c), t, c) for c in table.cell_columns), + ] + if old is not None and (k not in old.rows or old.rows[k][0] != mac): + table.values[k] = dict(row) + snap.tables[t] = table + return snap + + +# -- comparing ---------------------------------------------------------------- + + +@dataclass +class TableDiff: + """How one table moved between two snapshots. Counts and column names + only, never values.""" + + added: int = 0 + removed: int = 0 + changed: int = 0 + changed_columns: list[str] = field(default_factory=list) + #: Moved in a way no row comparison explains: why (a definition change, + #: an append-only table whose existing rows moved, a row written and + #: removed). + unread: list[str] = field(default_factory=list) + + +def unread_reasons( + old: Optional[Table], new: Optional[Table], dialect: str = "sqlite" +) -> list[str]: + """Why a table moved in a way no row comparison explains; [] if none.""" + + if old is None or new is None: + return ["created or dropped"] + if old.mode != new.mode: + return ["compared differently"] + if old.mode == "skip": + return [] + if old.schema != new.schema or old.columns != new.columns: + return ["definition changed"] + reasons = [] + if old.mode == "append_only" and any( + k not in new.rows or new.rows[k][0] != v[0] for k, v in old.rows.items() + ): + reasons.append("existing rows changed or removed") + if new.counter is not None: + # sqlite_sequence holds the last number used; AUTO_INCREMENT the next. + step = 1 if dialect == "mysql" else 0 + expected = max( + old.counter or 0, 0 if new.max_auto is None else new.max_auto + step + ) + if new.counter > expected: + reasons.append("a row was written and removed (counter moved)") + return reasons + + +def diff( + before: Snapshot, + after: Snapshot, + *, + key: bytes, + allow_tables: Sequence[str] = (), +) -> dict[str, TableDiff]: + """Every table that moved between ``before`` and ``after``, except + ``allow_tables`` and skipped ones. Empty when the database is back where + it was. Database code (triggers, views, routines, events) that changed + shows under ``"(database code)"``.""" + + out: dict[str, TableDiff] = {} + if before.objects != after.objects: + out["(database code)"] = TableDiff(unread=["changed"]) + skip = set(allow_tables) + for name in sorted(set(before.tables) | set(after.tables)): + if name in skip: + continue + a, b = before.tables.get(name), after.tables.get(name) + moved = TableDiff(unread=unread_reasons(a, b, after.dialect)) + if a is not None and b is not None and a.mode == "compare" == b.mode: + if a.columns == b.columns and a.cell_columns == b.cell_columns: + moved.added = len(b.rows.keys() - a.rows.keys()) + moved.removed = len(a.rows.keys() - b.rows.keys()) + columns: set[str] = set() + for k in a.rows.keys() & b.rows.keys(): + if a.rows[k][0] == b.rows[k][0]: + continue + moved.changed += 1 + old = unpack(a.cell_columns, a.rows[k][1]) + new = unpack(b.cell_columns, b.rows[k][1]) + cols = [c for c in a.cell_columns if old[c] != new[c]] + columns.update(cols or ["(columns not compared)"]) + moved.changed_columns = sorted(columns) + if moved.added or moved.removed or moved.changed or moved.unread: + out[name] = moved + return out + + +def same_state(a: Snapshot, b: Snapshot) -> bool: + """Do two reads taken after the save show the same database?""" + + if (a.objects, a.code_visible) != (b.objects, b.code_visible): + return False + if set(a.tables) != set(b.tables): + return False + return all( + (x.mode, x.schema, x.count, x.counter, x.max_auto, x.rows) + == (y.mode, y.schema, y.count, y.counter, y.max_auto, y.rows) + for x, y in ((a.tables[n], b.tables[n]) for n in a.tables) + ) diff --git a/packages/flow/openadapt_flow/runtime/effects/effect.py b/packages/flow/openadapt_flow/runtime/effects/effect.py index 00929dca1..7128ef4c2 100644 --- a/packages/flow/openadapt_flow/runtime/effects/effect.py +++ b/packages/flow/openadapt_flow/runtime/effects/effect.py @@ -56,6 +56,35 @@ ) from pydantic_core import to_jsonable_python +ValueKind = Literal["phone", "date", "email"] + + +def kind_form( + kind: Optional[str], value: Any, date_order: Optional[str] = None +) -> Optional[str]: + """``value`` in the canonical form of a declared value kind, or None when + it can't be read as one (which then matches nothing). The one normalizer + is :mod:`openadapt_flow.runtime.effects.readback_values`, shared with the + on-screen read-back: + + - ``phone``: the digits, with an extension kept apart; a value with + anything but digits, ``( ) + - . /``, spaces and one trailing extension + isn't a phone number; + - ``date``: ISO ``YYYY-MM-DD`` (a stored date-time keeps its date; nothing + else may follow); a numeric date needs ``date_order``, or it is + ambiguous; + - ``email``: one address, case folded. + """ + + from openadapt_flow.runtime.effects.readback_values import normalize_value + + if value is None: + return None + text = str(value).strip() + if kind is None: + return text + return normalize_value(text, kind, date_order=date_order) # type: ignore[arg-type] + class ValueExpr(BaseModel): """A single effect-contract value: either a static ``literal`` or a @@ -91,6 +120,19 @@ class ValueExpr(BaseModel): #: reads the entered value back as its code. A value the map doesn't name #: resolves to nothing, so the check matches nothing and fails closed. map: Optional[dict[str, str]] = None + #: For a ``param`` the system of record stores transformed: how it stores + #: what was entered (OpenEMR's New Patient form saves street and city in + #: capitals). Applied after :attr:`map`, with full Unicode case mapping, + #: as the browser's ``toUpperCase()`` does for OpenEMR's "U" fields + #: ("Peña" is stored "PEÑA", "Straße" "STRASSE"). The check stays exact: + #: it expects the stored form, not any casing. + stored: Optional[Literal["upper", "lower"]] = None + #: A declared value kind (:func:`kind_form`): the expected value and the + #: stored one are compared in canonical form, so formatting alone never + #: refutes, and a different value always does. Never inferred. + kind: Optional[ValueKind] = None + #: ``kind: date`` only: how a numeric date entered for this value reads. + date_order: Optional[Literal["mdy", "dmy"]] = None @model_validator(mode="after") def _exactly_one_source(self) -> "ValueExpr": @@ -102,6 +144,10 @@ def _exactly_one_source(self) -> "ValueExpr": raise ValueError("effect parameter name must not be empty") if self.map is not None and (self.param is None or not self.map): raise ValueError("a value map needs a parameter and at least one entry") + if self.stored is not None and self.param is None: + raise ValueError("a stored form applies to a parameter") + if self.date_order is not None and self.kind != "date": + raise ValueError("date_order applies to kind: date") return self @model_serializer(mode="wrap") @@ -109,8 +155,10 @@ def _serialize_compatible(self, handler: Any) -> Any: # A value without a map serializes exactly as before, so existing # effect contract hashes and bundle digests are unchanged. data = handler(self) - if isinstance(data, dict) and data.get("map") is None: - data.pop("map", None) + if isinstance(data, dict): + for key in ("map", "stored", "kind", "date_order"): + if data.get(key) is None: + data.pop(key, None) return data def resolve(self, params: Mapping[str, object]) -> Optional[str]: @@ -129,14 +177,36 @@ def resolve(self, params: Mapping[str, object]) -> Optional[str]: from openadapt_flow.runtime.authorization import runtime_param_text entered = runtime_param_text(params[self.param]) # type: ignore[arg-type] - if self.map is not None: - return self.map.get(entered) - return entered + return self.stored_form(entered) return self.literal + def stored_form(self, entered: str) -> Optional[str]: + """What the system of record stores for ``entered``: its code under + :attr:`map` (None when the map doesn't name it), then :attr:`stored`.""" + + value: Optional[str] = entered + if self.map is not None: + value = self.map.get(entered) + if value is not None and self.stored == "upper": + value = value.upper() + elif value is not None and self.stored == "lower": + value = value.lower() + return value + def resolved(self, params: Mapping[str, object]) -> "ValueExpr": """Return a pure-literal copy of this expression bound to ``params``.""" - return ValueExpr(literal=self.resolve(params)) + return ValueExpr( + literal=self.resolve(params), kind=self.kind, date_order=self.date_order + ) + + def matches(self, stored: Any) -> bool: + """Does a stored value equal this (resolved) value? In canonical form + when a kind is declared, as text otherwise.""" + + if self.kind is None: + return str(stored) == str(self) + want = kind_form(self.kind, self.literal, self.date_order) + return want is not None and kind_form(self.kind, stored) == want # -- transparent str-compatibility (back-compat with the v1 plain-string # form): a literal ValueExpr compares, hashes, stringifies, and reprs @@ -1210,7 +1280,10 @@ def record_matches(record: dict[str, Any], selector: dict[str, str]) -> bool: An empty selector matches every record (used by absence checks). """ for key, want in selector.items(): - if str(record.get(key, None)) != str(want): + if isinstance(want, ValueExpr) and want.kind is not None: + if not want.matches(record.get(key, None)): + return False + elif str(record.get(key, None)) != str(want): return False return True diff --git a/packages/flow/openadapt_flow/runtime/effects/frame.py b/packages/flow/openadapt_flow/runtime/effects/frame.py index 15f3fd3db..bfb09248b 100644 --- a/packages/flow/openadapt_flow/runtime/effects/frame.py +++ b/packages/flow/openadapt_flow/runtime/effects/frame.py @@ -37,6 +37,16 @@ the saved value may no longer stand, so it blocks like collateral. Only changes outside the declared delta can be concurrent edits. +With ``frame.database``, another actor's entry explains a row only when it +is a write of the matching kind (insert for an added row, delete for a +removed one, update for a changed one) on the same table, naming the record +as it was before the change; each entry explains at most one row, and a row +whose record key changed is never explained. What the log can't show stays a +limit: an entry the app files under the wrong record (a write logged against +the chart the bot has open), or one colleague insert whose own row never +landed, still reads as that colleague's. List every app account the bot +holds in ``self``. + The judge reads nothing itself; callers pass the snapshots. Fail-safe: a missing baseline, an unreadable read, a record without its identity field or an unresolved self id makes the frame ``not_checked``, never ``held``. @@ -46,7 +56,7 @@ import hashlib import json -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any, Literal, Optional @@ -97,6 +107,33 @@ class FrameAttribution(BaseModel): #: ``audit_read`` only: the read-only read of the log, as an ``effects:`` #: verifier section. Built by the caller (``verify-run``). read: Optional[dict[str, Any]] = None + #: ``audit_read`` only: the record field the audit entry's + #: ``record_field`` refers to, when it isn't the frame's identity field. + #: OpenEMR's log names a patient (``pid``) while ``lists`` or ``pnotes`` + #: rows are told apart by their own ``id``. + record_key: Optional[str] = None + #: ``audit_read`` only: per surface or table, the field that names the + #: record when it isn't ``record_key`` (OpenEMR's ``contact`` row points + #: at its chart through ``foreign_id``). + record_keys: dict[str, str] = Field(default_factory=dict) + #: ``audit_read`` only: the entry field naming what the entry did, and + #: the values that are writes. Set, only write entries attribute a change: + #: an entry for viewing a record never excuses a write to it. Required + #: with ``frame.database``. + event_field: Optional[str] = None + write_events: list[str] = Field(default_factory=list) + #: The write events that create a record (a subset of ``write_events``). + #: Another actor's later write to a record the bot created can be told + #: apart from the bot's own insert only through these. + insert_events: list[str] = Field(default_factory=list) + #: The write events that remove a record (a subset of ``write_events``). + #: An added row is explained only by an insert, a removed one only by a + #: delete, a changed one only by the other write events. + delete_events: list[str] = Field(default_factory=list) + #: ``audit_read`` only: the entry field naming the table written. Set, + #: an entry attributes a change only to a row of that table. Required + #: with ``frame.database``. + table_field: Optional[str] = None #: ``record_field`` only: a version, revision or ETag on each record. version_field: Optional[str] = None #: ``record_field`` only: the record fields this app sets on the server @@ -127,6 +164,16 @@ def _shape(self) -> "FrameAttribution": raise ValueError("version_field applies to record_field attribution") if self.server_set: raise ValueError("server_set applies to record_field attribution") + if bool(self.event_field) != bool(self.write_events): + raise ValueError("event_field and write_events go together") + if not (set(self.insert_events) | set(self.delete_events)) <= set( + self.write_events + ): + raise ValueError("insert_events and delete_events must be write_events") + elif self.record_keys or self.write_events or self.table_field: + raise ValueError( + "record_keys, write_events and table_field apply to audit_read" + ) else: if self.read is not None or self.record_field is not None: raise ValueError( @@ -145,6 +192,166 @@ def _shape(self) -> "FrameAttribution": return self +class FrameSideRecord(BaseModel): + """Rows the save adds beside each new declared record (a side record). + + OpenEMR's New Patient path writes one ``history_data`` row for the new + chart, linked by ``pid``. Declared here, such rows are part of the save, + not collateral, and a missing or extra one refutes the frame. + """ + + model_config = ConfigDict(extra="forbid") + + #: The surface the side rows are read from. + surface: str + #: The side row's field that names its record (``pid``). + link_field: str + #: The surface of the declared record, and the field the link names. + of_surface: str + of_field: str + #: How many side rows each new declared record gets. + count: int = Field(default=1, ge=1) + + +class AllowedTable(BaseModel): + """One allow-listed table: the app writes it on its own.""" + + model_config = ConfigDict(extra="forbid") + + table: str + #: ``append_only`` (the default): rows may be added, existing rows must + #: not change, and the table needs a generated key. ``skip``: not read at + #: all, which needs a ``reason``. + mode: Literal["append_only", "skip"] = "append_only" + reason: Optional[str] = None + + @model_validator(mode="after") + def _skip_needs_reason(self) -> "AllowedTable": + if self.mode == "skip" and not (self.reason or "").strip(): + raise ValueError(f"skipping table {self.table!r} needs a reason") + return self + + +class FrameDatabase(BaseModel): + """``frame.database``: diff the whole database, not only what is read. + + Every table and column is snapshotted before and after the save + (``db_snapshot.py``), and each table is judged as one more slice: a row + added, removed or changed is collateral unless a declared effect, a + declared side record or an attributed concurrent writer explains it. A + new declared or side row that sets a column the effects don't name, and + that ``new_row_fields`` doesn't list, is collateral too. + + ``allow_tables`` replaces a read-list with a reviewed allow-list: the + tables the app writes on its own for any signed-in user (logs, sessions, + id registries). They are append-only unless an entry says ``skip`` with a + reason. Everything else is compared. The receipt counts tables compared, + append-only and skipped, and binds the list's digest. + """ + + model_config = ConfigDict(extra="forbid") + + dialect: Literal["sqlite", "mysql"] + #: The read-only connection: ``sqlite_database``, or ``sql_driver`` with + #: ``sql_connect_args`` and ``sql_password_env``, as in a ``kind: sql`` + #: verifier (no query: every table is read). + read: dict[str, Any] + #: The app's own bookkeeping, reviewed per app: a table name + #: (append-only) or ``{table, mode, reason}``. + allow_tables: list[AllowedTable] = Field(default_factory=list) + #: Table -> the surface whose records it holds (``patient_data`` -> + #: ``patient``): an effect routed to the surface declares rows here too. + views: dict[str, str] = Field(default_factory=dict) + #: Table -> columns a new declared or side row may set beyond the fields + #: the effects name (ids, creation dates, columns the app's form fills + #: on its own). Any other column must hold its default. + new_row_fields: dict[str, list[str]] = Field(default_factory=dict) + #: Table -> columns that change on their own on every row (a last-seen + #: time). ``frame.ignore_fields`` applies to surfaces only. + ignore_columns: dict[str, list[str]] = Field(default_factory=dict) + #: Over this many rows a table keeps, per row, its key and contents MACs + #: and cell MACs only for the columns attribution and side records read. + max_rows: int = Field(default=2000, ge=1) + #: Schemas the read account must see (MySQL); the snapshot refuses when + #: one isn't visible, so a schema the app writes can't be left out. + schemas: list[str] = Field(default_factory=list) + #: An app profile (``openadapt_flow.app_profiles``): the app's own + #: bookkeeping, versioned and shared with rollback and the RL grader. Its + #: append-only and session tables and its column rules apply on top of + #: ``allow_tables`` and ``ignore_columns``. + profile: Optional[str] = None + #: The profile's digest, pinned: a package upgrade that changes what is + #: excused can't change it under a contract (``frame_sha256`` binds it). + profile_sha256: Optional[str] = None + #: Seconds between the two reads after the save. They must match: a + #: write the app makes a moment later (a background job) shows as a + #: difference, and the frame is not checked until the database settles. + settle_s: float = Field(default=1.0, ge=0) + + @model_validator(mode="after") + def _pinned_profile(self) -> "FrameDatabase": + if self.profile is None: + return self + profile = self.loaded_profile() + if self.profile_sha256 != profile.sha256(): + raise ValueError( + f"frame.database.profile_sha256 must pin {self.profile!r} " + f"({profile.version}, {profile.sha256()})" + ) + return self + + @model_validator(mode="before") + @classmethod + def _names(cls, data: Any) -> Any: + if isinstance(data, dict) and isinstance(data.get("allow_tables"), list): + data = dict(data) + data["allow_tables"] = [ + {"table": t} if isinstance(t, str) else t for t in data["allow_tables"] + ] + return data + + def modes(self) -> dict[str, str]: + return {t.table: t.mode for t in self.allow_tables} + + def loaded_profile(self) -> Any: + from openadapt_flow.app_profiles import load_profile + + return None if self.profile is None else load_profile(self.profile) + + def profile_modes(self) -> dict[str, str]: + """The profile's tables: append-only, or skipped (session state).""" + + profile = self.loaded_profile() + if profile is None: + return {} + return {t["table"]: t["mode"] for t in profile.allow_tables()} + + def all_ignore_columns(self) -> dict[str, list[str]]: + out = {t: list(c) for t, c in self.ignore_columns.items()} + profile = self.loaded_profile() + if profile is not None: + for table, cols in profile.ignore_columns().items(): + out.setdefault(table, []).extend(cols) + return out + + def ignore_columns_for(self, table: str) -> list[str]: + both = self.all_ignore_columns() + return [*both.get(table, []), *both.get("*", [])] + + def allow_list_sha256(self) -> str: + payload = [t.model_dump(mode="json") for t in self.allow_tables] + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + @model_validator(mode="after") + def _connection(self) -> "FrameDatabase": + if not (self.read.get("sqlite_database") or self.read.get("sql_driver")): + raise ValueError("frame.database.read needs sqlite_database or sql_driver") + if self.read.get("sql_query"): + raise ValueError("frame.database.read reads every table; drop sql_query") + return self + + class FrameSpec(BaseModel): """``frame:`` in ``verify.yaml``: the slice that must otherwise not move.""" @@ -159,6 +366,26 @@ class FrameSpec(BaseModel): #: the fields the effects name. target_fields: Optional[list[str]] = None attribution: Optional[FrameAttribution] = None + #: Rows the save adds beside each new declared record (``FrameSideRecord``). + side_records: list[FrameSideRecord] = Field(default_factory=list) + #: Diff the whole database (``FrameDatabase``). + database: Optional[FrameDatabase] = None + + @model_validator(mode="after") + def _database_attribution(self) -> "FrameSpec": + a = self.attribution + if ( + self.database is not None + and a is not None + and a.kind == "audit_read" + and not (a.write_events and a.table_field) + ): + raise ValueError( + "with frame.database, audit attribution needs event_field, " + "write_events and table_field: only a write to the same table " + "attributes a change" + ) + return self @model_validator(mode="after") def _identity_for_audit(self) -> "FrameSpec": @@ -209,6 +436,14 @@ class FrameVerdict: records_compared: int = 0 collateral: list[FrameChange] = field(default_factory=list) concurrent: list[FrameChange] = field(default_factory=list) + #: ``frame.database`` only: tables compared, append-only and skipped, + #: and the allow-list's digest. + database: Optional[dict[str, Any]] = None + #: In memory only: (surface, identity) of records another actor added, + #: as their surface names them, when a table is judged in its place. + added_by_others: set[tuple[str, str]] = field(default_factory=set) + #: In memory only: app profile rules that excused a row change. + excused_by: set[str] = field(default_factory=set) @property def held(self) -> bool: @@ -232,6 +467,7 @@ def evidence(self, spec: FrameSpec) -> dict[str, Any]: "target_changed_by_other": sum( 1 for c in self.collateral if c.other_on_target ), + **({"database": self.database} if self.database is not None else {}), } @@ -260,9 +496,15 @@ def declared_selectors(effect: Effect) -> list[dict[str, str]]: """ def plain(selector: Mapping[str, Any]) -> dict[str, str]: - out = {k: str(v) for k, v in selector.items()} + # A value with a declared kind stays an expression, so the record + # is matched in canonical form (``record_matches``). + out = { + k: (v if getattr(v, "kind", None) else str(v)) # type: ignore[misc] + for k, v in selector.items() + } if effect.idempotency_key is not None: - out[effect.key_field] = str(effect.idempotency_key) + key = effect.idempotency_key + out[effect.key_field] = key if getattr(key, "kind", None) else str(key) # type: ignore[assignment] return out if effect.kind is EffectKind.EXACT_NEW_SET: @@ -324,6 +566,18 @@ def judge_frame( audit_before: Optional[list[dict[str, Any]]] = None, audit_after: Optional[list[dict[str, Any]]] = None, run_actors: Sequence[str] = (), + identities: Optional[Mapping[str, str]] = None, + new_row_defaults: Optional[Mapping[str, Mapping[str, Optional[str]]]] = None, + unread_changes: Sequence[tuple[str, str]] = (), + hashers: Optional[Mapping[str, Callable[[str, Any], str]]] = None, + clock: tuple[str, str] = ("", ""), + database_evidence: Optional[dict[str, Any]] = None, + excuse: Optional[ + Callable[ + [str, Optional[Mapping[str, Any]], Optional[Mapping[str, Any]], list[str]], + Optional[list[str]], + ] + ] = None, ) -> FrameVerdict: """Judge the frame of ``effects`` over ``slices``. @@ -340,8 +594,26 @@ def judge_frame( audit_after: ``audit_read`` entries read after the save. run_actors: More ids of this run (the executor's), never "another actor". + identities: Slice -> its identity field, where it isn't + ``spec.identity_field`` (a database table's row key). + new_row_defaults: Slice -> column defaults, for slices whose new + declared and side rows must set nothing undeclared (database + tables). ``db_snapshot.ANY_VALUE`` marks the generated key and + ``CLOCK_DEFAULT`` a column the database fills with the time. + unread_changes: (table, why) for tables that moved in a way no row + comparison explains. Each is collateral. + hashers: Slice -> (column, value) -> the keyed MAC its baseline records hold instead of + values (database tables). A post-save record is plain; it is + hashed to compare with the baseline. + clock: The database clock at the baseline and after the save. + database_evidence: Counts for the receipt (tables compared, ...). + excuse: (table, before, after, changed fields) -> the app profile + rules that make this row change the app's own bookkeeping, or + None. Applied to rows no effect declares. """ + from openadapt_flow.runtime.effects.db_snapshot import KEY, ROW, is_default + attribution = spec.attribution mode: str = attribution.kind if attribution is not None else "none" self_ids: set[str] = set() @@ -355,10 +627,12 @@ def judge_frame( ) self_ids = resolved | {str(a) for a in run_actors if str(a)} - ignore = set(spec.ignore_fields) + hashers = hashers or {} + database = spec.database + surface_ignore = set(spec.ignore_fields) | {KEY} if attribution is not None and attribution.kind == "record_field": # Server-set bookkeeping moves on every write; it says who, not what. - ignore |= set(attribution.server_set) + surface_ignore |= set(attribution.server_set) written = declared_fields(effects) may_change = set(spec.target_fields) if spec.target_fields is not None else written if attribution is not None and attribution.kind == "record_field": @@ -371,19 +645,185 @@ def judge_frame( attribution = None mode = "attribution_refused" - def key_of(record: Mapping[str, Any]) -> Any: - if spec.identity_field: - value = record.get(spec.identity_field) + def ignore_for(name: str) -> set[str]: + if name in hashers: + # Database tables: only the columns declared for that table. + cols = database.ignore_columns_for(name) if database else [] + return set(cols) | {KEY} + return surface_ignore + + def identity_of(name: str) -> Optional[str]: + if identities is not None and name in identities: + return identities[name] + return spec.identity_field + + def key_of(record: Mapping[str, Any], name: str = "") -> Any: + identity = identity_of(name) + if identity: + value = record.get(identity) return None if value is None else str(value) - return stable_id(_without(record, ignore)) + return stable_id(_without(record, ignore_for(name))) + + def changed( + name: str, before: Mapping[str, Any], after: Mapping[str, Any] + ) -> list[str]: + """Fields that differ. A database baseline row holds cell MACs (for + all columns, or only the reference columns of a large table) and its + contents MAC: the post-save value is MACed to compare, and a row + whose contents moved in no compared column says so.""" + + ignore = ignore_for(name) + hasher = hashers.get(name) + if hasher is None: + return _changed_fields(before, after, ignore) + fields = sorted( + k + for k in before + if k not in ignore and k != ROW and before.get(k) != hasher(k, after.get(k)) + ) + if not fields and before.get(ROW) != after.get(ROW): + fields = ["(columns not compared)"] + return fields + + row_fields = database.new_row_fields if database is not None else {} + + def undeclared_columns( + name: str, record: Mapping[str, Any], also: Sequence[str] = () + ) -> list[str]: + """Columns a new row sets that nothing declares (database slices).""" + + defaults = (new_row_defaults or {}).get(name) + if defaults is None: + return [] + allowed = ( + written + | set(row_fields.get(name, ())) + | ignore_for(name) + | {ROW} + | set(also) + ) + return sorted( + c + for c, v in record.items() + if c not in allowed and not is_default(v, defaults.get(c), clock) + ) + # The audit window, kept to write entries when the frame says which they + # are: an entry for viewing a record never excuses a write to it. window_entries: Optional[list[dict[str, Any]]] = None if mode == "audit_read": if audit_before is not None and audit_after is not None: seen = {stable_id(e) for e in audit_before} window_entries = [e for e in audit_after if stable_id(e) not in seen] + if attribution is not None and attribution.write_events: + writes = set(attribution.write_events) + window_entries = [ + e + for e in window_entries + if str(get_path(e, attribution.event_field or "")) in writes + ] + + surface_tables = {v: k for k, v in (database.views if database else {}).items()} + + def table_of(name: str) -> Optional[str]: + return name if name in hashers else surface_tables.get(name) + + def event_kinds(change: Optional[str]) -> Optional[set[str]]: + """The write events that can explain this kind of change: an insert + for an added row, a delete for a removed one, an update otherwise. + None: any write (the frame names no event kinds).""" + + if attribution is None or not attribution.write_events or change is None: + return None + inserts, deletes = ( + set(attribution.insert_events), + set(attribution.delete_events), + ) + if change == "added": + return inserts + if change == "removed": + return deletes + return set(attribution.write_events) - inserts - deletes + + def entries_for( + name: str, ref: Any, field: Optional[str], kinds: Optional[set[str]] = None + ) -> list[dict[str, Any]]: + """Window entries that name this record (and its table, when the + attribution says which table an entry wrote). ``field`` is set when + ``ref`` is a baseline MAC of that column.""" + + if window_entries is None or ref is None or attribution is None: + return [] + hasher = hashers.get(name) if field else None + table = table_of(name) + out = [] + for e in window_entries: + entry_ref = get_path(e, attribution.record_field or "") + if entry_ref in (None, ""): + continue + if (hasher(field, entry_ref) if hasher else str(entry_ref)) != str(ref): + continue + if attribution.table_field: + if table is None or str(get_path(e, attribution.table_field)) != table: + continue + if get_path(e, attribution.actor_field) in (None, ""): + continue + if kinds is not None and ( + str(get_path(e, attribution.event_field or "")) not in kinds + ): + continue + out.append(e) + return out + + def ref_of( + name: str, + key: Any, + before: Optional[Mapping[str, Any]], + after: Optional[Mapping[str, Any]], + ) -> tuple[Any, Optional[str]]: + """What an audit entry names for this record, taken from the record + as it was BEFORE the change when there was one (so a write can't pick + its own excuse by moving a row), and the column when that value is a + baseline MAC.""" + + if attribution is None: + return key, None + field = attribution.record_keys.get(name, attribution.record_key) + if not field: + return key, None + record = before if before is not None else after + value = None if record is None else record.get(field) + hashed = name in hashers and before is not None + return (None if value in (None, "") else str(value)), ( + field if hashed else None + ) + + def key_moved( + name: str, before: Mapping[str, Any], after: Mapping[str, Any] + ) -> bool: + """Did the column an audit entry names change? Then no entry about + either chart can explain the row.""" + + if attribution is None or attribution.kind != "audit_read": + return False + field = attribution.record_keys.get(name, attribution.record_key) + if not field or field not in before: + return False + hasher = hashers.get(name) + if hasher is not None: + return before.get(field) != hasher(field, after.get(field)) + return not _same(before.get(field), after.get(field)) + + def actors( + name: str, ref: Any, field: Optional[str], kinds: Optional[set[str]] = None + ) -> set[str]: + return { + str(get_path(e, attribution.actor_field)) # type: ignore[union-attr] + for e in entries_for(name, ref, field, kinds) + } def other_on_target( + name: str, key: Any, before: Optional[Mapping[str, Any]], after: Optional[Mapping[str, Any]], @@ -393,20 +833,69 @@ def other_on_target( if attribution is None: return False if attribution.kind == "audit_read": - return window_entries is not None and any( - str(get_path(e, attribution.record_field or "")) == str(key) - and get_path(e, attribution.actor_field) not in (None, "") - and str(get_path(e, attribution.actor_field)) not in self_ids - for e in window_entries - ) + ref, field = ref_of(name, key, before, after) + return bool(actors(name, ref, field) - self_ids) if after is None: return False - if before is not None and not _changed_fields(before, after, ignore): + if before is not None and not changed(name, before, after): return False # untouched in the window actor = get_path(after, attribution.actor_field) return actor not in (None, "") and str(actor) not in self_ids + def bot_only_inserted(name: str, ref: Any) -> bool: + """The bot's write entries for this record are inserts only, so a + later write to it by another actor is theirs, not the bot's.""" + + if attribution is None or not attribution.insert_events: + return False + inserts = set(attribution.insert_events) + mine = [ + e + for e in entries_for(name, ref, None) + if str(get_path(e, attribution.actor_field)) in self_ids + ] + return all( + str(get_path(e, attribution.event_field or "")) in inserts for e in mine + ) + + def inserted_by_other(name: str, key: Any, record: Mapping[str, Any]) -> bool: + """Does another actor's insert entry name this new record? (A look + only: it doesn't use the entry up.)""" + + if attribution is None or attribution.kind != "audit_read": + return False + ref, field = ref_of(name, key, None, record) + found = actors(name, ref, field, event_kinds("added")) + return ( + bool(found) + and not (found & self_ids) + and not (actors(name, ref, field) & self_ids) + ) + + excused_by: set[str] = set() + + def bookkeeping( + name: str, + before: Optional[Mapping[str, Any]], + after: Optional[Mapping[str, Any]], + fields: list[str], + ) -> bool: + """Is this row change the app's own bookkeeping (its profile)?""" + + if excuse is None or name not in hashers: + return False + rules = excuse(name, before, after, fields) + if rules: + excused_by.update(rules) + return True + return False + + # Each other actor's entry explains at most one row: a colleague's one + # insert can't cover two stray rows. + used: set[int] = set() + def attribute( + name: str, key: Any, before: Optional[Mapping[str, Any]], after: Optional[Mapping[str, Any]], @@ -414,17 +903,31 @@ def attribute( if attribution is None: return "unattributed" if attribution.kind == "audit_read": - if window_entries is None: + ref, field = ref_of(name, key, before, after) + if window_entries is None or ref is None: return "unattributed" - actors = { - str(get_path(e, attribution.actor_field)) - for e in window_entries - if str(get_path(e, attribution.record_field or "")) == str(key) - and get_path(e, attribution.actor_field) not in (None, "") - } - if actors & self_ids: + if ( + before is not None + and after is not None + and key_moved(name, before, after) + ): + return "unattributed" # the row moved between charts + if actors(name, ref, field) & self_ids: return "self" - return "other" if actors else "unattributed" + change = ( + "added" if before is None else "removed" if after is None else "changed" + ) + kinds = event_kinds(change) + for e in entries_for(name, ref, field, kinds): + if ( + id(e) in used + or str(get_path(e, attribution.actor_field)) in self_ids + ): + continue + if attribution.write_events: + used.add(id(e)) + return "other" + return "unattributed" # record_field: the record says who last changed it. if after is None: return "unattributed" # a removed record can't say who removed it @@ -452,6 +955,8 @@ def attribute( collateral: list[FrameChange] = [] concurrent: list[FrameChange] = [] + added_by_others: set[tuple[str, str]] = set() + views_of = dict(database.views) if database is not None else {} declared_keys: set[str] = set() compared = 0 @@ -461,6 +966,63 @@ def note(change: FrameChange) -> None: else: collateral.append(change) + def selectors_for(name: str) -> list[Any]: + mine = [ + effect + for index, effect in enumerate(effects) + if routes is None or name in routes.get(str(index), (name,)) + ] + return [s for effect in mine for s in declared_selectors(effect)] + + def new_row( + name: str, key: Any, after: Mapping[str, Any], also: Sequence[str] = () + ) -> None: + """A new declared or side row may set only declared columns. Columns + another actor set later, by a write to this table, while the bot only + inserted the row, are that actor's (concurrent).""" + + extra = undeclared_columns(name, after, also) + if not extra: + return + ref, _hashed = ref_of(name, key, None, after) + others = actors(name, ref, None) - self_ids + change = FrameChange(name, "added", "self", extra, on_target=True, key=key) + if others and bot_only_inserted(name, ref): + change.attribution = "other" + concurrent.append(change) + else: + collateral.append(change) + + # Side records: the links each new declared record gives its side rows. + # A new record another actor created gives its side rows the same way: + # those rows are that actor's too (OpenEMR logs a contact row with no + # patient, so only its link to the colleague's chart attributes it). + side_links: list[set[str]] = [] + other_links: list[set[str]] = [] + side_found: list[dict[str, int]] = [] + for side in spec.side_records: + links: set[str] = set() + theirs: set[str] = set() + before_state, after_records = slices.get(side.of_surface, (None, None)) + if before_state is not None and after_records is not None: + sels = selectors_for(side.of_surface) + old = {key_of(r, side.of_surface) for r in before_state.records} + for record in after_records: + if key_of(record, side.of_surface) in old: + continue + value = record.get(side.of_field) + if value in (None, ""): + continue + if any(record_matches(dict(record), s) for s in sels): + links.add(str(value)) + elif inserted_by_other( + side.of_surface, key_of(record, side.of_surface), record + ): + theirs.add(str(value)) + side_links.append(links) + other_links.append(theirs - links) + side_found.append({link: 0 for link in links}) + for name, (before_state, after_records) in slices.items(): if not before_state.reachable: return _not_checked( @@ -468,14 +1030,19 @@ def note(change: FrameChange) -> None: ) if after_records is None: return _not_checked("the slice could not be read after the save", mode) - mine = [ - effect - for index, effect in enumerate(effects) - if routes is None or name in routes.get(str(index), (name,)) - ] - selectors = [s for effect in mine for s in declared_selectors(effect)] - - def declared(record: Mapping[str, Any]) -> bool: + if surface_tables.get(name) in hashers: + # A surface whose table the frame snapshots: the table, with its + # columns, defaults and write attribution, judges these records. + continue + selectors = selectors_for(name) + hasher = hashers.get(name) + + def declared(record: Mapping[str, Any], hashed: bool = False) -> bool: + if hashed and hasher is not None: + return any( + all(record.get(k) == hasher(k, v) for k, v in s.items()) + for s in selectors + ) return any(record_matches(dict(record), s) for s in selectors) before_map: dict[Any, dict[str, Any]] = {} @@ -485,16 +1052,16 @@ def declared(record: Mapping[str, Any]) -> bool: ("post-save", after_records, after_map), ): for record in records: - key = key_of(record) + key = key_of(record, name) if key is None: return _not_checked( - f"a {label} record carries no {spec.identity_field!r}; " + f"a {label} record carries no {identity_of(name)!r}; " "the slice can't be compared record by record", mode, ) - if key in into and spec.identity_field: + if key in into and identity_of(name): return _not_checked( - f"two {label} records share one {spec.identity_field!r}; " + f"two {label} records share one {identity_of(name)!r}; " "the slice can't be compared record by record", mode, ) @@ -504,38 +1071,79 @@ def declared(record: Mapping[str, Any]) -> bool: for key in set(before_map) | set(after_map): before = before_map.get(key) after = after_map.get(key) - is_declared = (before is not None and declared(before)) or ( - after is not None and declared(after) - ) + is_declared = ( + before is not None and declared(before, hashed=hasher is not None) + ) or (after is not None and declared(after)) if is_declared: - declared_keys.add(str(key)) - if other_on_target(key, before, after): - # The saved value may not stand: never excused. + declared_keys.add(str(ref_of(name, key, before, after)[0])) + if before is None and after is not None: + # A new declared record. In a database table, with write + # and insert events to tell actors apart, anything it sets + # beyond the declared fields must be its default or + # another actor's later write. Elsewhere nothing says + # which fields another actor touched: their entry blocks. + separable = ( + name in hashers + and attribution is not None + and bool(attribution.insert_events) + ) + if not separable and other_on_target(name, key, None, after): + collateral.append( + FrameChange( + name, + "changed", + "other", + [], + on_target=True, + other_on_target=True, + key=key, + ) + ) + continue + new_row(name, key, after) + continue + if after is None: + continue # a removed target: the delta check's case + fields = changed(name, before, after) # type: ignore[arg-type] + stray = [f for f in fields if f not in may_change] + if other_on_target(name, key, before, after): + ref, field = ref_of(name, key, before, after) + if ( + fields + and not (set(fields) & may_change) + and attribution is not None + and attribution.write_events + and attribution.table_field + and not (actors(name, ref, field) & self_ids) + ): + # Another actor wrote, by a write to this table, only + # fields this save doesn't declare, and the bot wrote + # nothing to this record: theirs. A declared field + # moved by another actor never passes (the saved + # value may not stand), and nor does a record both + # wrote, since nothing says whose change is whose. + concurrent.append( + FrameChange(name, "changed", "other", fields, key=key) + ) + continue collateral.append( FrameChange( name, "changed", "other", - _changed_fields(before or {}, after or {}, ignore), + fields, on_target=True, other_on_target=True, key=key, ) ) continue - if before is None or after is None: - continue # a new or removed target: the delta check's case - stray = [ - f - for f in _changed_fields(before, after, ignore) - if f not in may_change - ] if stray: note( FrameChange( name, "changed", - attribute(key, before, after), + attribute(name, key, before, after), stray, on_target=True, key=key, @@ -543,32 +1151,85 @@ def declared(record: Mapping[str, Any]) -> bool: ) continue if before is None and after is not None: - note( - FrameChange( - name, - "added", - attribute(key, None, after), - sorted(_without(after, ignore)), - key=key, - ) + side_index = next( + ( + i + for i, side in enumerate(spec.side_records) + if side.surface == name + and str(after.get(side.link_field)) in side_links[i] + ), + None, ) + if side_index is not None: + # A side record of a new declared record: part of the save. + side = spec.side_records[side_index] + link = str(after.get(side.link_field)) + side_found[side_index][link] += 1 + declared_keys.add(str(ref_of(name, key, None, after)[0])) + new_row(name, key, after, (side.link_field,)) + continue + if bookkeeping(name, None, after, []): + continue + theirs = any( + side.surface == name + and str(after.get(side.link_field)) in other_links[i] + for i, side in enumerate(spec.side_records) + ) + change = FrameChange( + name, + "added", + "other" if theirs else attribute(name, key, None, after), + sorted(_without(after, ignore_for(name))), + key=key, + ) + note(change) + view = views_of.get(name) + if change.attribution == "other" and view and spec.identity_field: + # The same record as its surface names it, for checks + # that read the surface (an exact_new_set's additions). + added_by_others.add((view, str(after.get(spec.identity_field)))) elif after is None and before is not None: + if bookkeeping(name, before, None, []): + continue note( - FrameChange(name, "removed", attribute(key, before, None), key=key) + FrameChange( + name, "removed", attribute(name, key, before, None), key=key + ) ) elif before is not None and after is not None: - fields = _changed_fields(before, after, ignore) + fields = changed(name, before, after) + if fields and bookkeeping(name, before, after, fields): + continue if fields: note( FrameChange( name, "changed", - attribute(key, before, after), + attribute(name, key, before, after), fields, key=key, ) ) + for table, why in unread_changes: + # Moved in a way no row comparison explains: nothing can say which + # rows, or who moved them. + collateral.append(FrameChange(table, "changed", "unattributed", [why])) + + for i, side in enumerate(spec.side_records): + for link, found in side_found[i].items(): + if found != side.count: + # Missing (or doubled) side record: the chart is not whole. + collateral.append( + FrameChange( + side.surface, + "removed" if found < side.count else "added", + "unattributed", + [f"{found} of {side.count} side record(s)"], + key=link, + ) + ) + if mode == "audit_read": if window_entries is None: # The log could not be read. Every change stays unattributed, and @@ -595,6 +1256,9 @@ def declared(record: Mapping[str, Any]) -> bool: records_compared=compared, collateral=collateral, concurrent=concurrent, + database=database_evidence, + added_by_others=added_by_others, + excused_by=excused_by, ) verdict.reason = _reason(verdict) return verdict diff --git a/packages/flow/openadapt_flow/runtime/effects/readback_values.py b/packages/flow/openadapt_flow/runtime/effects/readback_values.py index f70d4d7df..72c47492c 100644 --- a/packages/flow/openadapt_flow/runtime/effects/readback_values.py +++ b/packages/flow/openadapt_flow/runtime/effects/readback_values.py @@ -44,7 +44,7 @@ "normalize_value", ] -ValueKind = Literal["text", "code", "date", "currency"] +ValueKind = Literal["text", "code", "date", "currency", "phone", "email"] Comparison = Literal["match", "mismatch", "ambiguous"] ReadSource = Literal["ocr", "structure"] @@ -72,6 +72,42 @@ _MONTH_DAY = ("%m/%d/%Y", "%m-%d-%Y", "%m.%d.%Y") +#: A phone number: digits and the punctuation people type between them, with +#: at most one extension at the end. Anything else (a word, a note) makes the +#: value not a phone number, so it never matches one. +_PHONE = re.compile( + r"^\s*(?P
[\d\s()+\-./]+?)\s*" + r"(?:(?:x|ext\.?|extension)\s*(?P\d+))?\s*$", + re.IGNORECASE, +) +#: An ISO date, alone or followed by a time of day only. +_ISO_WITH_TIME = re.compile( + r"^(\d{4}-\d{2}-\d{2})(?:[ T]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?)?$" +) + + +def _phone(value: str) -> Optional[str]: + """The digits, with an extension kept apart (``6175550142x12``).""" + + match = _PHONE.match(value) + if match is None: + return None + digits = re.sub(r"\D", "", match["main"]) + if not digits: + return None + return digits + (f"x{match['ext']}" if match["ext"] else "") + + +def _email(value: str) -> Optional[str]: + """One address, case folded (the local part too: in practice mail + systems treat it case-insensitively, though the RFCs don't require it).""" + + text = value.strip() + if text.count("@") != 1 or any(c.isspace() for c in text): + return None + return text.lower() + + def _text(value: str) -> str: return " ".join(value.lower().split()) @@ -84,6 +120,9 @@ def _dates(value: str, order: Optional[str]) -> set[date]: """Every calendar date ``value`` can mean.""" cleaned = " ".join(value.split()) + stamped = _ISO_WITH_TIME.match(cleaned) + if stamped: + cleaned = stamped.group(1) # a stored date-time: its date found: set[date] = set() for layout in _DATE_FORMATS: try: @@ -136,6 +175,10 @@ def normalize_value( if kind == "code": return _code(value) or None + if kind == "phone": + return _phone(value) + if kind == "email": + return _email(value) if kind == "date": found = _dates(value, date_order) return found.pop().isoformat() if len(found) == 1 else None @@ -169,6 +212,12 @@ def compare_value( """ ocr = source == "ocr" + if kind in ("phone", "email"): + seen_form = normalize_value(observed, kind) + want_form = normalize_value(expected, kind) + if seen_form is not None and seen_form == want_form: + return "match" + return "mismatch" if kind == "date": seen = _dates(observed, date_order) want = _dates(expected, date_order) diff --git a/packages/flow/openadapt_flow/runtime/effects/sealed.py b/packages/flow/openadapt_flow/runtime/effects/sealed.py new file mode 100644 index 000000000..77056ad2f --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/effects/sealed.py @@ -0,0 +1,102 @@ +"""Keep a baseline's records as keyed MACs, and rebuild them when judging. + +verify-run's baseline is what lets a check tell a record the save added from +one that was there, and see what else moved. It used to hold the read set's +rows as values: a copy of the records on disk. A sealed baseline holds, per +record, a keyed MAC of its identity and one of each field (keyed BLAKE2b, 128 bits, bound +to its surface and field, with the database frame's key: the run's salt +keyed by a host secret never written into the baseline), never a value. + +At judging time one fresh read rebuilds the records as they were: + +- a record whose identity and fields all match is the record now, as read; +- a field whose MAC differs is set to :data:`CHANGED`, which no stored value + can equal, so it shows as changed; +- a field the record didn't have stays absent; +- a record that is gone comes back as a placeholder whose identity is + :data:`REMOVED` plus its MAC, so it shows as removed. + +So every check downstream (new records, an exact new set, collateral +changes, the frame) works on the rebuilt records unchanged. Values never +reach the baseline file. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any, Optional + +from openadapt_flow.runtime.effects.db_snapshot import ( + canonical, + cell_hash, + row_key_hash, +) + +#: A rebuilt field whose value moved since the baseline. +CHANGED = "\x00changed" +#: The identity prefix of a rebuilt record that is gone. +REMOVED = "\x00removed:" + + +def _identity_text(record: Mapping[str, Any], identity_field: Optional[str]) -> str: + if identity_field and identity_field in record: + return canonical(record[identity_field]) + if "id" in record: + return canonical(record["id"]) + return json.dumps(record, sort_keys=True, default=str) + + +def seal( + records: Sequence[Mapping[str, Any]], + key: bytes, + identity_field: Optional[str], + surface: str = "", +) -> list[dict[str, Any]]: + """Each record's identity MAC and field MACs (bound to the surface and + field), in read order. ``key`` is the run's MAC key (``mac_key``).""" + + where = f"surface:{surface}" + return [ + { + "id": row_key_hash(key, _identity_text(r, identity_field), where), + "cells": {f: cell_hash(key, v, where, f) for f, v in r.items()}, + } + for r in records + ] + + +def unseal( + sealed: Sequence[Mapping[str, Any]], + current: Sequence[Mapping[str, Any]], + key: bytes, + identity_field: Optional[str], + surface: str = "", +) -> list[dict[str, Any]]: + """The baseline's records, rebuilt from a read now (see the module).""" + + where = f"surface:{surface}" + now: dict[str, Mapping[str, Any]] = {} + for record in current: + now.setdefault( + row_key_hash(key, _identity_text(record, identity_field), where), record + ) + out: list[dict[str, Any]] = [] + for entry in sealed: + cells: Mapping[str, str] = entry["cells"] + record = now.get(str(entry["id"])) + if record is None: + gone = REMOVED + str(entry["id"]) + out.append({f: CHANGED for f in cells} | {identity_field or "id": gone}) + continue + out.append( + { + f: ( + record.get(f) + if cell_hash(key, record.get(f), where, f) == mac + else CHANGED + ) + for f, mac in cells.items() + } + ) + return out diff --git a/packages/flow/openadapt_flow/verify_run.py b/packages/flow/openadapt_flow/verify_run.py index 5f577636e..a03277fb4 100644 --- a/packages/flow/openadapt_flow/verify_run.py +++ b/packages/flow/openadapt_flow/verify_run.py @@ -51,6 +51,8 @@ import os import re import secrets +import time +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Any, Literal, Mapping, Optional, Sequence, cast @@ -145,6 +147,10 @@ #: Where the baseline keeps the audit log read for frame attribution. Not a #: surface name: those are plain identifiers. FRAME_AUDIT_DETAIL = "openadapt:frame_audit" +#: Where the baseline keeps the whole-database snapshot (``frame.database``). +FRAME_DB_DETAIL = "openadapt:frame_database" +#: Where the baseline keeps the read set as keyed MACs (``effects/sealed.py``). +SEALED_DETAIL = "openadapt:sealed_records" #: ``checked`` (``docs/VERIFICATION_COVERAGE.md``): which parts of the claim #: behind ``verified`` were checked, and how each came out. @@ -340,6 +346,14 @@ def parse_verify_config(data: Any, *, source: str = "verify config") -> VerifyCo "frame.attribution.read must be one read-only system-of-record " "read (rest, sql, fhir, graphql, ...)" ) + database = cfg.frame.database if cfg.frame is not None else None + if database is not None: + surfaces = set(cfg.verifier.surfaces) or {""} + unknown = sorted(set(database.views.values()) - surfaces) + if unknown: + raise VerifyRunError( + "frame.database.views names no such surface: " + ", ".join(unknown) + ) for effect in cfg.effects: if effect.needs_operator_confirmation: raise VerifyRunError( @@ -403,6 +417,356 @@ def _audit_verifier(cfg: VerifyConfig, params: Mapping[str, object]) -> Any: raise VerifyRunError(f"cannot build the audit read: {exc}") from exc +def _db_connect(read: Mapping[str, Any]) -> Any: + """A connect function for ``frame.database.read`` (read-only).""" + + if read.get("sqlite_database"): + import sqlite3 + + uri = Path(str(read["sqlite_database"])).resolve().as_uri() + "?mode=ro" + return lambda: sqlite3.connect(uri, uri=True) + import importlib + + from openadapt_flow.deployment import _require_env + + module = importlib.import_module(str(read["sql_driver"])) + args = dict(read.get("sql_connect_args") or {}) + if read.get("sql_password_env"): + args["password"] = _require_env( + str(read["sql_password_env"]), "frame.database.read.sql_password_env" + ) + return lambda: module.connect(**args) + + +def _same_connection( + audit: Optional[Mapping[str, Any]], read: Mapping[str, Any] +) -> bool: + """Does the audit read use the database read's own connection?""" + + if not audit or audit.get("kind") != "sql": + return False + keys = ("sqlite_database", "sql_driver", "sql_connect_args") + return all(audit.get(k) == read.get(k) for k in keys) + + +def _audit_sql(cfg: VerifyConfig) -> dict[str, str]: + """Reads that run inside the database snapshot: the audit query (when it + shares the connection), and the acting user's ids (for a profile's + own-row rules).""" + + frame = cfg.frame + if frame is None or frame.database is None: + return {} + out: dict[str, str] = {} + profile = frame.database.loaded_profile() + if profile is not None: + query = profile.actor_ids_query(_self_names(cfg), frame.database.dialect) + if query: + out["actor_ids"] = query + users = profile.users_query(frame.database.dialect) + if users: + out["users_map"] = users + if frame.attribution is None: + return out + audit = frame.attribution.read + if ( + _same_connection(audit, frame.database.read) + and audit + and audit.get("sql_query") + ): + out["audit"] = str(audit["sql_query"]) + return out + + +def _self_names(cfg: VerifyConfig) -> list[str]: + """The bot's own sign-in names the frame lists (literal ``self``).""" + + a = cfg.frame.attribution if cfg.frame is not None else None + if a is None: + return [] + return sorted({v.literal for v in a.self_actors if v.literal}) + + +def _ref_columns(cfg: VerifyConfig) -> list[str]: + """Columns attribution and side records read: kept as cell MACs in + every table, however large.""" + + frame = cfg.frame + if frame is None: + return [] + cols = {frame.identity_field} if frame.identity_field else set() + a = frame.attribution + if a is not None: + cols |= {a.record_key} if a.record_key else set() + cols |= set(a.record_keys.values()) + for side in frame.side_records: + cols |= {side.link_field, side.of_field} + profile = frame.database.loaded_profile() if frame.database else None + if profile is not None: + cols |= set(profile.ref_columns()) + return sorted(c for c in cols if c) + + +def _read_database( + cfg: VerifyConfig, salt: bytes, before: Optional[Any] = None +) -> Optional[Any]: + """The whole-database snapshot, or None when it can't be read.""" + + from openadapt_flow.runtime.effects.db_snapshot import mac_key, take_snapshot + + database = cfg.frame.database if cfg.frame is not None else None + if database is None: + return None + try: + return take_snapshot( + _db_connect(database.read), + database.dialect, + key=mac_key(salt), + modes=database.modes(), # type: ignore[arg-type] + max_rows=database.max_rows, + before=before, + extra_queries=_audit_sql(cfg), + ignore=database.all_ignore_columns(), + ref_columns=_ref_columns(cfg), + schemas=database.schemas, + optional_modes=database.profile_modes(), # type: ignore[arg-type] + ) + except Exception: # noqa: BLE001 - unreadable -> the frame is not checked + return None + + +def _mac_key(salt: bytes) -> bytes: + from openadapt_flow.runtime.effects.db_snapshot import mac_key + + return mac_key(salt) + + +def _audit_ids_each(entries: Sequence[Mapping[str, Any]], salt: bytes) -> list[str]: + from openadapt_flow.runtime.effects.effect import stable_id + + return [ + hmac.new( + _mac_key(salt), repr(stable_id(dict(e))).encode(), hashlib.sha256 + ).hexdigest() + for e in entries + ] + + +def _audit_ids(entries: Sequence[Mapping[str, Any]], salt: bytes) -> list[str]: + """Keyed hashes of audit entries: the baseline keeps which entries + existed, never what they say.""" + + from openadapt_flow.runtime.effects.effect import stable_id + + return sorted( + hmac.new( + _mac_key(salt), repr(stable_id(dict(e))).encode(), hashlib.sha256 + ).hexdigest() + for e in entries + ) + + +@dataclass +class _DatabaseFrame: + slices: dict[str, tuple[EffectState, Optional[list[dict[str, Any]]]]] + defaults: dict[str, dict[str, Optional[str]]] + unread: list[tuple[str, str]] + hashers: dict[str, Any] + clock: tuple[str, str] + evidence: dict[str, Any] + audit_after: Optional[list[dict[str, Any]]] + #: The read account can't see triggers: what the save caused can't be + #: ruled out, so the frame is not checked. + blind: bool = False + #: The two reads after the save differed: the database was still moving. + still_moving: bool = False + #: The app profile's judgment of one row change (``AppProfile.excuse``). + excuse: Any = None + #: Every user's id by sign-in name (in memory only), and the names and + #: ids of other users who wrote in the window. + users_map: dict[str, str] = field(default_factory=dict) + colleagues: tuple[set[str], set[str]] = (set(), set()) + + +def _rules_used(profile: Any, old: Any, new: Any) -> list[str]: + """The profile's unconditional rules that applied in this run: an + append-only table that gained rows, or a row of a compared table whose + only changes were in a column rule's columns. (Conditional rules are + reported by the judge when they excuse a row.)""" + + from openadapt_flow.app_profiles import table_name + from openadapt_flow.runtime.effects.db_snapshot import unpack + + used = [] + for rule in profile.rules: + if rule.conditional: + continue + tables = [n for n in new.tables if table_name(n) == table_name(rule.table)] + for name in tables: + a, b = old.tables.get(name), new.tables.get(name) + if a is None or b is None: + continue + if rule.mode == "append_only" and len(b.rows.keys() - a.rows.keys()): + used.append(rule.id) + break + cols = [c for c in rule.columns if c in a.cell_columns] + if ( + cols + and a.cell_columns == b.cell_columns + and any( + unpack(a.cell_columns, a.rows[k][1])[c] + != unpack(b.cell_columns, b.rows[k][1])[c] + for k in a.rows.keys() & b.rows.keys() + for c in cols + ) + ): + used.append(rule.id) + break + return used + + +def _database_frame( + cfg: VerifyConfig, before: EffectState, salt: bytes +) -> Optional[_DatabaseFrame]: + """The rows that moved, as slices, from the baseline and one read now. + + Only rows whose contents MAC differs (changed), and rows on one side only + (added or removed), reach the judge: a baseline row as its MACs, a row now + as read. None when there is no ``frame.database``. A baseline without a + readable snapshot, or an unreadable read now, gives one unreachable + slice, so the frame is not checked. + """ + + from openadapt_flow.runtime.effects.db_snapshot import ( + KEY, + ROW, + Snapshot, + SnapshotError, + cell_hash, + mac_key, + same_state, + unpack, + unread_reasons, + ) + + database = cfg.frame.database if cfg.frame is not None else None + if database is None: + return None + unreachable = _DatabaseFrame( + {"database": (EffectState(substrate="database", reachable=False), None)}, + {}, + [], + {}, + ("", ""), + {}, + None, + ) + stored = ( + before.detail.get(FRAME_DB_DETAIL) if isinstance(before.detail, dict) else None + ) + if not isinstance(stored, dict): + return unreachable + old = Snapshot.from_json(stored) + new = _read_database(cfg, salt, before=old) + if new is None: + return unreachable + # Read again a moment later: the two reads must match, or the app is + # still writing and what the save caused can't be told yet. + time.sleep(database.settle_s) + again = _read_database(cfg, salt, before=old) + if again is None: + return unreachable + if not same_state(new, again): + moving = _DatabaseFrame({}, {}, [], {}, ("", ""), {}, None) + moving.still_moving = True + return moving + if not (old.code_visible and new.code_visible): + blind = _DatabaseFrame({}, {}, [], {}, ("", ""), {}, None) + blind.blind = True + return blind + key = mac_key(salt) + out = _DatabaseFrame({}, {}, [], {}, (old.clock, new.clock), {}, None) + if "audit" in new.extra: + out.audit_after = list(new.extra["audit"]) + if old.objects != new.objects: + out.unread.append( + ("(database code)", "triggers, views, routines or events changed") + ) + counts = {"compare": 0, "append_only": 0, "skip": 0} + for name in sorted(set(old.tables) | set(new.tables)): + a, b = old.tables.get(name), new.tables.get(name) + try: + reasons = unread_reasons(a, b, new.dialect) + except SnapshotError as exc: + reasons = [str(exc)] + out.unread.extend((name, why) for why in reasons) + if b is not None: + counts[b.mode] = counts.get(b.mode, 0) + 1 + if ( + a is None + or b is None + or a.mode != "compare" + or b.mode != "compare" + or a.cell_columns != b.cell_columns + or (reasons and reasons[0] in ("definition changed", "created or dropped")) + ): + continue + gone = [k for k in a.rows if k not in b.rows or b.rows[k][0] != a.rows[k][0]] + now = [k for k in b.rows if k not in a.rows or a.rows[k][0] != b.rows[k][0]] + before_rows = [ + {KEY: k, ROW: a.rows[k][0], **unpack(a.cell_columns, a.rows[k][1])} + for k in gone + ] + after_rows = [{KEY: k, ROW: b.rows[k][0], **b.values.get(k, {})} for k in now] + out.slices[name] = ( + EffectState(substrate="database", reachable=True, records=before_rows), + after_rows, + ) + out.defaults[name] = dict(b.defaults) + out.hashers[name] = ( + lambda table: lambda column, value: cell_hash(key, value, table, column) + )(name) + out.evidence = { + "tables_compared": counts.get("compare", 0), + "tables_append_only": counts.get("append_only", 0), + "tables_skipped": counts.get("skip", 0), + "tables_without_snapshot_isolation": sum( + 1 for t in new.tables.values() if t.non_transactional + ), + "schemas": list(new.schemas), + "allow_list_sha256": database.allow_list_sha256(), + } + profile = database.loaded_profile() + if profile is not None: + out.evidence["profile"] = { + "version": profile.version, + "sha256": profile.sha256(), + "rules_used": _rules_used(profile, old, new), + } + names = set(_self_names(cfg)) + ids = {str(r.get("id")) for r in new.extra.get("actor_ids", [])} + hashers = out.hashers + out.users_map = { + str(r.get("name")): str(r.get("id")) for r in new.extra.get("users_map", []) + } + frame_out = out + + def excuse(name: str, before: Any, after: Any, fields: list[str]) -> Any: + return profile.excuse( + name, + before, + after, + fields, + hashers[name], + names, + ids, + frame_out.colleagues, + ) + + out.excuse = excuse + return out + + def _read_audit(verifier: Any) -> Optional[list[dict[str, Any]]]: if verifier is None: return None @@ -421,6 +785,7 @@ def judge_run_frame( *, audit_verifier: Any = None, run_actors: Sequence[str] = (), + salt: bytes = b"", ) -> FrameVerdict: """Read the slice after the save and judge the frame against ``before``.""" @@ -472,18 +837,83 @@ def read_of(name: str) -> str: ) else: slices[""] = (before, _post_records(after)) + identities: Optional[dict[str, str]] = None + db = _database_frame(cfg, before, salt) + if db is not None and db.still_moving: + return FrameVerdict( + status="not_checked", + reason=( + "the database was still changing after the save (two reads " + "differed); what the save caused can't be told yet" + ), + attribution=spec.attribution.kind if spec.attribution else "none", + ) + if db is not None and db.blind: + return FrameVerdict( + status="not_checked", + reason=( + "the database read account can't see triggers (on MySQL it " + "needs TRIGGER), so a trigger planted in the window could pass" + ), + attribution=spec.attribution.kind if spec.attribution else "none", + ) + if db is not None: + from openadapt_flow.runtime.effects.db_snapshot import KEY + + clash = sorted(set(db.slices) & set(slices)) + if clash: + return FrameVerdict( + status="not_checked", + reason="a table has the same name as a surface: " + ", ".join(clash), + ) + if routes is None: + routes = {str(i): [""] for i in range(len(bound_effects))} + views = spec.database.views if spec.database is not None else {} + routes = {k: list(v) for k, v in routes.items()} + for own in routes.values(): + own.extend(t for t, surface in views.items() if surface in own) + slices.update(db.slices) + identities = {name: KEY for name in db.slices} audit_before: Optional[list[dict[str, Any]]] = None audit_after: Optional[list[dict[str, Any]]] = None - if audit_verifier is not None: - stored = ( - before.detail.get(FRAME_AUDIT_DETAIL) - if isinstance(before.detail, dict) - else None + stored = ( + before.detail.get(FRAME_AUDIT_DETAIL) + if isinstance(before.detail, dict) + else None + ) + if isinstance(stored, dict) and stored.get("reachable"): + entries_now = ( + db.audit_after + if db is not None and db.audit_after is not None + else _read_audit(audit_verifier) ) - if isinstance(stored, dict) and stored.get("reachable"): - audit_before = list(stored.get("records", [])) - audit_after = _read_audit(audit_verifier) - return judge_frame( + if entries_now is not None: + if "ids" in stored: + # The window: entries now that the baseline didn't hold. + seen = set(stored["ids"]) + audit_before = [] + audit_after = [ + e + for e, h in zip(entries_now, _audit_ids_each(entries_now, salt)) + if h not in seen + ] + else: + audit_before = list(stored.get("records", [])) + audit_after = entries_now + if db is not None and audit_after is not None and spec.attribution is not None: + # Other users who wrote in the window: their own per-user state is + # their work (the profile's own_rows rules only). + a = spec.attribution + writes = set(a.write_events) + mine = set(_self_names(cfg)) | {str(x) for x in run_actors} + active = { + str(e.get(a.actor_field)) + for e in audit_after + if (not writes or str(e.get(a.event_field or "")) in writes) + and e.get(a.actor_field) not in (None, "") + } - mine + db.colleagues = (active, {db.users_map[n] for n in active if n in db.users_map}) + verdict = judge_frame( spec, bound_effects, slices, @@ -492,7 +922,96 @@ def read_of(name: str) -> str: audit_before=audit_before, audit_after=audit_after, run_actors=run_actors, + identities=identities, + new_row_defaults=None if db is None else db.defaults, + unread_changes=() if db is None else db.unread, + hashers=None if db is None else db.hashers, + clock=("", "") if db is None else db.clock, + database_evidence=None if db is None else db.evidence, + excuse=None if db is None else db.excuse, ) + profile = db.evidence.get("profile") if db is not None else None + if profile is not None: + # The rules that actually excused something: unconditional ones seen + # in the snapshots, conditional ones as the judge applied them. + profile["rules_used"] = sorted(set(profile["rules_used"]) | verdict.excused_by) + return verdict + + +def _excuse_attributed_additions( + cfg: VerifyConfig, + verifier: Any, + before: EffectState, + effects: Sequence[Effect], + verdicts: list[EffectVerdict], + rows: list[dict[str, Any]], + frame: FrameVerdict, +) -> None: + """Re-judge a refuted ``exact_new_set`` without records another user added. + + ``exact_new_set`` says the save added these records and nothing else. A + record the frame's attribution names another actor for (a colleague's + new patient in the same window) is not this save's addition. Only such + attributed additions are set aside; an unattributed one still refutes. + """ + + from openadapt_flow.runtime.effects._common import judge_records + from openadapt_flow.runtime.effects.effect import stable_id + from openadapt_flow.runtime.effects.surfaces import surface_of + + spec = cfg.frame or FrameSpec() + excused = { + (c.surface, str(c.key)) + for c in frame.concurrent + if c.change == "added" and c.key is not None + } + # When a database table is judged in place of the surface it holds, the + # frame names those additions as the surface does. + excused |= set(getattr(frame, "added_by_others", ())) + if not excused: + return + + def key_of(record: Mapping[str, Any]) -> str: + if spec.identity_field: + return str(record.get(spec.identity_field)) + ignore = set(spec.ignore_fields) + return str(stable_id({k: v for k, v in record.items() if k not in ignore})) + + for i, (effect, verdict) in enumerate(zip(effects, verdicts)): + if ( + effect.kind is not EffectKind.EXACT_NEW_SET + or verdict.verdict is not Verdict.REFUTED + ): + continue + try: + state = verifier.capture_pre_state() + except Exception: # noqa: BLE001 - unreadable: the refutation stands + continue + surface = surface_of(effect) or "" + routed = isinstance(getattr(verifier, "verifiers", None), dict) + if routed and surface: + post = (state.detail or {}).get(surface, {}) + pre = (before.detail or {}).get(surface, {}) + records = list(post.get("records", [])) if post.get("reachable") else None + base = EffectState( + substrate=str(pre.get("substrate", "")), + reachable=bool(pre.get("reachable", False)), + records=list(pre.get("records", [])), + ) + name = surface + else: + records = list(state.records) if state.reachable else None + base, name = before, "" + if records is None: + continue + kept = [r for r in records if (name, key_of(r)) not in excused] + if len(kept) == len(records): + continue + again = judge_records(effect, base, kept, substrate=verdict.substrate) + if again.verdict is Verdict.CONFIRMED: + verdicts[i] = again + rows[i]["verdict"] = _label(again.verdict) + rows[i]["observed_count"] = again.observed_count def _target_record( @@ -530,6 +1049,7 @@ def _judge_source( from openadapt_flow.provenance import ( bound_fields, + bound_maps, default_extractor, round_trip, ) @@ -545,7 +1065,14 @@ def _judge_source( } extractor = second or default_extractor(provenance) record = _target_record(effects, verdicts) if delta_held else None - result = round_trip(provenance, record, bound_fields(effects), documents, extractor) + result = round_trip( + provenance, + record, + bound_fields(effects), + documents, + extractor, + maps=bound_maps(effects), + ) evidence = result.summary() evidence["first_extractors"] = sorted( {p.extractor for p in provenance.params.values()} @@ -687,35 +1214,101 @@ def capture_baseline( identity = verifier_identity(verifier) execution_id = _new_execution_id() _tag_reads(verifier, execution_id) + salt = secrets.token_bytes(32) state = verifier.capture_pre_state() if not isinstance(state, EffectState): raise VerifyRunError("verifier returned no readable baseline") + detail: dict[str, Any] = {} + entries: Optional[list[dict[str, Any]]] = None + snapshot = None + if cfg.frame is not None and cfg.frame.database is not None: + snapshot = _read_database(cfg, salt) + if snapshot is not None: + detail[FRAME_DB_DETAIL] = snapshot.to_json() + entries = snapshot.extra.get("audit") audit_verifier = _audit_verifier(cfg, params) if audit_verifier is not None: - entries = _read_audit(audit_verifier) - state = state.model_copy( - update={ - "detail": { - **state.detail, - FRAME_AUDIT_DETAIL: { - "reachable": entries is not None, - "records": entries or [], - }, - } - } - ) + if entries is None: + entries = _read_audit(audit_verifier) + # Which entries existed, as keyed hashes: never what they say. + detail[FRAME_AUDIT_DETAIL] = { + "reachable": entries is not None, + "ids": _audit_ids(entries or [], salt), + } + if detail: + state = state.model_copy(update={"detail": {**state.detail, **detail}}) + state = _seal_state(cfg, state, salt) payload = { "schema_version": BASELINE_SCHEMA, "execution_id": execution_id, "baseline_read_at": _now(), "verifier_identity": identity, "state": state.model_dump(mode="json"), - "salt": secrets.token_bytes(32).hex(), + "salt": salt.hex(), } _write_private(Path(out), payload) return payload +def _seal_state(cfg: VerifyConfig, state: EffectState, salt: bytes) -> EffectState: + """The read set as keyed MACs: the baseline holds no record values.""" + + from openadapt_flow.runtime.effects.sealed import seal + + identity = cfg.frame.identity_field if cfg.frame is not None else None + key = _mac_key(salt) + sealed = {"": seal(state.records, key, identity)} + detail = dict(state.detail or {}) + for name, sub in (state.detail or {}).items(): + if name.startswith("openadapt:") or not isinstance(sub, dict): + continue + if isinstance(sub.get("records"), list): + sealed[name] = seal(sub["records"], key, identity, name) + detail[name] = {**sub, "records": []} + detail[SEALED_DETAIL] = sealed + return state.model_copy(update={"records": [], "detail": detail}) + + +def _unseal_state( + cfg: VerifyConfig, before: EffectState, verifier: Any, salt: bytes +) -> EffectState: + """Rebuild a sealed baseline's records from one read now.""" + + from openadapt_flow.runtime.effects.sealed import unseal + + sealed = (before.detail or {}).get(SEALED_DETAIL) + if not isinstance(sealed, dict): + return before # an older baseline that kept the records + try: + now = verifier.capture_pre_state() + except Exception: # noqa: BLE001 - unreadable: nothing can be rebuilt + now = None + if not isinstance(now, EffectState) or not now.reachable: + return before.model_copy(update={"reachable": False}) + identity = cfg.frame.identity_field if cfg.frame is not None else None + key = _mac_key(salt) + detail = dict(before.detail) + for name, entries in sealed.items(): + if not name: + continue + current = (now.detail or {}).get(name, {}) + if not (isinstance(current, dict) and current.get("reachable")): + detail[name] = {**detail.get(name, {}), "reachable": False} + continue + detail[name] = { + **detail.get(name, {}), + "records": unseal( + entries, list(current.get("records", [])), key, identity, name + ), + } + return before.model_copy( + update={ + "records": unseal(sealed.get("", []), list(now.records), key, identity), + "detail": detail, + } + ) + + def baseline_digest(baseline: Mapping[str, Any], salt: bytes) -> str: """Salted digest (label ``baseline``) of the five bound baseline fields.""" @@ -971,6 +1564,7 @@ def verify_run( ).total_seconds() if 0 <= elapsed <= cfg.max_execution_window_s: before = EffectState.model_validate(baseline["state"]) + before = _unseal_state(cfg, before, verifier, salt) baseline_sha = baseline_digest(baseline, salt) baseline_read_at = str(baseline["baseline_read_at"]) else: @@ -988,6 +1582,7 @@ def verify_run( verdicts: list[EffectVerdict] = [] rows: list[dict[str, Any]] = [] bound_effects: list[Effect] = [] + judged_effects: list[Effect] = [] weakest: Optional[VerificationTier] = None tiers: list[Optional[VerificationTier]] = [] for effect in cfg.effects: @@ -1010,6 +1605,7 @@ def verify_run( else _audit_effect(effect, bound) ) bound_effects.append(bound) + judged_effects.append(judged) tier = verifier_effect_tier(verifier, judged) try: verdict = verify_effect_without_mutation(verifier, judged, before) @@ -1062,12 +1658,19 @@ def verify_run( audit_verifier=_audit_verifier(cfg, params), # The executor's own id is this run, never "another user". run_actors=[str(run_record.get("executor_id") or "")], + salt=salt, ) else: frame = FrameVerdict( status="not_checked", reason="no baseline, so what else changed can't be known", ) + if frame.concurrent and any(c.change == "added" for c in frame.concurrent): + _excuse_attributed_additions( + cfg, verifier, before, judged_effects, verdicts, rows, frame + ) + delta_verdict = _aggregate(verdicts) + observed_effect = _observed(verdicts) # A record outside the declared delta moved: the save's effect on the # store is not the declared one, whatever the per-effect reads say. effect_verdict: EffectVerdictLabel = ( diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 472686119..0d69c78f0 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -339,6 +339,7 @@ "examples/claude-code-demo/policy.yaml", "examples/legacy_records/answers.demo.json", "examples/legacy_records/answers.ledger.demo.json", + "openadapt_flow/app_profiles/openemr.yaml", "openadapt_flow/console/static/console.css", "openadapt_flow/console/static/console.js", "openadapt_flow/console/static/index.html", diff --git a/packages/flow/tests/conftest.py b/packages/flow/tests/conftest.py index 84da92ac8..c4ccc8e68 100644 --- a/packages/flow/tests/conftest.py +++ b/packages/flow/tests/conftest.py @@ -45,3 +45,7 @@ def _isolate_durable_authority(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) # Offline Seals are on by default; keep each test's Seal key and batch # queue out of the real home directory. monkeypatch.setenv("OPENADAPT_SEAL_HOME", str(tmp_path / ".seal-home")) + # The frame's host secret: a per-test value, so a test never creates or + # reads the real ~/.config/openadapt/frame.key. + monkeypatch.setenv("OPENADAPT_FRAME_SECRET", "11" * 32) + monkeypatch.delenv("OPENADAPT_FRAME_SECRET_FILE", raising=False) diff --git a/packages/flow/tests/test_app_profile.py b/packages/flow/tests/test_app_profile.py new file mode 100644 index 000000000..860762b31 --- /dev/null +++ b/packages/flow/tests/test_app_profile.py @@ -0,0 +1,273 @@ +"""One per-app profile of the app's own bookkeeping, shared by the frame, +rollback and the RL grader. + +Every table a profile names is still compared: credentials only excuse the +acting user's own sign-in counters, a uuid only a backfill from NULL, and a +rule may name only a reviewed bookkeeping table. Records are made up; the +database is SQLite with OpenEMR-like tables (from the review of #1604). +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import Any, Callable + +import pytest + +from openadapt_flow.app_profiles import AppProfile, load_profile +from openadapt_flow.verify_run import capture_baseline, parse_verify_config, verify_run + +SCHEMA = """ +CREATE TABLE patient_data (id INTEGER PRIMARY KEY, pid INTEGER, fname TEXT, + lname TEXT, email TEXT, uuid TEXT); +CREATE TABLE history_data (id INTEGER PRIMARY KEY, pid INTEGER, tobacco TEXT DEFAULT '', + uuid TEXT); +CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT, authorized INTEGER, + uuid TEXT, last_updated TEXT); +CREATE TABLE users_secure (id INTEGER PRIMARY KEY, username TEXT, password TEXT, + last_update_password TEXT, login_fail_counter INTEGER DEFAULT 0, last_update TEXT); +CREATE TABLE login_mfa_registrations (user_id INTEGER, name TEXT, method TEXT, + var1 TEXT, last_challenge TEXT, PRIMARY KEY (user_id, name)); +CREATE TABLE keys (id INTEGER PRIMARY KEY, name TEXT, value TEXT); +CREATE TABLE session_tracker (uuid TEXT PRIMARY KEY, number_scripts INTEGER); +CREATE TABLE background_services (name TEXT PRIMARY KEY, active INTEGER, + running INTEGER, next_run TEXT); +CREATE TABLE user_settings (setting_user INTEGER, setting_label TEXT, + setting_value TEXT, PRIMARY KEY (setting_user, setting_label)); +CREATE TABLE sequences (id INTEGER); +CREATE TABLE log (id INTEGER PRIMARY KEY, user TEXT, patient_id INTEGER, + event TEXT, tbl TEXT); +INSERT INTO patient_data VALUES (1, 1, 'Wendell', 'Prior', 'w@example.test', 'u-1'); +INSERT INTO patient_data VALUES (3, 3, 'Mira', 'Stone', 'm@example.test', 'u-3'); +INSERT INTO history_data VALUES (1, 1, '', NULL); +INSERT INTO users VALUES (1, 'clinic2', 1, 'uu-1', '2026-01-01 00:00:00'); +INSERT INTO users VALUES (2, 'bot', 1, NULL, '2026-01-01 00:00:00'); +INSERT INTO users_secure VALUES (1, 'clinic2', 'hash1', '2026-01-01', 0, '2026-01-01'); +INSERT INTO users_secure VALUES (2, 'bot', 'hash2', '2026-01-01', 0, '2026-01-01'); +INSERT INTO login_mfa_registrations VALUES (2, 'phone', 'TOTP', 'seed', NULL); +INSERT INTO keys VALUES (1, 'oauth2key', 'k1'); +INSERT INTO session_tracker VALUES ('s1', 1); +INSERT INTO background_services VALUES ('phimail', 1, 0, '2026-10-28 00:00:00'); +INSERT INTO user_settings VALUES (1, 'gacl_protect', '1'); +INSERT INTO sequences VALUES (5); +""" +NEW = {"fname": "Ada", "lname": "Quill", "email": "ada@example.test"} + + +def _doc(db: Path, **database: Any) -> dict[str, Any]: + profile = load_profile("openemr") + return { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient_data", + }, + "record": { + "name": "patient", + "match": ["email", "lname"], + "fields": {"fname": "fname", "lname": "lname", "email": "email"}, + }, + "frame": { + "identity_field": "id", + "attribution": { + "kind": "audit_read", + "actor_field": "user", + "self": ["bot"], + "record_field": "patient_id", + "record_key": "pid", + "event_field": "event", + "write_events": ["insert", "update", "delete"], + "insert_events": ["insert"], + "delete_events": ["delete"], + "table_field": "tbl", + "read": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM log", + }, + }, + "database": { + "dialect": "sqlite", + "read": {"sqlite_database": str(db)}, + "profile": "openemr", + "profile_sha256": profile.sha256(), + "views": {"patient_data": ""}, + "new_row_fields": {"patient_data": ["pid", "uuid"]}, + "settle_s": 0, + **database, + }, + }, + } + + +def _run(tmp_path: Path, after: Callable[[sqlite3.Connection], Any]) -> dict: + tmp_path.mkdir(parents=True, exist_ok=True) + db = tmp_path / "app.sqlite" + con = sqlite3.connect(db) + con.executescript(SCHEMA) + con.commit() + con.close() + cfg = parse_verify_config(_doc(db)) + capture_baseline(cfg, NEW, tmp_path / "baseline.json") + con = sqlite3.connect(db) + con.execute( + "INSERT INTO patient_data (pid, fname, lname, email, uuid) " + "VALUES (2, 'Ada', 'Quill', 'ada@example.test', 'u-2')" + ) + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('bot', 2, 'insert', 'patient_data')" + ) + after(con) + con.commit() + con.close() + return verify_run( + cfg, + params=NEW, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="test", + baseline_path=tmp_path / "baseline.json", + ) + + +def _frame(result: dict) -> str: + return result["receipt"]["checked"]["frame"] + + +def _x(*sql: str) -> Callable[[sqlite3.Connection], None]: + return lambda con: [con.execute(s) for s in sql] + + +def test_the_apps_own_bookkeeping_holds(tmp_path: Path) -> None: + result = _run( + tmp_path, + _x( + # The acting user's own sign-in bookkeeping and UI state. + "UPDATE users_secure SET login_fail_counter = 0, last_update = '2026-10-29' " + "WHERE username = 'bot'", + "INSERT INTO user_settings VALUES (2, 'dem_expand', '1')", + # Session and scheduler state, counters, a key added lazily. + "INSERT INTO session_tracker VALUES ('s2', 1)", + "UPDATE session_tracker SET number_scripts = 9 WHERE uuid = 's1'", + "UPDATE background_services SET next_run = '2026-10-29', running = 1", + "UPDATE sequences SET id = 6", + # Backfills: a uuid from NULL, and the stamp it bumps. + "UPDATE history_data SET uuid = 'h-1' WHERE id = 1", + "UPDATE users SET uuid = 'uu-2', last_updated = '2026-10-29' WHERE id = 2", + ), + ) + assert _frame(result) == "held", result["evidence"]["frame"] + used = set(result["evidence"]["frame"]["database"]["profile"]["rules_used"]) + assert { + "users-secure-sign-in", + "user-settings", + "session-tracker", + "background-services", + "sequences", + "uuid-backfill", + "users-uuid-backfill", + "users-last-updated", + } <= used + + +@pytest.mark.parametrize( + "write", + [ + # Credentials and access (the review's silent passes). + "UPDATE users_secure SET password = 'attacker' WHERE username = 'clinic2'", + "UPDATE users_secure SET password = 'x' WHERE username = 'bot'", + "UPDATE users_secure SET login_fail_counter = 5 WHERE username = 'clinic2'", + "INSERT INTO login_mfa_registrations VALUES (1, 'evil', 'TOTP', 'seed', NULL)", + "UPDATE keys SET value = 'k2' WHERE id = 1", + "UPDATE background_services SET active = 0", + "INSERT INTO user_settings VALUES (1, 'gacl_protect2', '0')", + "UPDATE user_settings SET setting_value = '0' WHERE setting_user = 1", + # uuid: a backfill only, never a swap, an overwrite or a clear. + "UPDATE patient_data SET uuid = 'u-3' WHERE id = 1", + "UPDATE patient_data SET uuid = NULL WHERE id = 3", + "UPDATE users SET uuid = 'other' WHERE id = 1", + "UPDATE users SET last_updated = '2026-10-29' WHERE id = 1", + # Removed bookkeeping rows. + "DELETE FROM session_tracker", + "DELETE FROM keys", + "INSERT INTO keys (name, value) VALUES ('oauth2passphrase', 'p')", + "UPDATE login_mfa_registrations SET last_challenge = '2026-10-29' WHERE user_id = 2", + ], +) +def test_anything_else_in_a_bookkeeping_table_refutes( + tmp_path: Path, write: str +) -> None: + assert _frame(_run(tmp_path, _x(write))) == "refuted", write + + +@pytest.mark.parametrize( + "table", ["transactions", "lbf_data", "Patient_data", "site2.patient_data"] +) +def test_a_rule_may_name_only_a_reviewed_bookkeeping_table(table: str) -> None: + base = load_profile("openemr").model_dump(mode="json") + rule = {"id": "x", "table": table, "mode": "append_only", "reason": "no"} + with pytest.raises(ValueError, match="bookkeeping"): + AppProfile.model_validate({**base, "rules": [*base["rules"], rule]}) + + +def test_every_table_only_for_a_system_column_backfill() -> None: + base = load_profile("openemr").model_dump(mode="json") + for rule in ( + { + "id": "x", + "table": "*", + "mode": "columns", + "columns": ["email"], + "fill_only": True, + "reason": "no", + }, + { + "id": "x", + "table": "*", + "mode": "columns", + "columns": ["uuid"], + "reason": "no", + }, + ): + with pytest.raises(ValueError): + AppProfile.model_validate({**base, "rules": [*base["rules"], rule]}) + + +def test_profiles_load_only_from_the_package_by_name() -> None: + with pytest.raises(ValueError): + load_profile("../../../../tmp/evil") + + +def test_the_contract_pins_the_profile(tmp_path: Path) -> None: + db = tmp_path / "app.sqlite" + with pytest.raises(Exception, match="profile_sha256"): + parse_verify_config(_doc(db, profile_sha256="sha256:" + "0" * 64)) + + +def test_a_clean_run_lists_only_the_rules_it_used(tmp_path: Path) -> None: + result = _run(tmp_path, _x()) + assert _frame(result) == "held" + used = result["evidence"]["frame"]["database"]["profile"]["rules_used"] + assert used == ["log"] + + +def test_a_colleague_who_wrote_in_the_window_owns_their_ui_state( + tmp_path: Path, +) -> None: + active = _x( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('clinic2', 3, 'update', 'patient_data')", + "INSERT INTO user_settings VALUES (1, 'dem_expand', '1')", + ) + assert _frame(_run(tmp_path / "a", active)) == "held" + idle = _x("INSERT INTO user_settings VALUES (1, 'dem_expand', '1')") + assert _frame(_run(tmp_path / "b", idle)) == "refuted" + # Credentials never widen: an active colleague's password still refutes. + password = _x( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('clinic2', 3, 'update', 'patient_data')", + "UPDATE users_secure SET login_fail_counter = 3 WHERE username = 'clinic2'", + ) + assert _frame(_run(tmp_path / "c", password)) == "refuted" diff --git a/packages/flow/tests/test_exact_new_set_attribution.py b/packages/flow/tests/test_exact_new_set_attribution.py new file mode 100644 index 000000000..76d2ec36d --- /dev/null +++ b/packages/flow/tests/test_exact_new_set_attribution.py @@ -0,0 +1,86 @@ +"""``exact_new_set`` sets aside a record the frame attributes to another user. + +A colleague registering a different patient in the same window is not this +save's addition. Only an addition the audit log names another actor for is +set aside; an unattributed one still refutes. Records are made up. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from openadapt_flow.verify_run import capture_baseline, parse_verify_config, verify_run + + +def _db(path: Path) -> sqlite3.Connection: + con = sqlite3.connect(path) + con.execute( + "CREATE TABLE patient (id INTEGER PRIMARY KEY, pid INTEGER, lname TEXT, email TEXT)" + ) + con.execute( + "CREATE TABLE log (id INTEGER PRIMARY KEY, user TEXT, patient_id INTEGER)" + ) + con.execute("INSERT INTO patient VALUES (1, 1, 'Old', 'old@example.test')") + con.commit() + return con + + +def _run(tmp_path: Path, colleague_logged: bool) -> str: + db = tmp_path / "s.db" + con = _db(db) + cfg = parse_verify_config( + { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient", + }, + "record": { + "name": "patient", + "match": ["email"], + "exact_new_set": True, + "fields": {"email": "email", "lname": "lname"}, + }, + "frame": { + "identity_field": "pid", + "attribution": { + "kind": "audit_read", + "actor_field": "user", + "record_field": "patient_id", + "self": ["bot"], + "read": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM log", + }, + }, + }, + } + ) + params = {"email": "new@example.test", "lname": "New"} + capture_baseline(cfg, params, tmp_path / "b.json") + con.execute("INSERT INTO patient VALUES (2, 2, 'New', 'new@example.test')") + con.execute("INSERT INTO log (user, patient_id) VALUES ('bot', 2)") + con.execute("INSERT INTO patient VALUES (3, 3, 'Colleague', 'c@example.test')") + if colleague_logged: + con.execute("INSERT INTO log (user, patient_id) VALUES ('clinic2', 3)") + con.commit() + con.close() + receipt = verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="t", + baseline_path=tmp_path / "b.json", + )["receipt"] + return receipt["outcome"] + + +def test_a_colleagues_new_patient_does_not_refute_the_save(tmp_path: Path) -> None: + assert _run(tmp_path, colleague_logged=True) == "verified" + + +def test_an_unattributed_new_record_still_refutes(tmp_path: Path) -> None: + assert _run(tmp_path, colleague_logged=False) == "reconciliation_required" diff --git a/packages/flow/tests/test_frame_database.py b/packages/flow/tests/test_frame_database.py new file mode 100644 index 000000000..a49eb0a50 --- /dev/null +++ b/packages/flow/tests/test_frame_database.py @@ -0,0 +1,685 @@ +"""The whole-database frame: every table and column, not a read-list. + +A frame over the surfaces a contract reads passes any write outside them. +``frame.database`` diffs the whole database instead, with a reviewed +allow-list for the app's own bookkeeping. Each test below is one way a write +could slip past (most from the adversarial review of #1582), judged through +``verify_run`` on SQLite. Records are made up. +""" + +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any, Callable, Optional + +import pytest + +from openadapt_flow.runtime.effects.db_snapshot import ( + CLOCK_DEFAULT, + cell_hash, + is_default, + take_snapshot, + unpack, +) +from openadapt_flow.verify_run import ( + capture_baseline, + parse_verify_config, + verify_run, +) + +SCHEMA = """ +PRAGMA journal_mode = WAL; +CREATE TABLE patient_data (id INTEGER PRIMARY KEY, pid INTEGER, fname TEXT, + lname TEXT, email TEXT, phone_cell TEXT DEFAULT '', regdate TEXT, + active INTEGER DEFAULT 1, created TEXT DEFAULT CURRENT_TIMESTAMP); +CREATE TABLE history_data (id INTEGER PRIMARY KEY, pid INTEGER, date TEXT, + tobacco TEXT DEFAULT ''); +CREATE TABLE calendar (id INTEGER PRIMARY KEY AUTOINCREMENT, pid INTEGER, title TEXT); +CREATE TABLE access (pid INTEGER, user TEXT, role TEXT DEFAULT '', + PRIMARY KEY (pid, user)); +CREATE TABLE codes (id INTEGER PRIMARY KEY, code TEXT); +CREATE TABLE contact (id INTEGER PRIMARY KEY, foreign_id INTEGER); +CREATE TABLE log (id INTEGER PRIMARY KEY, user TEXT, patient_id INTEGER, + event TEXT, tbl TEXT); +INSERT INTO patient_data (id, pid, fname, lname, email, regdate) + VALUES (1, 1, 'Wendell', 'Prior', 'w@example.test', '2026-01-01'); +INSERT INTO history_data VALUES (1, 1, '2026-01-01', ''); +INSERT INTO codes (code) VALUES ('A'), ('B'), ('C'); +""" +NEW = {"fname": "Ada", "lname": "Quill", "email": "ada@example.test"} +ATTRIBUTION = { + "kind": "audit_read", + "actor_field": "user", + "self": ["bot"], + "record_field": "patient_id", + "record_key": "pid", + "event_field": "event", + "write_events": ["insert", "update", "delete"], + "insert_events": ["insert"], + "delete_events": ["delete"], + "table_field": "tbl", +} + + +def _db(tmp_path: Path) -> Path: + tmp_path.mkdir(parents=True, exist_ok=True) + path = tmp_path / "app.sqlite" + con = sqlite3.connect(path) + con.executescript(SCHEMA) + con.commit() + con.close() + return path + + +def _doc(db: Path, frame: Optional[dict] = None, **database: Any) -> dict[str, Any]: + spec: dict[str, Any] = { + "identity_field": "id", + "side_records": [ + { + "surface": "history_data", + "link_field": "pid", + "of_surface": "patient_data", + "of_field": "pid", + } + ], + "attribution": { + **ATTRIBUTION, + "read": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM log", + }, + }, + "database": { + "dialect": "sqlite", + "read": {"sqlite_database": str(db)}, + "allow_tables": ["log"], + "views": {"patient_data": ""}, + "new_row_fields": { + "patient_data": ["pid", "regdate"], + "history_data": ["date"], + }, + **database, + }, + } + spec.update(frame or {}) + return { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient_data", + }, + "record": { + "name": "patient", + "match": ["email", "lname"], + "fields": {"fname": "fname", "lname": "lname", "email": "email"}, + }, + "frame": spec, + } + + +def _save(con: sqlite3.Connection, cols: str = "", vals: str = "") -> None: + """The app's own save: the chart, its history row, the bot's log lines.""" + + con.execute( + "INSERT INTO patient_data (id, pid, fname, lname, email, regdate" + + cols + + ") VALUES (2, 2, 'Ada', 'Quill', 'ada@example.test', '2026-10-25'" + + vals + + ")" + ) + con.execute("INSERT INTO history_data (pid, date) VALUES (2, '2026-10-25')") + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) VALUES " + "('bot', 2, 'insert', 'patient_data'), ('bot', 2, 'insert', 'history_data')" + ) + + +def _run( + tmp_path: Path, + after: Callable[[sqlite3.Connection], Any] = lambda con: None, + save: Callable[[sqlite3.Connection], None] = _save, + frame: Optional[dict] = None, + **database: Any, +) -> dict[str, Any]: + db = _db(tmp_path) + cfg = parse_verify_config(_doc(db, frame, **database)) + capture_baseline(cfg, NEW, tmp_path / "baseline.json") + con = sqlite3.connect(db) + save(con) + after(con) + con.commit() + con.close() + return verify_run( + cfg, + params=NEW, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="test", + baseline_path=tmp_path / "baseline.json", + ) + + +def _frame(result: dict[str, Any]) -> str: + return result["receipt"]["checked"]["frame"] + + +def test_a_clean_save_holds(tmp_path: Path) -> None: + result = _run(tmp_path) + assert _frame(result) == "held" + assert result["receipt"]["outcome"] == "verified" + + +@pytest.mark.parametrize( + "stray", + [ + "UPDATE patient_data SET phone_cell = '617-555-0142' WHERE pid = 2", + "UPDATE history_data SET tobacco = 'Never' WHERE pid = 2", + "INSERT INTO calendar (pid, title) VALUES (2, 'New Patient')", + "INSERT INTO history_data (pid, date) VALUES (2, '2026-10-25')", + "UPDATE patient_data SET email = 'x@example.test' WHERE pid = 1", + "DELETE FROM codes WHERE code = 'A'", + ], +) +def test_any_write_outside_the_declared_delta_refutes( + tmp_path: Path, stray: str +) -> None: + result = _run(tmp_path, lambda con: con.execute(stray)) + assert _frame(result) == "refuted", stray + assert result["receipt"]["outcome"] != "verified" + + +# -- the review's silent passes, one test each ------------------------------------ + + +def test_1_a_value_counts_as_default_only_if_it_is_the_declared_one( + tmp_path: Path, +) -> None: + # active defaults to 1; a new chart saved inactive is a write. + result = _run(tmp_path, save=lambda con: _save(con, ", active", ", 0")) + assert _frame(result) == "refuted" + + +def test_2_a_clock_default_must_fall_in_the_window(tmp_path: Path) -> None: + backdated = ", '2001-01-01 00:00:00'" + result = _run(tmp_path, save=lambda con: _save(con, ", created", backdated)) + assert _frame(result) == "refuted" + + +def test_3_ignore_fields_apply_to_surfaces_not_tables(tmp_path: Path) -> None: + other = "UPDATE patient_data SET regdate = '1999-01-01' WHERE pid = 1" + result = _run( + tmp_path, lambda con: con.execute(other), frame={"ignore_fields": ["regdate"]} + ) + assert _frame(result) == "refuted" + + +def test_4_a_row_written_and_removed_refutes(tmp_path: Path) -> None: + def write_then_delete(con: sqlite3.Connection) -> None: + con.execute("INSERT INTO calendar (pid, title) VALUES (1, 'x')") + con.execute("DELETE FROM calendar") + + assert _frame(_run(tmp_path, write_then_delete)) == "refuted" + + +def test_5_a_trigger_created_in_the_window_refutes(tmp_path: Path) -> None: + trigger = ( + "CREATE TRIGGER t AFTER INSERT ON history_data BEGIN " + "UPDATE patient_data SET email = 'x@example.test'; END" + ) + assert _frame(_run(tmp_path, lambda con: con.execute(trigger))) == "refuted" + + +def test_6_only_a_generated_key_may_hold_any_value(tmp_path: Path) -> None: + side = [ + { + "surface": "history_data", + "link_field": "pid", + "of_surface": "patient_data", + "of_field": "pid", + }, + { + "surface": "access", + "link_field": "pid", + "of_surface": "patient_data", + "of_field": "pid", + }, + ] + outside = "INSERT INTO access (pid, user) VALUES (2, 'outside_user')" + result = _run( + tmp_path, lambda con: con.execute(outside), frame={"side_records": side} + ) + assert _frame(result) == "refuted" + + +def test_7_a_view_entry_never_excuses_a_write(tmp_path: Path) -> None: + def viewed_then_written(con: sqlite3.Connection) -> None: + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('clinic2', 1, 'view', 'history_data')" + ) + con.execute("UPDATE history_data SET tobacco = 'Never' WHERE pid = 1") + + assert _frame(_run(tmp_path, viewed_then_written)) == "refuted" + + +def test_7_a_write_entry_for_another_table_never_excuses_a_write( + tmp_path: Path, +) -> None: + def other_table(con: sqlite3.Connection) -> None: + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('clinic2', 1, 'update', 'patient_data')" + ) + con.execute("UPDATE history_data SET tobacco = 'Never' WHERE pid = 1") + + assert _frame(_run(tmp_path, other_table)) == "refuted" + + +def test_another_users_write_to_another_chart_is_concurrent(tmp_path: Path) -> None: + def colleague(con: sqlite3.Connection) -> None: + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('clinic2', 1, 'update', 'history_data')" + ) + con.execute("UPDATE history_data SET tobacco = 'Never' WHERE pid = 1") + + assert _frame(_run(tmp_path, colleague)) == "held" + + +def test_another_users_later_write_to_the_new_chart_is_theirs(tmp_path: Path) -> None: + def colleague(con: sqlite3.Connection) -> None: + con.execute("UPDATE patient_data SET phone_cell = '617-555-0199' WHERE pid = 2") + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) " + "VALUES ('clinic2', 2, 'update', 'patient_data')" + ) + + assert _frame(_run(tmp_path, colleague)) == "held" + + +def test_the_bots_own_later_write_to_the_new_chart_still_fails(tmp_path: Path) -> None: + def both(con: sqlite3.Connection) -> None: + con.execute("UPDATE patient_data SET phone_cell = '617-555-0199' WHERE pid = 2") + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) VALUES " + "('clinic2', 2, 'update', 'patient_data'), " + "('bot', 2, 'update', 'patient_data')" + ) + + assert _frame(_run(tmp_path, both)) == "refuted" + + +# -- append-only bookkeeping, size, consistency, PHI ------------------------------- + + +def test_the_audit_log_is_append_only(tmp_path: Path) -> None: + db = _db(tmp_path) + con = sqlite3.connect(db) + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) VALUES ('x', 1, 'view', 'p')" + ) + con.commit() + con.close() + cfg = parse_verify_config(_doc(db)) + capture_baseline(cfg, NEW, tmp_path / "baseline.json") + con = sqlite3.connect(db) + _save(con) + con.execute("DELETE FROM log WHERE user = 'x'") + con.commit() + con.close() + result = verify_run( + cfg, + params=NEW, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="test", + baseline_path=tmp_path / "baseline.json", + ) + assert _frame(result) == "refuted" + + +def test_skipping_a_table_needs_a_reason(tmp_path: Path) -> None: + db = _db(tmp_path) + with pytest.raises(Exception, match="reason"): + parse_verify_config(_doc(db, allow_tables=[{"table": "log", "mode": "skip"}])) + cfg = parse_verify_config( + _doc(db, allow_tables=[{"table": "log", "mode": "skip", "reason": "test"}]) + ) + assert cfg.frame is not None and cfg.frame.database is not None + + +def test_a_large_table_holds_a_clean_save_and_catches_a_stray_one( + tmp_path: Path, +) -> None: + # Over max_rows, existing rows are compared by key bucket and the rows + # the save adds are read in full. + assert _frame(_run(tmp_path / "clean", max_rows=1)) == "held" + stray = "UPDATE patient_data SET email = 'x@example.test' WHERE pid = 1" + result = _run(tmp_path / "stray", lambda con: con.execute(stray), max_rows=1) + assert _frame(result) == "refuted" + cell = "UPDATE patient_data SET phone_cell = '1' WHERE pid = 2" + result = _run(tmp_path / "cell", lambda con: con.execute(cell), max_rows=1) + assert _frame(result) == "refuted" + + +def test_the_snapshot_is_one_consistent_read(tmp_path: Path) -> None: + db = _db(tmp_path) + opened: list[int] = [] + + class Interrupted: + """A connection whose reads let another writer commit midway.""" + + def __init__(self) -> None: + opened.append(1) + self.con = sqlite3.connect(db) + self.reads = 0 + + def cursor(self) -> Any: + outer = self + + class Cur: + description: Any = None + + def __init__(self) -> None: + self.cur = outer.con.cursor() + + def execute(self, sql: str) -> None: + outer.reads += 1 + if outer.reads == 5: + w = sqlite3.connect(db) + w.execute("INSERT INTO calendar (pid, title) VALUES (1, 'x')") + w.commit() + w.close() + self.cur.execute(sql) + self.description = self.cur.description + + def fetchall(self) -> list: + return self.cur.fetchall() + + return Cur() + + def close(self) -> None: + self.con.close() + + snap = take_snapshot(Interrupted, "sqlite", key=b"k") + assert len(opened) == 1 + calendar = snap.tables["calendar"] + assert calendar.count == len(calendar.rows or {}) + + +def test_the_baseline_holds_no_values(tmp_path: Path) -> None: + _run(tmp_path) + detail = json.loads((tmp_path / "baseline.json").read_text())["state"]["detail"] + snapshot = json.dumps(detail["openadapt:frame_database"]) + assert "Wendell" not in snapshot and "w@example.test" not in snapshot + assert "clinic2" not in json.dumps(detail["openadapt:frame_audit"]) + + +def test_the_receipt_counts_tables_compared_and_allowed(tmp_path: Path) -> None: + frame = _run(tmp_path)["evidence"]["frame"] + counts = frame["database"] + assert counts["tables_compared"] == 6 and counts["tables_append_only"] == 1 + assert counts["tables_skipped"] == 0 + assert counts["allow_list_sha256"].startswith("sha256:") + + +def test_an_unreadable_database_is_not_checked(tmp_path: Path) -> None: + db = _db(tmp_path) + doc = _doc(db) + doc["frame"]["database"]["read"] = {"sqlite_database": str(tmp_path / "gone")} + cfg = parse_verify_config(doc) + capture_baseline(cfg, NEW, tmp_path / "baseline.json") + result = verify_run( + cfg, + params=NEW, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="test", + baseline_path=tmp_path / "baseline.json", + ) + assert _frame(result) == "not_checked" + assert result["receipt"]["outcome"] != "verified" + + +def test_frame_database_needs_write_attribution(tmp_path: Path) -> None: + doc = _doc(_db(tmp_path)) + for key in ("event_field", "write_events", "insert_events", "table_field"): + doc["frame"]["attribution"].pop(key) + with pytest.raises(Exception, match="write"): + parse_verify_config(doc) + + +def test_a_table_wider_than_one_json_call_is_read(tmp_path: Path) -> None: + cols = ", ".join(f"c{i} TEXT" for i in range(150)) + path = tmp_path / "wide.sqlite" + con = sqlite3.connect(path) + con.execute(f"CREATE TABLE wide (id INTEGER PRIMARY KEY, {cols})") + con.execute("INSERT INTO wide (id, c149) VALUES (1, 'last')") + con.commit() + con.close() + snap = take_snapshot(lambda: sqlite3.connect(path), "sqlite", key=b"k") + table = snap.tables["wide"] + ((_row_mac, cells),) = table.rows.values() + assert unpack(table.cell_columns, cells)["c149"] == cell_hash( + b"k", "last", "wide", "c149" + ) + assert len(table.cell_columns) == 151 + assert "last" not in json.dumps(snap.to_json()) + + +def test_declared_defaults() -> None: + assert is_default(None, None) + assert not is_default("0", "1") and is_default("1", "1") and is_default(1, "1") + assert not is_default("", None) + assert is_default("NO", "NO") + assert is_default( + "2026-10-25 11:00:00", + CLOCK_DEFAULT, + ("2026-10-25 10:59:59", "2026-10-25 11:00:05"), + ) + assert not is_default( + "2001-01-01 00:00:00", + CLOCK_DEFAULT, + ("2026-10-25 10:59:59", "2026-10-25 11:00:05"), + ) + + +def test_a_null_default_is_no_default(tmp_path: Path) -> None: + from openadapt_flow.runtime.effects.db_snapshot import _default_text + + assert _default_text("NULL", "sqlite") is None + assert _default_text("NULL", "mysql") is None + assert _default_text("'NULL'", "sqlite") == "NULL" # the text NULL + + +def test_a_side_row_of_another_users_new_chart_is_theirs(tmp_path: Path) -> None: + contact = { + "surface": "contact", + "link_field": "foreign_id", + "of_surface": "patient_data", + "of_field": "pid", + } + history = { + "surface": "history_data", + "link_field": "pid", + "of_surface": "patient_data", + "of_field": "pid", + } + + def save(con: sqlite3.Connection) -> None: + _save(con) + con.execute("INSERT INTO contact (foreign_id) VALUES (2)") + + def colleague(con: sqlite3.Connection) -> None: + # Another user's new chart: logged per table, but its contact row is + # logged with no patient (as OpenEMR does). + con.execute( + "INSERT INTO patient_data (id, pid, fname, lname, email, regdate) " + "VALUES (3, 3, 'Theo', 'Marsh', 't@example.test', '2026-10-25')" + ) + con.execute("INSERT INTO history_data (pid, date) VALUES (3, '2026-10-25')") + con.execute("INSERT INTO contact (foreign_id) VALUES (3)") + con.execute( + "INSERT INTO log (user, patient_id, event, tbl) VALUES " + "('clinic2', 3, 'insert', 'patient_data'), " + "('clinic2', 3, 'insert', 'history_data'), " + "('clinic2', 0, 'insert', 'contact')" + ) + + frame = {"side_records": [history, contact]} + assert _frame(_run(tmp_path / "a", colleague, save=save, frame=frame)) == "held" + # A contact row for a chart nobody created in the window still refutes. + stray = "INSERT INTO contact (foreign_id) VALUES (1)" + result = _run( + tmp_path / "b", lambda con: con.execute(stray), save=save, frame=frame + ) + assert _frame(result) == "refuted" + + +# -- round 2 of the review --------------------------------------------------------- + + +def _colleague(pid: int, event: str, table: str = "history_data") -> str: + return ( + "INSERT INTO log (user, patient_id, event, tbl) " + f"VALUES ('clinic2', {pid}, '{event}', '{table}')" + ) + + +def test_l1_a_colleagues_write_on_a_large_table_is_concurrent(tmp_path: Path) -> None: + def colleague(con: sqlite3.Connection) -> None: + con.execute("UPDATE patient_data SET phone_cell = '1' WHERE pid = 1") + con.execute(_colleague(1, "update", "patient_data")) + + assert _frame(_run(tmp_path, colleague, max_rows=1)) == "held" + + +def test_l2_ignore_columns_hold_on_a_large_table(tmp_path: Path) -> None: + moved = "UPDATE patient_data SET phone_cell = 'seen' WHERE pid = 1" + result = _run( + tmp_path, + lambda con: con.execute(moved), + max_rows=1, + ignore_columns={"patient_data": ["phone_cell"]}, + ) + assert _frame(result) == "held" + + +def test_a1_a_row_moved_to_a_colleagues_chart_is_not_excused(tmp_path: Path) -> None: + def moved(con: sqlite3.Connection) -> None: + con.execute("UPDATE history_data SET pid = 3 WHERE id = 1") + con.execute(_colleague(3, "update")) + + assert _frame(_run(tmp_path, moved)) == "refuted" + + +def test_a2_an_update_entry_never_excuses_an_added_row(tmp_path: Path) -> None: + def stray(con: sqlite3.Connection) -> None: + con.execute("INSERT INTO history_data (pid, date) VALUES (1, '2026-10-25')") + con.execute(_colleague(1, "update")) + + assert _frame(_run(tmp_path, stray)) == "refuted" + + +def test_a3_one_insert_entry_excuses_one_row(tmp_path: Path) -> None: + def two(con: sqlite3.Connection) -> None: + con.execute("INSERT INTO history_data (pid, date) VALUES (1, '2026-10-25')") + con.execute("INSERT INTO history_data (pid, date) VALUES (1, '2026-10-26')") + con.execute(_colleague(1, "insert")) + + assert _frame(_run(tmp_path, two)) == "refuted" + + +def test_a4_an_update_entry_never_excuses_a_removed_row(tmp_path: Path) -> None: + def gone(con: sqlite3.Connection) -> None: + con.execute("DELETE FROM history_data WHERE pid = 1") + con.execute(_colleague(1, "update")) + + assert _frame(_run(tmp_path, gone)) == "refuted" + + +def test_a_delete_entry_excuses_a_removed_row(tmp_path: Path) -> None: + def gone(con: sqlite3.Connection) -> None: + con.execute("DELETE FROM history_data WHERE pid = 1") + con.execute(_colleague(1, "delete")) + + assert _frame(_run(tmp_path, gone)) == "held" + + +def test_the_baseline_alone_cant_be_tested_against(tmp_path: Path) -> None: + from openadapt_flow.runtime.effects.db_snapshot import cell_hash as mac + + _run(tmp_path) + base = json.loads((tmp_path / "baseline.json").read_text()) + salt = bytes.fromhex(base["salt"]) + table = base["state"]["detail"]["openadapt:frame_database"]["tables"][ + "patient_data" + ] + cells = [ + unpack(table["cell_columns"], packed)["fname"] + for _row, packed in table["rows"].values() + ] + # The salt in the baseline is not the MAC key: guesses can't be tested. + assert mac(salt, "Wendell", "patient_data", "fname") not in cells + + +def test_a_counter_on_a_column_that_is_not_the_key(tmp_path: Path) -> None: + db = tmp_path / "auto.sqlite" + con = sqlite3.connect(db) + con.executescript( + "CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT);" + "INSERT INTO t (v) VALUES ('a');" + ) + con.commit() + con.close() + before = take_snapshot(lambda: sqlite3.connect(db), "sqlite", key=b"k") + con = sqlite3.connect(db) + con.execute("INSERT INTO t (v) VALUES ('b')") + con.commit() + con.close() + after = take_snapshot( + lambda: sqlite3.connect(db), "sqlite", key=b"k", before=before + ) + from openadapt_flow.runtime.effects.db_snapshot import unread_reasons + + assert unread_reasons(before.tables["t"], after.tables["t"]) == [] + + +def test_a_read_that_cant_see_triggers_is_not_checked( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import openadapt_flow.runtime.effects.db_snapshot as ds + + real = ds.take_snapshot + + def blind(*args: Any, **kwargs: Any) -> Any: + snap = real(*args, **kwargs) + snap.code_visible = False + return snap + + monkeypatch.setattr(ds, "take_snapshot", blind) + result = _run(tmp_path) + assert _frame(result) == "not_checked" + assert result["receipt"]["outcome"] != "verified" + + +def test_a_write_between_the_two_reads_leaves_the_frame_unchecked( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The app writes again a moment after the save (a background job): the + # two reads after the save differ, so nothing is claimed yet. + import openadapt_flow.verify_run as vr + + db = tmp_path / "app.sqlite" + + def late_write(seconds: float) -> None: + con = sqlite3.connect(db) + con.execute("INSERT INTO calendar (pid, title) VALUES (2, 'late')") + con.commit() + con.close() + + monkeypatch.setattr(vr.time, "sleep", late_write) + result = _run(tmp_path) + assert _frame(result) == "not_checked" + assert result["receipt"]["outcome"] != "verified" diff --git a/packages/flow/tests/test_frame_links.py b/packages/flow/tests/test_frame_links.py new file mode 100644 index 000000000..729b9f7cd --- /dev/null +++ b/packages/flow/tests/test_frame_links.py @@ -0,0 +1,122 @@ +"""Frame links: audit entries keyed by a record field, and side records. + +OpenEMR's audit log names a patient (``pid``) while its ``pnotes`` and +``lists`` rows are told apart by their own ``id``, and its New Patient path +adds one ``history_data`` row beside each new chart. Records are made up. +""" + +from __future__ import annotations + +from openadapt_flow.runtime.effects import Effect, EffectKind +from openadapt_flow.runtime.effects.effect import EffectState +from openadapt_flow.runtime.effects.frame import ( + FrameAttribution, + FrameSideRecord, + FrameSpec, + judge_frame, +) + + +def _state(*records: dict) -> EffectState: + return EffectState( + substrate="t", reachable=True, records=[dict(r) for r in records] + ) + + +ATTRIBUTION = FrameAttribution( + kind="audit_read", + actor_field="user", + record_field="patient_id", + record_key="pid", + **{"self": ["bot"], "read": {"kind": "sql", "sql_query": "SELECT 1"}}, +) +NOTE = {"id": 10, "pid": 1, "body": "follow-up"} + + +def _notes_change(spec: FrameSpec, author: str) -> str: + verdict = judge_frame( + spec, + [Effect(kind=EffectKind.RECORD_WRITTEN, match={"lname": "Newchart"})], + {"": (_state(NOTE), [{**NOTE, "body": "changed"}])}, + audit_before=[], + audit_after=[{"id": 1, "user": author, "patient_id": "1"}], + ) + return verdict.status + + +def test_an_audit_entry_names_a_row_through_its_record_key() -> None: + spec = FrameSpec(identity_field="id", attribution=ATTRIBUTION) + # Another user's edit to patient 1's note: concurrent, not collateral. + assert _notes_change(spec, "clinic2") == "held" + # The bot's own: collateral. + assert _notes_change(spec, "bot") == "refuted" + + +def test_without_a_record_key_the_entry_can_t_name_the_row() -> None: + plain = ATTRIBUTION.model_copy(update={"record_key": None}) + spec = FrameSpec(identity_field="id", attribution=plain) + assert _notes_change(spec, "clinic2") == "refuted" # unattributed + + +PATIENT = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"lname": "Newchart"}, + count_new_only=True, + probe="surface:patient", +) +SIDE = FrameSideRecord( + surface="history", link_field="pid", of_surface="patient", of_field="pid" +) + + +def _with_history(*history_after: dict) -> str: + verdict = judge_frame( + FrameSpec(identity_field="id", side_records=[SIDE]), + [PATIENT], + { + "patient": ( + _state({"id": 1, "pid": 1, "lname": "Old"}), + [ + {"id": 1, "pid": 1, "lname": "Old"}, + {"id": 2, "pid": 2, "lname": "Newchart"}, + ], + ), + "history": ( + _state({"id": 5, "pid": 1}), + [{"id": 5, "pid": 1}, *history_after], + ), + }, + routes={"0": ["patient"]}, + ) + return verdict.status + + +def test_a_side_record_of_the_new_chart_is_part_of_the_save() -> None: + assert _with_history({"id": 6, "pid": 2}) == "held" + + +def test_a_missing_or_doubled_side_record_refutes() -> None: + assert _with_history() == "refuted" + assert _with_history({"id": 6, "pid": 2}, {"id": 7, "pid": 2}) == "refuted" + + +def test_a_row_for_another_chart_is_still_collateral() -> None: + assert _with_history({"id": 6, "pid": 2}, {"id": 7, "pid": 1}) == "refuted" + + +def test_a_table_can_name_its_record_through_another_field() -> None: + # OpenEMR's contact row points at its chart through foreign_id. + keyed = ATTRIBUTION.model_copy(update={"record_keys": {"contact": "foreign_id"}}) + contact = {"id": 3, "foreign_id": 1} + + def status(spec: FrameSpec) -> str: + return judge_frame( + spec, + [Effect(kind=EffectKind.RECORD_WRITTEN, match={"lname": "Newchart"})], + {"contact": (_state(), [contact])}, + audit_before=[], + audit_after=[{"id": 1, "user": "clinic2", "patient_id": "1"}], + ).status + + assert status(FrameSpec(identity_field="id", attribution=keyed)) == "held" + assert status(FrameSpec(identity_field="id", attribution=ATTRIBUTION)) == "refuted" diff --git a/packages/flow/tests/test_frame_secret.py b/packages/flow/tests/test_frame_secret.py new file mode 100644 index 000000000..413a4f293 --- /dev/null +++ b/packages/flow/tests/test_frame_secret.py @@ -0,0 +1,30 @@ +"""The frame's host secret: created once, whole, whatever the race.""" + +from __future__ import annotations + +import threading +from pathlib import Path + +import pytest + +from openadapt_flow.runtime.effects import db_snapshot + + +def test_racing_first_uses_share_one_whole_secret( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENADAPT_FRAME_SECRET", raising=False) + path = tmp_path / "frame.key" + monkeypatch.setenv("OPENADAPT_FRAME_SECRET_FILE", str(path)) + seen: list[bytes] = [] + threads = [ + threading.Thread(target=lambda: seen.append(db_snapshot._host_secret())) + for _ in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + assert len(set(seen)) == 1 and len(seen[0]) == 32 + assert path.stat().st_mode & 0o777 == 0o600 + assert [p.name for p in tmp_path.iterdir()] == ["frame.key"] diff --git a/packages/flow/tests/test_provenance.py b/packages/flow/tests/test_provenance.py index d8f856ec5..a9b9dd3d3 100644 --- a/packages/flow/tests/test_provenance.py +++ b/packages/flow/tests/test_provenance.py @@ -328,3 +328,90 @@ def test_a_task_whose_values_differ_from_its_provenance_is_refused( with pytest.raises(GuardRefused) as refused: load_task(path) assert refused.value.code == "task_invalid" + + +def test_round_trip_maps_a_label_to_the_stored_code() -> None: + """The referral prints the state's name; the record stores its code.""" + + from openadapt_flow.provenance import bound_fields, bound_maps + + document = IntakeDocument( + "r.txt", "text", ["Last name: Okafor", "State: Massachusetts"] + ) + sha = sha256_bytes(document.text.encode("utf-8")) + prov = ProvenanceFile( + params={ + name: ParamProvenance( + value=value, + source=SourceRef(kind="document", sha256=sha, line=line, label=label), + extractor="model:first-reader", + ) + for name, value, line, label in ( + ("lname", "Okafor", 1, "Last name"), + ("state_label", "Massachusetts", 2, "State"), + ) + } + ) + match = {"lname": {"param": "lname"}} + effects = [ + Effect(kind=EffectKind.RECORD_WRITTEN, match=match), + Effect( + kind=EffectKind.FIELD_EQUALS, + match=match, + field="state", + value={"param": "state_label", "map": {"Massachusetts": "MA"}}, + ), + ] + from openadapt_flow.provenance import default_extractor + + def judge(stored: str) -> str: + return round_trip( + prov, + {"id": 1, "lname": "Okafor", "state": stored}, + bound_fields(effects), + {sha: document}, + default_extractor(prov), + maps=bound_maps(effects), + ).status + + assert judge("MA") == "held" + assert judge("RI") == "refuted" + + +def test_round_trip_applies_the_stored_form() -> None: + """OpenEMR saves the street in capitals; the referral prints mixed case.""" + + from openadapt_flow.provenance import bound_fields, bound_maps, default_extractor + + document = IntakeDocument("r.txt", "text", ["Street: 41 Larch Way"]) + sha = sha256_bytes(document.text.encode("utf-8")) + prov = ProvenanceFile( + params={ + "street": ParamProvenance( + value="41 Larch Way", + source=SourceRef(kind="document", sha256=sha, line=1, label="Street"), + extractor="model:first-reader", + ) + } + ) + effects = [ + Effect( + kind=EffectKind.FIELD_EQUALS, + match={"id": "1"}, + field="street", + value={"param": "street", "stored": "upper"}, + ) + ] + + def judge(stored: str) -> str: + return round_trip( + prov, + {"id": 1, "street": stored}, + bound_fields(effects), + {sha: document}, + default_extractor(prov), + maps=bound_maps(effects), + ).status + + assert judge("41 LARCH WAY") == "held" + assert judge("14 LARCH WAY") == "refuted" diff --git a/packages/flow/tests/test_record_surface.py b/packages/flow/tests/test_record_surface.py new file mode 100644 index 000000000..e0ea1728c --- /dev/null +++ b/packages/flow/tests/test_record_surface.py @@ -0,0 +1,89 @@ +"""``record.surface``: a record contract read from one of several surfaces. + +The record shorthand (what ``qualify`` and ``verify-run`` read) can now name +the surface its record comes from, so a contract can also read other tables +for its frame: a stray write there refutes. Records are made up. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from openadapt_flow.verify_run import capture_baseline, parse_verify_config, verify_run + + +def _cfg(db: Path, surface: str | None = "patient") -> dict: + record = { + "name": "patient", + "match": ["email"], + "fields": {"email": "email", "lname": "lname"}, + } + if surface: + record["surface"] = surface + return { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "surfaces", + "surfaces": { + "patient": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient", + }, + "notes": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM notes", + }, + }, + }, + "record": record, + } + + +def _run(tmp_path: Path, stray_note: bool) -> str: + db = tmp_path / "s.db" + con = sqlite3.connect(db) + con.execute("CREATE TABLE patient (id INTEGER PRIMARY KEY, lname TEXT, email TEXT)") + con.execute( + "CREATE TABLE notes (id INTEGER PRIMARY KEY, patient_id INTEGER, body TEXT)" + ) + con.execute("INSERT INTO patient VALUES (1, 'Old', 'old@example.test')") + con.commit() + cfg = parse_verify_config(_cfg(db)) + params = {"email": "new@example.test", "lname": "New"} + capture_baseline(cfg, params, tmp_path / "b.json") + con.execute("INSERT INTO patient VALUES (2, 'New', 'new@example.test')") + if stray_note: + con.execute("INSERT INTO notes VALUES (1, 1, 'stray')") + con.commit() + con.close() + return verify_run( + cfg, + params=params, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="t", + baseline_path=tmp_path / "b.json", + )["receipt"]["outcome"] + + +def test_the_record_is_read_from_its_surface(tmp_path: Path) -> None: + assert _run(tmp_path, stray_note=False) == "verified" + + +def test_a_stray_write_in_another_surface_refutes(tmp_path: Path) -> None: + assert _run(tmp_path, stray_note=True) == "reconciliation_required" + + +def test_an_unset_surface_keeps_the_contract_digest(tmp_path: Path) -> None: + from openadapt_flow.qualify.campaign import record_effects # noqa: F401 + from openadapt_flow.verify_run import parse_verify_config as parse + + single = { + "schema_version": "openadapt.verify/v1", + "verifier": {"kind": "sql", "sqlite_database": "x.db", "sql_query": "SELECT 1"}, + "record": {"name": "p", "match": ["email"], "fields": {"email": "email"}}, + } + effects = parse(single).effects + assert all("surface:" not in (e.probe or "") for e in effects) diff --git a/packages/flow/tests/test_sealed_baseline.py b/packages/flow/tests/test_sealed_baseline.py new file mode 100644 index 000000000..4057f55aa --- /dev/null +++ b/packages/flow/tests/test_sealed_baseline.py @@ -0,0 +1,97 @@ +"""verify-run's baseline holds no record values: the read set is sealed. + +The baseline keeps a keyed MAC of each record's identity and fields; one +read at judging time rebuilds the records. A change, a removal and an +addition all still show. Records are made up. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from openadapt_flow.runtime.effects.sealed import CHANGED, REMOVED, seal, unseal +from openadapt_flow.verify_run import capture_baseline, parse_verify_config, verify_run + +PARAMS = {"fname": "Ada", "lname": "Quill", "email": "ada@example.test"} + + +def _setup(tmp_path: Path) -> tuple[Path, object]: + db = tmp_path / "app.sqlite" + con = sqlite3.connect(db) + con.executescript( + "CREATE TABLE patient (id INTEGER PRIMARY KEY, fname TEXT, lname TEXT, " + "email TEXT, phone TEXT);" + "INSERT INTO patient VALUES (1, 'Wendell', 'Prior', 'w@example.test', '617');" + "INSERT INTO patient VALUES (5, 'Mabel', 'Ortiz', 'm@example.test', '781');" + ) + con.commit() + con.close() + cfg = parse_verify_config( + { + "schema_version": "openadapt.verify/v1", + "verifier": { + "kind": "sql", + "sqlite_database": str(db), + "sql_query": "SELECT * FROM patient", + }, + "record": { + "name": "patient", + "match": ["email", "lname"], + "fields": {"fname": "fname", "lname": "lname", "email": "email"}, + }, + "frame": {"identity_field": "id"}, + } + ) + capture_baseline(cfg, PARAMS, tmp_path / "baseline.json") + return db, cfg + + +def _judge(tmp_path: Path, *statements: str) -> dict: + tmp_path.mkdir(parents=True, exist_ok=True) + db, cfg = _setup(tmp_path) + con = sqlite3.connect(db) + con.execute( + "INSERT INTO patient (fname, lname, email) " + "VALUES ('Ada', 'Quill', 'ada@example.test')" + ) + for statement in statements: + con.execute(statement) + con.commit() + con.close() + return verify_run( + cfg, + params=PARAMS, + run_record={"executor_kind": "agent", "executor_id": "bot"}, + authorization_ref="test", + baseline_path=tmp_path / "baseline.json", + )["receipt"] + + +def test_the_baseline_file_holds_no_record_values(tmp_path: Path) -> None: + _setup(tmp_path) + text = (tmp_path / "baseline.json").read_text() + for value in ("Wendell", "Prior", "w@example.test", "Mabel", "m@example.test"): + assert value not in text + + +def test_a_clean_save_still_verifies(tmp_path: Path) -> None: + assert _judge(tmp_path)["outcome"] == "verified" + + +def test_a_change_and_a_removal_still_show(tmp_path: Path) -> None: + changed = _judge(tmp_path / "a", "UPDATE patient SET phone = '999' WHERE id = 1") + assert changed["checked"]["frame"] == "refuted" + removed = _judge(tmp_path / "b", "DELETE FROM patient WHERE id = 5") + assert removed["checked"]["frame"] == "refuted" + + +def test_unseal_rebuilds_what_it_can_and_marks_the_rest() -> None: + key = b"k" * 32 + before = [{"id": 1, "a": "x", "b": "y"}, {"id": 2, "a": "z"}] + sealed = seal(before, key, "id") + now = [{"id": 1, "a": "x", "b": "moved"}, {"id": 3, "a": "new"}] + rebuilt = unseal(sealed, now, key, "id") + assert rebuilt[0] == {"id": 1, "a": "x", "b": CHANGED} + assert rebuilt[1]["id"].startswith(REMOVED) + assert all("new" not in str(r) for r in rebuilt) diff --git a/packages/flow/tests/test_value_kinds.py b/packages/flow/tests/test_value_kinds.py new file mode 100644 index 000000000..1cffe8103 --- /dev/null +++ b/packages/flow/tests/test_value_kinds.py @@ -0,0 +1,229 @@ +"""Declared value kinds: formatting alone never refutes, a different value +always does. + +A clerk types a phone as ``(617) 555-0142`` or a birth date as +``03/07/1984`` into a US-format field. The record check compares the +expected value and the stored one in the kind's canonical form, so the same +value passes and a transposed digit or a swapped day and month still fails. +Kinds are declared on the contract, never inferred. Records are made up. +""" + +from __future__ import annotations + +import pytest + +from openadapt_flow.runtime.effects import Effect, EffectKind +from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects.effect import ( + EffectState, + ValueExpr, + kind_form, + record_matches, +) + + +@pytest.mark.parametrize( + "kind,value,order,form", + [ + ("phone", "(617) 555-0142", None, "6175550142"), + ("phone", "617.555.0142 x12", None, "6175550142x12"), + ("phone", "617-555-0142 ext. 12", None, "6175550142x12"), + ("date", "03/07/1984", "mdy", "1984-03-07"), + ("date", "03/07/1984", "dmy", "1984-07-03"), + ("date", "03/07/1984", None, None), # ambiguous: no order declared + ("date", "1984-03-07 00:00:00", None, "1984-03-07"), + ("date", "02/31/1984", "mdy", None), + ("email", " Ada.Quill@Example.TEST ", None, "ada.quill@example.test"), + ("email", "not an address", None, None), + ], +) +def test_canonical_forms(kind: str, value: str, order, form) -> None: + assert kind_form(kind, value, order) == form + + +def _verdict(value: dict, stored: str, entered: str) -> str: + effect = Effect( + kind=EffectKind.FIELD_EQUALS, match={"id": "1"}, field="f", value=value + ).resolve({"x": entered}) + before = EffectState(substrate="t", reachable=True, records=[]) + return judge_records( + effect, before, [{"id": "1", "f": stored}], substrate="t" + ).verdict.value + + +def test_a_phone_is_compared_by_its_digits() -> None: + phone = {"param": "x", "kind": "phone"} + assert _verdict(phone, "(617) 555-0142", "617-555-0142") == "confirmed" + assert _verdict(phone, "617-555-0124", "617-555-0142") == "refuted" + # An extension is part of the number. + assert _verdict(phone, "617-555-0142 x12", "617-555-0142") == "refuted" + + +def test_a_date_is_compared_as_a_calendar_date() -> None: + dob = {"param": "x", "kind": "date", "date_order": "mdy"} + assert _verdict(dob, "1984-03-07", "03/07/1984") == "confirmed" + # A day and month swapped is a different date. + assert _verdict(dob, "1984-07-03", "03/07/1984") == "refuted" + + +def test_an_email_ignores_case() -> None: + email = {"param": "x", "kind": "email"} + assert ( + _verdict(email, "ada.quill@example.test", "Ada.Quill@Example.TEST") + == "confirmed" + ) + assert ( + _verdict(email, "ada.quil@example.test", "ada.quill@example.test") == "refuted" + ) + + +def test_without_a_kind_the_comparison_stays_exact() -> None: + assert _verdict({"param": "x"}, "(617) 555-0142", "617-555-0142") == "refuted" + + +def test_a_selector_with_a_kind_matches_in_canonical_form() -> None: + want = ValueExpr(param="email", kind="email").resolved({"email": "ADA@X.TEST"}) + assert record_matches({"email": "ada@x.test"}, {"email": want}) + assert not record_matches({"email": "eda@x.test"}, {"email": want}) + + +def test_date_order_needs_a_date_kind() -> None: + with pytest.raises(ValueError): + ValueExpr(param="x", date_order="mdy") + + +def test_an_unset_kind_keeps_the_contract_digest() -> None: + assert ValueExpr(param="x").model_dump() == {"literal": None, "param": "x"} + + +def test_exact_new_set_matches_a_declared_kind() -> None: + effect = Effect( + kind=EffectKind.EXACT_NEW_SET, + match={}, + identity_field="id", + new_records=[{"phone": {"param": "x", "kind": "phone"}}], + ).resolve({"x": "(617) 555-0142"}) + before = EffectState(substrate="t", reachable=True, records=[]) + + def verdict(stored: str) -> str: + return judge_records( + effect, before, [{"id": "1", "phone": stored}], substrate="t" + ).verdict.value + + assert verdict("617-555-0142") == "confirmed" + assert verdict("617-555-0124") == "refuted" + + +def test_a_duplicate_key_with_a_kind_matches_in_canonical_form() -> None: + email = {"param": "x", "kind": "email"} + effect = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"email": email}, + idempotency_key=email, + key_field="email", + ).resolve({"x": "Ada.Quill@Example.TEST"}) + before = EffectState(substrate="t", reachable=True, records=[]) + verdict = judge_records( + effect, before, [{"id": "1", "email": "Ada.Quill@Example.TEST"}], substrate="t" + ) + assert verdict.verdict.value == "confirmed" + + +# -- round 2 of the review --------------------------------------------------------- + + +@pytest.mark.parametrize( + "stored", + ["617-555-0142 do not call", "Tel 617 555 0142", "617-555-0142; 617-555-0199"], +) +def test_a_phone_with_any_other_text_never_matches(stored: str) -> None: + want = ValueExpr(param="p", kind="phone").resolved({"p": "617-555-0142"}) + assert not want.matches(stored) + + +@pytest.mark.parametrize( + "value,form", + [ + ("1984-03-07 00:00:00", "1984-03-07"), + ("1984-03-07T23:30:00", "1984-03-07"), + ("1984-03-07 junk", None), + ("1984-03-07T23:30:00-05:00", None), + ], +) +def test_only_a_time_of_day_may_follow_an_iso_date(value: str, form) -> None: + assert kind_form("date", value) == form + + +def test_resolve_returns_the_entered_value() -> None: + # Kinds apply only where values are compared, so an on-screen read-back + # or an undo request sees what was typed. + expr = ValueExpr(param="p", kind="phone") + assert expr.resolve({"p": "(617) 555-0142"}) == "(617) 555-0142" + assert expr.resolved({"p": "(617) 555-0142"}).matches("617-555-0142") + + +def test_the_screen_read_back_and_the_record_check_share_one_normalizer() -> None: + from openadapt_flow.runtime.effects.readback_values import compare_value + + assert compare_value("(617) 555-0142", "617-555-0142", kind="phone") == "match" + assert ( + compare_value("617-555-0142 call me", "617-555-0142", kind="phone") + == "mismatch" + ) + assert compare_value("ADA@X.TEST", "ada@x.test", kind="email") == "match" + + +def _round_trip(printed: str, stored: str, order) -> str: + from openadapt_flow.intake.document import IntakeDocument + from openadapt_flow.provenance import ( + ParamProvenance, + ProvenanceFile, + SourceRef, + bound_fields, + bound_maps, + default_extractor, + round_trip, + sha256_bytes, + ) + + document = IntakeDocument("r.txt", "text", [f"Date of birth: {printed}"]) + sha = sha256_bytes(document.text.encode("utf-8")) + prov = ProvenanceFile( + params={ + "DOB": ParamProvenance( + value=printed, + source=SourceRef( + kind="document", sha256=sha, line=1, label="Date of birth" + ), + extractor="model:first-reader", + value_kind="date", + date_order=order, + ) + } + ) + effects = [ + Effect( + kind=EffectKind.FIELD_EQUALS, + match={"id": "1"}, + field="DOB", + value={"param": "DOB", "kind": "date", "date_order": "mdy"}, + ) + ] + return round_trip( + prov, + {"id": 1, "DOB": stored}, + bound_fields(effects), + {sha: document}, + default_extractor(prov), + maps=bound_maps(effects), + ).status + + +def test_the_document_is_read_in_its_own_date_order() -> None: + # The referral prints 03/07/1984 day-first (7 March): the app, set to + # month-day, stored 1984-03-07 from a typed "03/07/1984". The round trip + # reads the document day-first and refutes the misread. + assert _round_trip("03/07/1984", "1984-03-07", "dmy") == "refuted" + assert _round_trip("03/07/1984", "1984-07-03", "dmy") == "held" + # With no order for the document, a date it prints two ways stays unchecked. + assert _round_trip("03/07/1984", "1984-03-07", None) == "not_checked" diff --git a/packages/flow/tests/test_value_stored_form.py b/packages/flow/tests/test_value_stored_form.py new file mode 100644 index 000000000..03267af91 --- /dev/null +++ b/packages/flow/tests/test_value_stored_form.py @@ -0,0 +1,87 @@ +"""A contract value can say how the system of record stores what was entered. + +OpenEMR's New Patient form saves street and city in capitals. With +``stored: upper`` the check expects exactly the capitals, so a correct save +isn't refuted and a wrong value still is. Records are made up. +""" + +from __future__ import annotations + +import pytest + +from openadapt_flow.runtime.effects import Effect, EffectKind +from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects.effect import EffectState, ValueExpr + + +def test_the_stored_form_is_applied_after_the_map() -> None: + upper = ValueExpr(param="street", stored="upper") + assert upper.resolve({"street": "41 Larch Way"}) == "41 LARCH WAY" + coded = ValueExpr(param="state", map={"Massachusetts": "ma"}, stored="upper") + assert coded.resolve({"state": "Massachusetts"}) == "MA" + assert coded.resolve({"state": "Maine"}) is None # fails closed + + +def test_a_literal_has_no_stored_form() -> None: + with pytest.raises(ValueError): + ValueExpr(literal="x", stored="upper") + + +def test_an_unset_stored_form_keeps_the_contract_digest() -> None: + assert ValueExpr(param="street").model_dump() == { + "literal": None, + "param": "street", + } + + +def test_field_equals_expects_the_stored_capitals() -> None: + effect = Effect( + kind=EffectKind.FIELD_EQUALS, + match={"id": "1"}, + field="street", + value={"param": "street", "stored": "upper"}, + ).resolve({"street": "41 Larch Way"}) + before = EffectState(substrate="t", reachable=True, records=[]) + + def verdict(stored: str) -> str: + return judge_records( + effect, before, [{"id": "1", "street": stored}], substrate="t" + ).verdict.value + + assert verdict("41 LARCH WAY") == "confirmed" + assert verdict("41 Larch Way") != "confirmed" # exact: the stored form only + assert verdict("14 LARCH WAY") != "confirmed" + + +def test_the_stored_form_uses_full_unicode_case() -> None: + # OpenEMR's "U" fields upper-case in the browser (toUpperCase): measured + # on the pinned fixture, "12 Calle Peña" is stored "12 CALLE PEÑA" and + # "Straße" is stored "STRASSE". + upper = ValueExpr(param="street", stored="upper") + assert upper.resolve({"street": "12 Calle Peña"}) == "12 CALLE PEÑA" + assert upper.resolve({"street": "Straße"}) == "STRASSE" + lower = ValueExpr(param="email", stored="lower") + assert lower.resolve({"email": "ÉLodie@Example.TEST"}) == "élodie@example.test" + + +def test_a_binding_comes_from_one_effect() -> None: + from openadapt_flow.provenance import bindings, bound_maps + + city = {"param": "city", "stored": "upper"} + one = [ + Effect( + kind=EffectKind.FIELD_EQUALS, match={"id": "1"}, field="city", value=city + ) + ] + assert bindings(one)["city"][0] == "city" + assert bound_maps(one)["city"].stored == "upper" + # The same parameter written to two fields two ways: not compared. + two = one + [ + Effect( + kind=EffectKind.FIELD_EQUALS, + match={"id": "1"}, + field="town", + value={"param": "city"}, + ) + ] + assert "city" not in bindings(two)