benchmark: Phase 0 held-out and independent fault suites against the OpenEMR record check

packages/flow/benchmark/lifecycle_proof/readonly_db.py

@@ -0,0 +1,138 @@
+"""A minimal DB-API connection to a fixture's MariaDB through ``docker exec``.
+
+The OpenEMR fixture publishes no database port. This lets a ``kind: sql``
+verifier (``sql_driver: benchmark.lifecycle_proof.readonly_db``) read it as a
+SELECT-only user, the way a deployment would read through its own driver.
+
+One connection is one ``mariadb`` client session, kept open for the
+connection's life, so a transaction spans statements (the whole-database
+snapshot reads everything inside one ``START TRANSACTION WITH CONSISTENT
+SNAPSHOT``). After each statement a marker query ends its output. The client
+runs with ``--force``, so a failed statement (an optional ``SET``) raises in
+Python without ending the session.
+
+Benchmark plumbing only: no query parameters, rows as text (``NULL`` is None).
+"""
+
+from __future__ import annotations
+
+import os
+import queue
+import subprocess
+import threading
+import time
+from typing import Any, Optional
+
+_MARK = "__oa_end__"
+
+
+class Error(Exception):
+    pass
+
+
+class _Cursor:
+    def __init__(self, conn: "Connection") -> None:
+        self._conn = conn
+        self.description: Optional[list[tuple[str, ...]]] = None
+        self._rows: list[tuple[Any, ...]] = []
+
+    def execute(self, sql: str, params: Any = None) -> None:
+        if params:
+            raise ValueError("exec_mariadb takes no query parameters")
+        lines = self._conn._run(sql)
+        if not lines:
+            self.description, self._rows = [], []
+            return
+        header = lines[0].split("\t")
+        self.description = [(name,) for name in header]
+        self._rows = [
+            tuple(None if cell == "NULL" else cell for cell in line.split("\t"))
+            for line in lines[1:]
+        ]
+
+    def fetchall(self) -> list[tuple[Any, ...]]:
+        return list(self._rows)
+
+
+class Connection:
+    def __init__(self, container: str, user: str, password: str, database: str):
+        self._proc = subprocess.Popen(
+            [
+                "docker",
+                "exec",
+                "-i",
+                "-e",
+                "MYSQL_PWD",
+                container,
+                "mariadb",
+                "-u",
+                user,
+                "--batch",
+                "--raw",
+                "--unbuffered",
+                "--force",
+                database,
+            ],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+            env={**os.environ, "MYSQL_PWD": password},
+        )
+        self._errors: "queue.Queue[str]" = queue.Queue()
+        threading.Thread(target=self._drain, daemon=True).start()
+
+    def _drain(self) -> None:
+        assert self._proc.stderr is not None
+        for line in self._proc.stderr:
+            self._errors.put(line.strip())
+
+    def _run(self, sql: str, timeout_s: float = 300.0) -> list[str]:
+        assert self._proc.stdin is not None and self._proc.stdout is not None
+        statement = sql.strip().rstrip(";")
+        self._proc.stdin.write(f"{statement};\nSELECT '{_MARK}' AS {_MARK};\n")
+        self._proc.stdin.flush()
+        out: list[str] = []
+        done = threading.Event()
+
+        def read() -> None:
+            assert self._proc.stdout is not None
+            for line in self._proc.stdout:
+                line = line.rstrip("\n")
+                if line == _MARK and out and out[-1] == _MARK:
+                    out.pop()  # the marker's header line
+                    break
+                out.append(line)
+            done.set()
+
+        threading.Thread(target=read, daemon=True).start()
+        if not done.wait(timeout_s):
+            self.close()
+            raise Error(f"no answer in {timeout_s:.0f} s")
+        time.sleep(0.02)  # stderr is its own pipe: let a statement's error land
+        errors = []
+        while not self._errors.empty():
+            errors.append(self._errors.get())
+        if self._proc.poll() is not None:
+            raise Error("the mariadb session ended: " + " ".join(errors))
+        if errors:
+            raise Error(" ".join(errors))
+        return out
+
+    def cursor(self) -> _Cursor:
+        return _Cursor(self)
+
+    def close(self) -> None:
+        try:
+            if self._proc.stdin is not None and not self._proc.stdin.closed:
+                self._proc.stdin.close()
+            self._proc.wait(timeout=10)
+        except Exception:  # noqa: BLE001 - closing best effort
+            self._proc.kill()
+
+
+def connect(
+    container: str, user: str, password: str = "", database: str = "openemr"
+) -> Connection:
+    return Connection(container, user, password, database)

packages/flow/benchmark/lifecycle_proof/verify.database.yaml

@@ -0,0 +1,118 @@
+# CANDIDATE (not admitted): the OpenEMR referral contract with a whole-database
+# frame. verify.yaml is the admitted one; verify.frame.yaml is the earlier
+# candidate, whose frame covered only the tables it read.
+#
+# The frame diffs every table and column of the OpenEMR database before and
+# after the save (frame.database). Any row added, removed or changed is
+# collateral unless the record contract, the chart's history row
+# (side_records) or a concurrent writer named by OpenEMR's audit trail
+# explains it. contact (a link row per chart) is append-only bookkeeping
+# in the OpenEMR profile. A new chart or history row that sets a column the contract
+# doesn't name, outside new_row_fields, is collateral too.
+#
+# allow_tables is the reviewed list of tables OpenEMR writes on its own for any
+# signed-in user: SPEC.md F3's bookkeeping list, and the tables the witness run
+# found the New Patient path writing (WITNESS.md).
+#
+# Needs #1570, #1571, #1572, #1576 and the whole-database frame. Made-up
+# patients only.
+schema_version: openadapt.verify/v1
+environment_id: openemr-local-fixture-7c96c8eefe46
+minimum_effect_strength: independent_system_of_record
+
+verifier:
+  kind: surfaces
+  surfaces:
+    patient:
+      kind: rest
+      base_url: ${OPENEMR_API_URL:-https://127.0.0.1:9300}
+      records_path: /apis/default/api/patient?_limit=1000
+      records_key: data
+      auth:
+        bearer_env: OPENEMR_ORACLE_TOKEN
+
+record:
+  name: patient
+  surface: patient
+  match: [email, lname]
+  duplicate_key: email
+  exact_new_set: true
+  identity_field: uuid
+  fields:
+    email: {param: email, kind: email}
+    lname: lname
+    title: title
+    fname: fname
+    DOB: {param: DOB, kind: date, date_order: mdy}
+    sex: sex
+    # OpenEMR's New Patient form saves these in capitals (WITNESS.md, #1524).
+    street: {param: street, stored: upper}
+    city: {param: city, stored: upper}
+    state: {param: state_label, map: {Massachusetts: MA}}
+    postal_code: postal_code
+    phone_home: {param: phone_home, kind: phone}
+    country_code: country_code
+    # The fixture's two providers (users.id as seeded by the suite).
+    providerID: {param: provider, map: {"Dr. Iris Holt": "6", "Dr. Owen Hale": "7"}}
+
+frame:
+  identity_field: id
+  side_records:
+    - {surface: history_data, link_field: pid, of_surface: patient, of_field: pid}
+  attribution:
+    kind: audit_read
+    actor_field: user
+    record_field: patient_id
+    record_key: pid
+    # Only a write to the same table attributes a change; a chart viewed is
+    # never a chart written.
+    event_field: event
+    write_events: [patient-record-insert, patient-record-update,
+                   patient-record-delete, api-POST, api-PUT, api-PATCH, api-DELETE]
+    insert_events: [patient-record-insert, api-POST]
+    delete_events: [patient-record-delete, api-DELETE]
+    table_field: tbl
+    self: [openadapt_actor]
+    read:
+      kind: sql
+      sql_driver: benchmark.lifecycle_proof.readonly_db
+      sql_connect_args: &db
+        container: ${OPENEMR_DB_CONTAINER}
+        user: ${OPENEMR_READONLY_DB_USER:-oa_audit_reader}
+        database: openemr
+      sql_password_env: OPENEMR_READONLY_DB_PASSWORD
+      # A UI write's table is decoded from the statement OpenEMR logs in
+      # `comments`; an API write's from its endpoint.
+      sql_query: >-
+        SELECT CONCAT('log-', id) AS id, user, patient_id, event,
+        LOWER(REGEXP_REPLACE(FROM_BASE64(comments),
+        '(?is)^\\s*(?:insert\\s+into|update|delete\\s+from|replace\\s+into)\\s+`?([a-z_0-9]+).*$',
+        '\\1')) AS tbl FROM log
+        WHERE event LIKE 'patient-record-%' AND event <> 'patient-record-select'
+        AND patient_id > 0 AND user <> ''
+        UNION ALL
+        SELECT CONCAT('api-', a.id) AS id, u.username AS user, a.patient_id,
+        CONCAT('api-', a.method) AS event,
+        CASE WHEN a.request_url LIKE '%/allergy%' THEN 'lists'
+        WHEN a.request_url LIKE '%/message%' THEN 'pnotes'
+        WHEN a.request_url LIKE '%/insurance%' THEN 'insurance_data'
+        WHEN a.request_url LIKE '%/patient%' THEN 'patient_data' ELSE '' END AS tbl
+        FROM api_log a JOIN users u ON u.id = a.user_id
+        WHERE a.method IN ('POST', 'PUT', 'PATCH', 'DELETE') AND a.patient_id > 0
+  database:
+    dialect: mysql
+    read:
+      sql_driver: benchmark.lifecycle_proof.readonly_db
+      sql_connect_args: *db
+      sql_password_env: OPENEMR_READONLY_DB_PASSWORD
+    # OpenEMR's own bookkeeping, shared with rollback and the RL grader
+    # (openadapt_flow/app_profiles/openemr.yaml).
+    profile: openemr
+    profile_sha256: "sha256:a3daefcf7b7f96e0c68a7d5c1829c3e358dc1ce9366e3098d18bc80441685bed"  # openemr-frame-profile/2026-10-29.2
+    views: {patient_data: patient}
+    new_row_fields:
+      # What OpenEMR's New Patient form fills on a new chart beyond the
+      # referral's fields (live witness, WITNESS_LIVE.md): ids, dates, and
+      # the form's empty strings where the column's default is NULL.
+      patient_data: [billing_note, birth_fname, birth_lname, birth_mname, care_team_status, created_by, date, dupscore, financial_review, gender_identity, guardianaddress, guardiancity, guardiancountry, guardianemail, guardianphone, guardianpostalcode, guardianrelationship, guardiansex, guardiansname, guardianstate, guardianworkphone, hipaa_allowemail, hipaa_allowsms, imm_reg_status, industry, interpreter_needed, nationality_country, occupation, patient_groups, pid, preferred_name, pronoun, protect_indicator, publicity_code, pubpid, ref_providerID, regdate, sex_identified, sexual_orientation, street_line_2, suffix, tribal_affiliations, updated_by, uuid]
+      history_data: [created_by, date, uuid]

packages/flow/benchmark/lifecycle_proof/verify.frame.yaml

@@ -0,0 +1,92 @@
+# CANDIDATE (not admitted): the OpenEMR referral contract with the fixes the
+# Phase 0 independent fault suite asked for. verify.yaml is the admitted one.
+#
+# Beyond verify.yaml it:
+#   - reads history_data, pnotes, lists and insurance_data (read-only SQL)
+#     beside the patient, so the frame sees writes there (record.surface);
+#   - requires one history_data row per new chart (frame.side_records);
+#   - checks the primary care provider, mapped from the name the referral
+#     prints to the users.id the chart stores;
+#   - attributes changes through OpenEMR's audit trail: UI writes in `log` by
+#     user name, API writes in `api_log` by user id, keyed by pid on every
+#     table (frame.attribution.record_key).
+# The source check (provenance against the referral) is a run input
+# (verify-run --provenance/--source), not part of this file.
+#
+# Needs #1570, #1571, #1572 and #1576. Made-up patients only.
+schema_version: openadapt.verify/v1
+environment_id: openemr-local-fixture-7c96c8eefe46
+minimum_effect_strength: independent_system_of_record
+
+verifier:
+  kind: surfaces
+  surfaces:
+    patient:
+      kind: rest
+      base_url: ${OPENEMR_API_URL:-https://127.0.0.1:9300}
+      records_path: /apis/default/api/patient?_limit=1000
+      records_key: data
+      auth:
+        bearer_env: OPENEMR_ORACLE_TOKEN
+    history: &readonly
+      kind: sql
+      sql_driver: benchmark.lifecycle_proof.readonly_db
+      sql_connect_args:
+        container: ${OPENEMR_DB_CONTAINER}
+        user: ${OPENEMR_READONLY_DB_USER:-oa_audit_reader}
+        database: openemr
+      sql_password_env: OPENEMR_READONLY_DB_PASSWORD
+      sql_query: SELECT * FROM history_data
+    notes:
+      <<: *readonly
+      sql_query: SELECT * FROM pnotes
+    allergies:
+      <<: *readonly
+      sql_query: SELECT * FROM lists
+    insurance:
+      <<: *readonly
+      sql_query: SELECT * FROM insurance_data
+
+record:
+  name: patient
+  surface: patient
+  match: [email, lname]
+  duplicate_key: email
+  exact_new_set: true
+  identity_field: uuid
+  fields:
+    email: email
+    lname: lname
+    title: title
+    fname: fname
+    DOB: DOB
+    sex: sex
+    street: street
+    city: city
+    state: {param: state_label, map: {Massachusetts: MA}}
+    postal_code: postal_code
+    phone_home: phone_home
+    country_code: country_code
+    # The fixture's two providers (users.id as seeded by the suite).
+    providerID: {param: provider, map: {"Dr. Iris Holt": "6", "Dr. Owen Hale": "7"}}
+
+frame:
+  identity_field: id
+  side_records:
+    - {surface: history, link_field: pid, of_surface: patient, of_field: pid}
+  attribution:
+    kind: audit_read
+    actor_field: user
+    record_field: patient_id
+    record_key: pid
+    self: [openadapt_actor]
+    read:
+      <<: *readonly
+      sql_query: >-
+        SELECT CONCAT('log-', id) AS id, user, patient_id FROM log
+        WHERE event LIKE 'patient-record-%' AND event <> 'patient-record-select'
+        AND patient_id > 0 AND user <> ''
+        UNION ALL
+        SELECT CONCAT('api-', a.id) AS id, u.username AS user, a.patient_id
+        FROM api_log a JOIN users u ON u.id = a.user_id
+        WHERE a.method IN ('POST', 'PUT', 'PATCH', 'DELETE') AND a.patient_id > 0