From e8465f2884d75df54039eec0e149b0dd4eb1809f Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 12:35:02 -0500 Subject: [PATCH 01/12] Zero-touch setup: literature review, design, sandbox validator prototype (WIP) --- .../flow/docs/research/ZERO_TOUCH_DESIGN.md | 184 ++++ .../docs/research/ZERO_TOUCH_ONBOARDING.md | 443 +++++++++ .../generate/verify_contract.py | 61 ++ .../openadapt_flow/generate/zero_touch.py | 914 ++++++++++++++++++ packages/flow/tests/test_zero_touch.py | 160 +++ 5 files changed, 1762 insertions(+) create mode 100644 packages/flow/docs/research/ZERO_TOUCH_DESIGN.md create mode 100644 packages/flow/docs/research/ZERO_TOUCH_ONBOARDING.md create mode 100644 packages/flow/openadapt_flow/generate/zero_touch.py create mode 100644 packages/flow/tests/test_zero_touch.py diff --git a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md new file mode 100644 index 0000000000..6ff4682612 --- /dev/null +++ b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md @@ -0,0 +1,184 @@ +# Zero-touch setup: design + +Written 2026-11-03. Goal: setting up the write verifier for a new app needs +**no person decision** and finishes in minutes, and the verifier is no +easier to fool than with a person's setup. The evidence behind each choice +is in [ZERO_TOUCH_ONBOARDING.md](ZERO_TOUCH_ONBOARDING.md); bracketed tags +like [C5] point there. + +## The rule + +> An agent proposes every piece of setup as data. A machine admits a piece +> only after it confirms held-out correct writes and stops planted faults, +> in a disposable copy of the app's store. Anything it can't admit is not +> excused: those tasks stop, and the piece is listed for a person to look +> at. That list is the measure, and its target is zero. + +A person is never asked a question during setup. They may read the list. + +## Where person decisions came from + +The retests' setup logs list each decision. Grouped by type, with what +replaces each: + +| Decision type | Today | Zero-touch replacement | Checked by | +| --- | --- | --- | --- | +| Read-only account, a `TRIGGER` grant, a view | A person runs SQL | The deployer gives one admin connection once (an input, not a judgment); the setup applies grants and views itself, as the generator's `--grant-trigger` and `--create-views` already can | The generator's read-only check | +| Bookkeeping: which changed columns are the app's own | A person answers the generator's question | The generator's own proposal, unchanged, plus `now_checks` (an excused time stamp must be near the read). Nothing is added: no `logs_add`, no `columns_add` | Bookkeeping faults planted in every changed cell; an unchecked cell fails its kind | +| Recheck window | A person picks seconds | A policy constant (300 s) | Late-write faults | +| Phone-number comparison | A person confirms | Every listed parameter | Wrong-digit faults | +| Where a field is stored, when two places fit | A person picks | Try each candidate; keep the one whose kind validates, only if exactly one does | Held-out writes and wrong-value faults | +| Which demonstrations to make | A person chooses values | The onboarding agent makes them, with values chosen to differ (different dates, list lengths 0, 1 and 2, optional fields on and off), plus held-out writes it never shows the generator | Held-out writes | +| How a task's value is stored (`high` as 3, a flag as text) | A person writes a mapping | The agent writes the mapping as exact code | Every demonstration and held-out write must place every parameter through it | +| Hand edits to contracts | A person patches YAML | None. A kind that needs one isn't admitted | Listed in the exception report | +| Shadow routes (which request feeds which contract) | A person maps them | The agent proposes them from recorded traffic | A route that doesn't place every parameter of a held-out write is refused | +| Learned excuses | A person approves the proposal | Admitted per excuse by planted faults at its edge: the same column on another user's row, another column on the actor's row, one more row than `max_rows` | Those faults must stop | +| Identity region on screen | A person picks | Candidates whose label is fixed and value varies across recordings | The region must read the right record on every recording and the wrong one on a different record (a negative test) | + +## Pipeline + +``` +schema + demonstrations ──► generator propose ──► questions + ▲ │ + │ auto-answer by rule + │ │ + propose again with answers ◄─────┘ + │ + ▼ + sandbox copy ◄── held-out writes + planted faults + │ + admitted kinds ─┴─ exception report + │ + production contracts (same answers) +``` + +1. **Sandbox.** A copy of the app's tables beside the app's own, made with + the admin connection: a database `_oa_sandbox` (MySQL, MariaDB) or a + schema `oa_sandbox` (PostgreSQL). The read-only account is granted read + on it. The setup refuses to write or drop anything not named like a + sandbox. Faults are never planted in the app's own tables. +2. **Propose.** The existing generator, pointed at the sandbox, with the + training demonstrations. No answers. +3. **Answer by rule.** The table above. Each answer goes in a ledger with + the rule that produced it. +4. **Propose again** with the answers. +5. **Validate.** For each held-out write: put its rows back as they were + before it, choose the contract, `begin_write`, put the write's rows (its + own time stamps moved to now), `verify_write`, then the recheck. It must + be `CONFIRMED`. Then, from the same write, plant faults (below); each + must end not `CONFIRMED`. Every touched row is put back after each case. +6. **Admit** a kind when every held-out write of it is confirmed and every + fault stopped. Write the production contracts with the same answers + against the app's own read account, and drop every kind not admitted + from the manifest, so `choose` finds no contract and its tasks stop. +7. **Report** `ONBOARDING.json` (minutes, answers, ledger, per-kind + results) and `EXCEPTIONS.md`. + +## The faults + +Built only from the held-out write's recorded diff, the schema and the +task's values. The planter never reads a contract. + +| Class | What is planted | +| --- | --- | +| `lost_write` | Nothing is written | +| `wrong_value` | A cell holding a task value gets another value of its type (a key gets another existing row's key) | +| `bookkeeping` | A cell the write changed that holds no task value gets a far value (a time stamp in 2001, a number plus 7, text plus " (x)") | +| `wrong_record` | An update's changes land on another row of the same table | +| `look_alike` | A create's values land on an existing row instead | +| `duplicate` | Every added row is added twice | +| `collateral_target` | A column of the target the write didn't change is changed | +| `collateral_row` | Another row of a table the write touched is changed | +| `extra_row` | A row is added to a table the write didn't touch | +| `late_write` | The write is correct; a collateral change lands after the check and before the recheck | + +The `bookkeeping` class is what keeps excuses honest. Every cell the write +changes is either a task value (checked), a value a rule checks (the acting +user, a constant, a copy, the current time), or an excuse. A fault planted +in an excused cell with no value rule passes, so its kind is not admitted. +That turns "is this column bookkeeping?" from a judgment into a test. + +## Keeping validation independent + +The fault planter and the contract generator must not share blind spots +[B19, B20]. Four things keep them apart: + +1. **Different method.** The generator infers places and roles from + demonstrations. The planter mutates every cell and row the held-out + write touched, plus rows it didn't, by fixed operators. No model writes + either side. +2. **Different input.** The planter reads the held-out diff, never the + contract or the generator's report. +3. **Held-out writes.** The generator never sees the writes it is + validated on [A8]. +4. **Hidden faults written by someone else.** For measurement, the + pre-registered packs' faults (written by separate pack authors, frozen + before setup) are run against the admitted contracts. They answer pass + or fail, and are never shown to the setup [C18, C19]. + +An LLM fault generator can be added on top later, from a different model +family and a separate prompt, but only above this floor [C12]. + +## What stays general + +No app-specific code or tests. Everything per app is data: + +- the kinds file, demonstrations and held-out writes (made by the agent + through the app's own API or screens); +- value mappings (exact code the agent writes, checked mechanically); +- the generated contracts and the answers ledger. + +The code paths are the same for every app: the generator, the sandbox (by +SQL dialect only), the fault operators, the verifier. Tests use a made-up +schema on a real PostgreSQL. + +## What this does not do yet + +- **Counters and derived columns.** A column the app recomputes (a + duration from two times, an amount from a rate) has no value rule yet. Its + bookkeeping fault passes, so its kind is not admitted. A next step is + enumerating exact expressions over the row's columns (Daikon-style + templates [A1]) and admitting one only if it holds on every demonstration + and held-out write and catches its fault. +- **Log tables.** A table whose added rows vary in number (an audit log) is + never named a log automatically. Those kinds aren't admitted until a rule + pins such rows to the write (its record or the acting user) and a + planted extra row is still caught. +- **Excuses on rows other than the main record** (a parent's counter) have + no value rule and fail their bookkeeping fault. +- **Learned excuses** (`actor_row`) are still approved by a person in the + prototype. The replacement is in the table above. +- **The identity region** is designed here, not built. + +Each gap shows up as an exception, never as a pass. + +## Measuring it + +On development apps only (the first study's and the first retest's), never +on a pre-registered held-out app or its pack. Per app: + +- setup minutes, wall clock, from first command to admitted contracts; +- person decisions (target 0) and exception items; +- kinds admitted; +- planted-fault coverage (stopped / planted, per class); +- the pack's own faults and correct writes run against the admitted + contracts: false stops per 100 correct writes, faults passed as verified, + faults caught. + +A kind that isn't admitted stops all of its tasks. That is safe but costs +false stops, so coverage (kinds admitted) is reported next to the decision +count. Zero decisions with nothing admitted is not success. + +## Safety review + +Changes to generator logic (`now_checks`) need a recorded independent +adversarial review before merge. Points for the reviewer: + +- `now_checks` only adds a check; it never removes one. A time stamp + already pinned or unchecked by a person's answer stays as it was unless + the answer sets `now_checks: true`. +- The window is the execution window plus the recheck plus a minute, in + either the database's local clock or UTC. A wrong time inside the window + passes; a NULL fails. +- The sandbox writes only to a store named `*_oa_sandbox` or `oa_sandbox`. +- A kind not admitted has no contract in the production manifest. diff --git a/packages/flow/docs/research/ZERO_TOUCH_ONBOARDING.md b/packages/flow/docs/research/ZERO_TOUCH_ONBOARDING.md new file mode 100644 index 0000000000..1bb3b6d2b0 --- /dev/null +++ b/packages/flow/docs/research/ZERO_TOUCH_ONBOARDING.md @@ -0,0 +1,443 @@ +# Setting up the write verifier with no person decisions: what the literature says + +Written 2026-11-03. This review asks one question: can an agent produce all +of an app's verifier setup, with each piece checked by a machine instead of +by a person, without making the verifier easier to fool? The design that +follows from it is in [ZERO_TOUCH_DESIGN.md](ZERO_TOUCH_DESIGN.md). + +Every source below was opened at a primary copy (the paper, its arXiv page, +the publisher's page, or the vendor's own documentation). Where only an +abstract was read, the entry says so. Sources that could not be opened are +listed at the end and are not used as evidence. + +## The problem + +Setting up the verifier for an app needs four kinds of per-app material: + +- **Contracts:** what a correct write of each kind looks like. Which rows + and columns change, to what, and what must not change. +- **Read access:** which stores the verifier reads, with which account. +- **Excuses:** writes the app makes on its own (a time stamp, a counter, a + last-login row) that may change without failing the check. +- **Identity:** which row is "the record", and on screen, which region + shows which record is open. + +The contract generator (`docs/GENERATE_CONTRACTS.md`) already proposes +contracts from the schema and a few demonstration writes. It asks a person +what it can't decide. In two pre-registered retests on apps it had never +seen, that left too much to a person. Across the first retest's four apps, +setup needed 9 to 15 person decisions (the bar is 3). In the second, one app +took 64.5 minutes and 12 decisions (the bars are 60 minutes and 3), and +another reached the 120-minute limit with no usable setup. The decisions +were of a few recurring types: + +- which changed columns are the app's own bookkeeping; +- how long the app may keep writing after a write; +- how a task's value is stored (a word stored as a number, a flag as text); +- where a field is stored when two places fit; +- which demonstrations to make; +- hand edits to generated contracts; +- approving the learned excuses. + +Two facts frame the review. First, an excuse is exactly where a false +`CONFIRMED` comes from: in the first retest, the one fault that passed as +verified passed through a bookkeeping answer that excused a column on every +row of a table. Second, a person approving generated material is a weak +check. Polikarpova et al. found that about a third of the clauses an +invariant miner produced were wrong or irrelevant [A6]. So the target is +not "a person approves faster". It is "nothing is approved by a person, and +what can't be checked stays a stop". + +## 1. Mining specifications from traces + +**Likely invariants (Daikon).** Ernst et al. check a fixed grammar of +candidate properties (constants, ranges, linear relations, orderings) +against every recorded sample, and keep a property only if no sample +breaks it [A1]. A surviving property must also pass a chance test: it is +reported only if the probability that it held by accident is below a limit +the authors set at 0.01. Their companion paper adds two filters that +transfer directly [A2]. Compare only variables of compatible types, and +don't count a value that wasn't reassigned as fresh evidence. The tool +paper shows the trace format is neutral, so database row snapshots can be +fed in as samples [A3]. + +What it says about our data: with three to five demonstrations, almost no +property passes a chance test. A Daikon-style learner over a handful of +write diffs reports bounds and constants that hold by coincidence [A1]. +Nimmer and Ernst say plainly that dynamic inference is unsound because it +depends on the samples, and pair it with a static checker to remove +properties that aren't universally true [A5]. Hellendoorn et al. label each +mined invariant by whether it survives held-out tests [A8]. That is a cheap, +mechanical form of the same idea: mine from some demonstrations, check on +others. + +**Online learning and approximate rules.** DIDUCE starts with the +strictest hypothesis and relaxes it on each violation [A9]. Perracotta +keeps temporal rules that hold in a fraction of subtraces above a +threshold, and its property count is very sensitive to that threshold +[A10]. Texada reports support and confidence, so it can mine rules that +almost always hold [A11]. These methods are useful for proposing candidate +background writes. They must never accept an excuse: a rule with +confidence below 1, or one that widens on each violation, tolerates the +rare deviation the verifier exists to catch. + +**Checking mined specifications by mutation.** SpecFuzzer generates +candidate assertions, drops those the tests falsify, then ranks the +survivors by how many code mutants they kill [A13]. Schuler, Dallmeier and +Zeller use mined invariants the other way round, as a signal of which +mutants matter [A14]. Both measure a specification by the faults it +catches. That is the gate this design needs. + +**Invariants over database state.** Yang et al. found that Rails apps keep +most data constraints in app code, not in the database [A16]. The Coco +study extracted about 289 constraints per app from code and found that +most extra dependencies reported by data profilers were false positives +[A17]. The profilers treat time columns as structure, and their sampling +reports random columns as unique. A schema alone underspecifies a correct +write. + +## 2. LLM-generated specifications and test oracles + +**Postconditions from intent.** nl2postcond turns a natural-language +description into executable postconditions and scores them twice [B1]: +correctness (the reference code passes) and completeness (how many buggy +variants the postcondition fails). The best GPT-4 setting was correct for +158 of 164 problems, with mean bug-completeness 0.85. The trade-off is the +lesson. `true` is trivially correct. Type-check postconditions were the +most common kind (47.4%) but caught only 14% of natural bugs, against 82% +for arithmetic equalities. A simpler prompt raised correctness and cut +completeness by up to about 30%. Lahiri scores Dafny specifications the +same two ways, with the verifier perturbing outputs instead of needing +mutant programs [B2]. + +**Generate, check, repair.** SpecGen [B3] and AutoSpec [B4] loop an LLM +with a verifier until the specification verifies. Lemur formalizes the +loop so that a wrong LLM proposal can't produce a wrong verdict [B5]. +Clover accepts generated code only when code, docstring and annotations +agree, and reports no false positives on adversarial instances [B6]. The +caution: "it verifies" rewards weakness. A repair loop driven only by +"this correct write was refuted" will loosen a contract until everything +passes, unless a drop in strength rejects the repair. + +**The oracle problem.** Barr et al. sort oracles into specified, derived, +implicit and human [B8]. Contracts generated from demonstrations are +derived oracles: they can be no more correct than the demonstrations. +Konstantinou, Degiovanni and Papadakis found that LLM oracles mostly encode +what the code does, not what it should do [B12]. Hossain et al. re-ran a +published neural oracle generator at scale: more than 47% of its emitted +assertions were false positives, and the true ones added 0.3% fault +detection [B10]. Shin et al. found no correlation between how much a +generated oracle resembles a reference and how many faults it finds [B11]. +So a generated contract is judged by behaviour (what it confirms and what +it refutes), never by how plausible it looks. + +**Metamorphic relations.** Metamorphic testing checks relations between +several runs instead of expected values [B14, B15]. Rigger and Su found 175 +database bugs by checking that a query's three-valued partitions add up to +the whole [B16]. Relations useful here: a write read back through a second +path agrees; a repeated "make sure it exists" write adds nothing; untouched +rows stay identical. Only 34% to 50% of LLM-proposed relations were valid +in a 37-system study (abstract only) [B17]. A relation may add a stop, +never a pass, and is enabled only if it holds on all clean traffic. + +## 3. Validating generated oracles by planted faults + +**Two-sided oracle assessment.** Jahangirova et al. assess an assertion +oracle on both sides [C5]. Generated correct runs expose false positives, +and mutants expose false negatives. A person refines the assertion and the +loop repeats. Oracles improved this way caught 48.6% more faults. The plan +here (replay correct writes, plant faults) is that method applied to +database writes, with the person removed. An excuse is an oracle weakening. +It is justified only by a false positive it removes, and it is exposed only +when a planted fault lands inside it. + +**What a full kill rate does and does not mean.** Just et al. found mutant +detection correlated with real-fault detection on 357 real faults, but 27% +of real faults were not coupled to any mutant the triggering tests caught, +and 17% matched no mutation operator [C3]. Once suite size is controlled, +the correlation between mutation score and real-fault detection is weak +(abstract only) [C4]. The Papadakis survey shows that redundant and +subsumed mutants inflate a single score, and that equivalent-mutant +detection is undecidable in general [C2]. So catching every planted fault is +necessary for admission, not proof of correctness. Report results per fault +class and next to the false-stop rate. + +**Faults for data systems.** SQLMutation's operators cover wrong columns +and constants, joins and NULL handling, and only 5.8% of its mutants were +equivalent [C9]. Elle checks histories for duplicate writes, lost updates +and real-time-order violations, and found a default retry in one database +that silently re-applied writes [C10]. Elle makes every write traceable by +using unique values, which makes "which write made this row" decidable. + +**LLM-made faults.** LLM mutants caught more real bugs than rule-based ones +(76.5% against 44.2% in the latest version of Wang et al.), with more +duplicates, more mutants that don't build, and more equivalent ones [C12]. +Meta's mutation-guided test generator found 25% of its LLM mutants +trivially equivalent, and its LLM equivalence judge had recall 0.47 before +preprocessing [C13]. Naively prompted bug injectors collapse into one-line +edits, and self-play pipelines add a check that every part of a fault is +needed [C14]. CodeT [C15] and CURE [C16] show that agreement between +artefacts from one model is consensus, not ground truth; CURE works because +fixed ground-truth tests anchor it. + +**Independence.** Knight and Leveson ran 27 independently written versions +of one program on a million inputs and rejected the independence of their +failures at 99% confidence (read from a course copy) [B19]. Kim et al. +found that when two LLMs are both wrong they give the same wrong answer 60% +of the time on one leaderboard, and that stronger models are more +correlated across providers [B20]. LLM judges favour their own outputs +[B21]. So a different model for the fault generator is not enough. +Independence has to come from a different method: a rule-based fault floor +that no model writes, faults derived from the recorded write rather than +from the contract, and faults written by someone else. + +**Holding faults out.** A generator that iterates against a fault suite will +fit it. Thresholdout answers adaptive queries against a holdout with +provable validity, spending an overfitting budget when it reveals a value +[C18]. The Ladder reports a new score only on a clear improvement [C19]. +Recht et al. rebuilt two test sets and saw accuracy drop 3% to 15% [C20]. +For this design: keep a hidden fault set that answers only pass or fail, +and regenerate the visible suite from time to time. + +## 4. Change capture and schema understanding + +**Change data capture.** Debezium streams each row change with its before +and after images, its transaction id and its order in the transaction [A18]. +A table's before image holds only the key unless its replica identity is +full. A trigger-based audit table records the session user, the +application name and the changed fields [A20]. Transaction ids separate the +agent's commit from a background job in another transaction, which is the +strongest mechanical signal for "this row is bookkeeping". It fails when +the app writes its bookkeeping in the same transaction. + +**Bookkeeping columns.** No reviewed work infers bookkeeping columns +directly. The recipe that follows from the sources: a column is a candidate +when it changes outside the agent's transaction, or on writes of every +kind whatever the task's values; it is excused only with a value rule (the +current time, a step of one, the acting user), and only after planted +faults in it are still caught. + +**Keys and foreign keys.** Rostin et al. classify inclusion dependencies as +foreign keys with features such as name similarity and out-of-range share +(F-measure above 0.8) [A21]. Their errors come from small-integer columns +overlapping the ids of small tables. Zhang et al. show auto-increment +ranges creating false inclusions and add a randomness test [A22]. Binder +[A23] and Faida [A24] find inclusion dependencies exactly or approximately, +and HyUCC finds unique column combinations [A25]. All of these propose +identity; none can accept it. A uniqueness that holds in today's rows is +not a key, and a coincidental join is how a contract binds the wrong record. + +**Column roles.** Sherlock [A27], Sato [A28], Doduo [A29] and zero-shot +ChatGPT annotation [A30] label columns with semantic types at weighted F1 +around 0.85 to 0.93 on web tables. None is trained on operational roles +(update time, version counter, acting user), and rare classes score much +worse (Sato's macro F1 is 0.735). A role label is a nomination. A column +named like a time stamp may hold a date that matters clinically. + +**Provenance.** Why-provenance and where-provenance [A31], generalized as +semirings [A32], give the vocabulary for evidence: a `CONFIRMED` names the +row and column that hold the value, and the write that put it there. + +## 5. GUI element and record identity + +**Locators.** Similo scores elements on 14 weighted attributes and failed +to locate 11% of 801 targets, against 27% for the best multi-locator +baseline [D1]. Its authors note that without a threshold it returns a +matching but wrong element when the target is missing. Contextual clues +locate an element by the visible labels near it [D5]. Across records, an +identifying field is the one whose label stays fixed while its value +changes. VISTA reads each element through the DOM and through pixels and +hands any disagreement to a person [D6]. Over RDP or Citrix only pixels +exist, so the second reading has to be a read-back from the system of +record. + +**Accessibility identity.** UIA's `AutomationId` should be the same in every +instance of an app, but it is optional and not guaranteed stable across +builds; `Name` should match the visible label and is not a unique key +[D9]. Many thick clients expose no `AutomationId`, so the descriptor must be +validated per app. + +**Grounding models.** OmniParser [D10], ScreenAI [D11] and SeeClick [D12] +find UI elements in screenshots. On professional high-resolution apps the +best model reached 18.9% [D14]. A model may propose a region; it can't be +the gate. + +**Self-healing tools.** Healenium heals to the best candidate above a score +cap (0.5 in its example config) [D16]. Testim improves a locator when its +score drops below 70% [D17]. UiPath's Healing Agent picks a selector by +vote and can be limited to recommendations [D18]. Every one of them heals to +keep a test running. For a write gate the cost is reversed: a wrong heal +that keeps a run going is worse than a stop. + +**Record identity in the store.** FHIR's `Patient/$match` with +`onlyCertainMatches` returns nothing when the match is ambiguous [D15]. That +is the right shape for an identity check: one certain match, or a stop. + +## 6. How commercial tools onboard an app + +| Tool | Generated automatically | A person still decides | How it validates | +| --- | --- | --- | --- | +| UiPath [D18, D21, D22, D23] | Element descriptors, anchors, fallback targets; a healed selector by vote | Which element to indicate; heal or only recommend; document fields in Validation Station | "Verify execution": an element appears, disappears or changes text | +| Power Automate desktop [D24, D25] | Selectors from a Ctrl+click | Which element; selector order | "Test selector": success, failure, or several matches (the top-left one wins at run time) | +| Blue Prism [D26] | Every attribute of the spied element | Which attributes count, how each matches, which are dynamic | Attribute match at run time | +| Automation Anywhere [D27] | Default search criteria | Removes volatile criteria | Separate validation properties after the element is found | +| Anthropic and OpenAI computer use [D28, D29] | Actions from screenshots | Confirms consequential actions | A screenshot after each step; check the outcome in the app | +| Airbyte [D30] | The discovered schema and its changes | Approves non-breaking changes; fixes breaking ones | A removed key or cursor pauses the connection | + +Every tool keeps a person on the choice of element or record identity, and +none checks a write against the system of record. "Verify execution" +checks that the screen reacted. + +## Which ideas transfer + +1. **Admit by planted faults, on both sides.** A generated contract is + admitted only if it confirms every held-out correct write and stops + every planted fault [C5, A13, B1]. Score correctness and strength + separately, per fault class, next to the false-stop rate. +2. **No excuse without a value rule, and plant faults inside every + excuse.** An unchecked cell is a weakening. Replace "any value" with a + rule the read can check (the current time, the acting user, a copy of + another column), and plant a fault in that cell. If no rule fits, the + kind isn't admitted [C5, C2]. +3. **Faults from the recorded write, not from the contract.** The fault + planter reads the held-out write's diff, the schema and the task's + values. It never reads the contract. Add LLM-written faults only on top + of a rule-based floor, and faults written by someone else as a hidden + holdout [B19, B20, C12, C18]. +4. **Hold demonstrations out.** Mine from some demonstrations; validate on + others the generator never saw [A8]. +5. **Exact rules only; approximate mining proposes.** Confidence-below-1 + rules, relax-on-violation learners and role labels can nominate an + excuse or an identity rule. Only the sandbox can accept it [A9, A10, + A27–A30]. +6. **No model in the gate.** A model may propose. A deterministic check + decides, and a model never judges its own output [B5, B21, B11]. +7. **Fail closed where it can't be checked.** What the sandbox can't + validate stays `UNCERTAIN` or stops, and is listed for a person to look + at, never excused [D1, D15]. + +## Risks + +- **An autogenerated excuse is where a false `CONFIRMED` comes from.** The + first retest's only fault that passed as verified went through one. The + defence is item 2 above: an excused cell has a value rule, and a fault + planted in it must be caught. +- **Shared blind spots.** If the fault planter and the contract generator + share an idea of which columns matter, a full kill rate means nothing. + The planter must not read contracts, and its operators must cover every + column the write touched and rows it didn't touch. +- **Admission is necessary, not sufficient.** 17% to 27% of real faults + aren't coupled to synthetic ones [C3]. Product copy must not present an + admitted contract as proof. +- **The sandbox isn't production.** Look-alike records, other users and + background jobs may be rarer in a sandbox copy than in a live store + [C10]. The live verifier keeps its whole-database frame and recheck. +- **Overfitting the gate.** A generator that sees which faults survived can + patch case by case. Keep a hidden fault set and regenerate the visible + one [C18, C19]. +- **Few demonstrations.** With three to five writes, coincidences pass as + rules [A1]. Held-out writes and planted faults catch some of them; + demonstrations chosen to differ catch more. + +## Sources + +Opened at a primary copy unless marked. + +**A. Mining and databases** + +- [A1] Ernst, Cockrell, Griswold, Notkin. "Dynamically discovering likely program invariants to support program evolution." IEEE TSE 27(2), 2001. https://homes.cs.washington.edu/~mernst/pubs/invariants-tse2001.pdf +- [A2] Ernst, Czeisler, Griswold, Notkin. "Quickly detecting relevant program invariants." ICSE 2000. https://homes.cs.washington.edu/~mernst/pubs/invariants-relevance-icse2000.pdf +- [A3] Ernst et al. "The Daikon system for dynamic detection of likely invariants." Science of Computer Programming 69, 2007 (abstract page). https://people.csail.mit.edu/cpacheco/publications/daikon-tool-scp2006-abstract.html +- [A5] Nimmer, Ernst. "Invariant inference for static checking: an empirical evaluation." FSE 2002. https://homes.cs.washington.edu/~mernst/pubs/annotation-study-fse2002.pdf +- [A6] Polikarpova, Ciupa, Meyer. "A comparative study of programmer-written and automatically inferred contracts." ISSTA 2009 (abstract page). https://plse.cs.washington.edu/daikon/pubs/PolikarpovaCM2009-abstract.html +- [A8] Hellendoorn, Devanbu, Polozov, Marron. "Are my invariants valid? A learning approach." arXiv:1903.06089, 2019. https://www.microsoft.com/en-us/research/?p=605031 +- [A9] Hangal, Lam. "Tracking down software bugs using automatic anomaly detection" (DIDUCE). ICSE 2002. https://cs.fit.edu/~pkc/id/related/hangal-icse02.pdf +- [A10] Yang, Evans, Bhardwaj, Bhat, Das. "Perracotta: mining temporal API rules from imperfect traces." ICSE 2006. https://www.cs.virginia.edu/~evans/pubs/perracotta-packaged.pdf +- [A11] Lemieux, Beschastnikh. "Investigating program behavior using the Texada LTL specifications miner." ASE 2015 tool demo. https://www.cs.ubc.ca/~bestchai/papers/ase15-texada-tool-demo.pdf +- [A13] Molina, d'Amorim, Aguirre. "Fuzzing class specifications" (SpecFuzzer). arXiv:2201.10874. https://arxiv.org/abs/2201.10874 +- [A14] Schuler, Dallmeier, Zeller. "Efficient mutation testing by checking invariant violations." ISSTA 2009 (abstract page). https://plse.cs.washington.edu/daikon/pubs/SchulerDZ2009-abstract.html +- [A16] Yang, Sethi, Yan, Cheung, Lu. "Managing data constraints in database-backed web applications." ICSE 2020 (program page). https://2020.icse-conferences.org/details/icse-2020-papers/85/Managing-data-constraints-in-database-backed-web-applications +- [A17] Liu et al. "Leveraging application data constraints to optimize database-backed web applications" (Coco). PVLDB 16(6), 2023. https://www.vldb.org/pvldb/vol16/p1208-liu.pdf +- [A18] Debezium. "PostgreSQL connector" documentation. https://debezium.io/documentation/reference/stable/connectors/postgresql.html +- [A20] PostgreSQL wiki. "Audit trigger 91plus." https://wiki.postgresql.org/wiki/Audit_trigger_91plus +- [A21] Rostin, Albrecht, Bauckmann, Naumann, Leser. "A machine learning approach to foreign key discovery." WebDB 2009. https://hpi.de/oldsite/fileadmin/user_upload/fachgebiete/naumann/publications/2009/WebDB09_crc.pdf +- [A22] Zhang, Hadjieleftheriou, Ooi, Procopiuc, Srivastava. "On multi-column foreign key discovery." PVLDB 3(1), 2010. https://vldb.org/pvldb/vol3/R72.pdf +- [A23] Papenbrock, Kruse, Quiané-Ruiz, Naumann. "Divide & conquer-based inclusion dependency discovery" (Binder). PVLDB 8(7), 2015. https://hpi.de/oldsite/fileadmin/user_upload/fachgebiete/naumann/publications/PDFs/2015_papenbrock_divide.pdf +- [A24] Kruse et al. "Fast approximate discovery of inclusion dependencies" (Faida). BTW 2017. https://dl.gi.de/items/5a813877-a923-4236-95cc-d08ba2c9752a/full +- [A25] Papenbrock, Naumann. "A hybrid approach for efficient unique column combination discovery" (HyUCC). BTW 2017. https://dl.gi.de/items/f9fc2a40-8c8a-4250-b92f-228c0b26807d/full +- [A27] Hulsebos et al. "Sherlock: a deep learning approach to semantic data type detection." KDD 2019. https://arxiv.org/abs/1905.10688 +- [A28] Zhang et al. "Sato: contextual semantic type detection in tables." PVLDB 2020. https://arxiv.org/abs/1911.06311 +- [A29] Suhara et al. "Annotating columns with pre-trained language models" (Doduo). SIGMOD 2022. https://arxiv.org/abs/2104.01785 +- [A30] Korini, Bizer. "Column type annotation using ChatGPT." arXiv:2306.00745, 2023. https://arxiv.org/abs/2306.00745 +- [A31] Buneman, Khanna, Tan. "Why and where: a characterization of data provenance." ICDT 2001. http://homepages.inf.ed.ac.uk/opb/papers/ICDT2001.pdf +- [A32] Green, Karvounarakis, Tannen. "Provenance semirings." PODS 2007. https://www.cs.ucdavis.edu/~green/papers/pods07.pdf + +**B. LLM specifications, oracles, independence** + +- [B1] Endres, Fakhoury, Chakraborty, Lahiri. "Can large language models transform natural language intent into formal method postconditions?" FSE 2024. https://arxiv.org/abs/2310.01831 +- [B2] Lahiri. "Evaluating LLM-driven user-intent formalization for verification-aware languages." FMCAD 2024 (abstract). https://arxiv.org/abs/2406.09757 +- [B3] Ma, Liu, Li, Xie, Bu. "SpecGen: automated generation of formal program specifications via large language models." ICSE 2025. https://arxiv.org/abs/2401.08807 +- [B4] Wen et al. "Enchanting program specification synthesis by large language models using static analysis and program verification" (AutoSpec). arXiv:2404.00762. https://arxiv.org/abs/2404.00762 +- [B5] Wu, Barrett, Narodytska. "Lemur: integrating large language models in automated program verification." ICLR 2024. https://arxiv.org/abs/2310.04870 +- [B6] Sun, Sheng, Padon, Barrett. "Clover: closed-loop verifiable code generation." arXiv:2310.17807. https://arxiv.org/abs/2310.17807 +- [B8] Barr, Harman, McMinn, Shahbaz, Yoo. "The oracle problem in software testing: a survey." IEEE TSE 41(5), 2015. https://web.eecs.umich.edu/~weimerw/2018-481/readings/testoracles.pdf +- [B10] Hossain, Filieri, Dwyer, Elbaum, Visser. "Neural-based test oracle generation: a large-scale evaluation and lessons learned." ESEC/FSE 2023. https://arxiv.org/abs/2307.16023 +- [B11] Shin, Hemmati, Wei, Wang. "Assessing evaluation metrics for neural test oracle generation." IEEE TSE 50(9), 2024. https://arxiv.org/abs/2310.07856 +- [B12] Konstantinou, Degiovanni, Papadakis. "Do LLMs generate test oracles that capture the actual or the expected program behaviour?" arXiv:2410.21136. https://arxiv.org/abs/2410.21136 +- [B14] Segura, Fraser, Sanchez, Ruiz-Cortés. "A survey on metamorphic testing." IEEE TSE 42(9), 2016. https://eprints.whiterose.ac.uk/110335/1/segura16-tse.pdf +- [B15] Chen et al. "Metamorphic testing: a review of challenges and opportunities." ACM CSUR 51(1), 2018. https://i.cs.hku.hk/~tse/Papers/2010s/hlmtCSUR.html +- [B16] Rigger, Su. "Finding bugs in database systems via query partitioning." OOPSLA 2020 (program page). https://2020.splashcon.org/details/splash-2020-oopsla/87/Finding-Bugs-in-Database-Systems-via-Query-Partitioning +- [B17] Zhang, Sun, Dong, Liu. "Can large language models discover metamorphic relations? A large-scale empirical study." SANER 2025 (abstract page only). https://conf.researchr.org/details/saner-2025/saner-2025-papers/41/Can-Large-Language-Models-Discover-Metamorphic-Relations-A-Large-Scale-Empirical-Stu +- [B19] Knight, Leveson. "An experimental evaluation of the assumption of independence in multiversion programming." IEEE TSE 12(1), 1986 (course copy, not the IEEE version). https://www.kth.se/social/files/564df871f2765419e306178d/KnightLeveson.pdf +- [B20] Kim, Garg, Peng, Garg. "Correlated errors in large language models." ICML 2025. https://arxiv.org/abs/2506.07962 +- [B21] Panickssery, Bowman, Feng. "LLM evaluators recognize and favor their own generations." arXiv:2404.13076. https://arxiv.org/abs/2404.13076 + +**C. Mutation, fault injection, holdouts** + +- [C2] Papadakis, Kintis, Zhang, Jia, Le Traon, Harman. "Mutation testing advances: an analysis and survey." Advances in Computers 112, 2019. https://discovery.ucl.ac.uk/id/eprint/10056704/1/survey.pdf +- [C3] Just, Jalali, Inozemtseva, Ernst, Holmes, Fraser. "Are mutants a valid substitute for real faults in software testing?" FSE 2014. https://homes.cs.washington.edu/~rjust/publ/mutants_real_faults_fse_2014.pdf +- [C4] Papadakis, Shin, Yoo, Bae. "Are mutation scores correlated with real fault detection?" ICSE 2018 (abstract page only). https://conf.researchr.org/details/icse-2018/icse-2018-Technical-Papers/78/ +- [C5] Jahangirova, Clark, Harman, Tonella. "Test oracle assessment and improvement." ISSTA 2016. https://discovery.ucl.ac.uk/1493269/1/main.pdf +- [C9] Tuya, Suárez-Cabal, de la Riva. "Mutating database queries." IST 49(4), 2007. https://digibuo.uniovi.es/dspace/bitstream/10651/29255/2/ist-2006-sqlmutation.pdf +- [C10] Kingsbury, Alvaro. "Elle: inferring isolation anomalies from experimental observations." PVLDB 14(3), 2021. https://arxiv.org/abs/2003.10554 +- [C12] Wang et al. "A comprehensive study on large language models for mutation testing." arXiv:2406.09843 (v5). https://arxiv.org/abs/2406.09843 +- [C13] Foster et al. "Mutation-guided LLM-based test generation at Meta." arXiv:2501.12862. https://arxiv.org/abs/2501.12862 +- [C14] Wei et al. "Toward training superintelligent software agents through self-play SWE-RL." arXiv:2512.18552. https://arxiv.org/abs/2512.18552 +- [C15] Chen et al. "CodeT: code generation with generated tests." arXiv:2207.10397. https://arxiv.org/abs/2207.10397 +- [C16] Wang, Yang, Tian, Shen, Wang. "Co-evolving LLM coder and unit tester via reinforcement learning" (CURE). arXiv:2506.03136. https://arxiv.org/abs/2506.03136 +- [C18] Dwork, Feldman, Hardt, Pitassi, Reingold, Roth. "Generalization in adaptive data analysis and holdout reuse." arXiv:1506.02629. https://arxiv.org/abs/1506.02629 +- [C19] Blum, Hardt. "The Ladder: a reliable leaderboard for machine learning competitions." arXiv:1502.04585. https://arxiv.org/abs/1502.04585 +- [C20] Recht, Roelofs, Schmidt, Shankar. "Do ImageNet classifiers generalize to ImageNet?" arXiv:1902.10811. https://arxiv.org/abs/1902.10811 + +**D. GUI identity and commercial tools** + +- [D1] Nass, Alégroth, Feldt, Leotta, Ricca. "Similarity-based web element localization for robust test automation." ACM TOSEM 32(3), 2023. https://arxiv.org/abs/2208.00677 +- [D5] Yandrapally, Thummalapenta, Sinha, Chandra. "Robust test automation using contextual clues." ISSTA 2014. https://www.microsoft.com/en-us/research/?p=167470 +- [D6] Stocco, Yandrapally, Mesbah. "Vista: web test repair using computer vision." ESEC/FSE 2018 demo. https://people.ece.ubc.ca/~astocco/pubs/2018-Stocco-FSE18-demo.pdf +- [D9] Microsoft. "Automation element property identifiers." https://learn.microsoft.com/en-us/windows/win32/winauto/uiauto-automation-element-propids +- [D10] Lu, Yang, Shen, Awadallah. "OmniParser for pure vision based GUI agent." arXiv:2408.00203. https://arxiv.org/abs/2408.00203 +- [D11] Baechler et al. "ScreenAI: a vision-language model for UI and infographics understanding." IJCAI 2024. https://arxiv.org/abs/2402.04615 +- [D12] Cheng et al. "SeeClick: harnessing GUI grounding for advanced visual GUI agents." arXiv:2401.10935. https://arxiv.org/abs/2401.10935 +- [D14] Li et al. "ScreenSpot-Pro: GUI grounding for professional high-resolution computer use." arXiv:2504.07981. https://arxiv.org/abs/2504.07981 +- [D15] HL7. "Patient-match operation", FHIR R4. https://hl7.org/fhir/R4/patient-operation-match.html +- [D16] Healenium. "How Healenium works" and the healenium-web README. https://healenium.io/docs/how_healenium_works +- [D17] Tricentis. "Locators: auto improve" (Testim). https://docs.tricentis.com/testim/content/test-management/locators-auto-improve.htm +- [D18] UiPath. "What is Healing Agent?" https://docs.uipath.com/agents/automation-cloud/latest/user-guide-ha/what-is-healing-agent +- [D21] UiPath. "About Object Repository." https://docs.uipath.com/studio/standalone/2023.10/user-guide/about-object-repository.md +- [D22] UiPath. "Click" activity. https://docs.uipath.com/activities/other/latest/ui-automation/n-click +- [D23] UiPath. "Validation Station." https://docs.uipath.com/document-understanding/automation-cloud/latest/classic-USER-GUIDE/validation-station +- [D24] Microsoft. "Automate using UI elements" (Power Automate desktop). https://learn.microsoft.com/en-us/power-automate/desktop-flows/ui-elements +- [D25] Microsoft. "Test a selector" (Power Automate desktop). https://learn.microsoft.com/en-us/power-automate/desktop-flows/test-selectors +- [D26] Blue Prism. "Application Modeller" (v7.0). https://documentation.blueprism.com/bp-7-0/en-us/frmIntegrationAssistant.htm +- [D27] Automation Anywhere. "Search criteria (Recorder package)." http://docs.automationanywhere.com/r/automation-360/search-criteria-the-recorder-package +- [D28] Anthropic. "Computer use tool." https://platform.claude.com/docs/en/docs/agents-and-tools/tool-use/computer-use-tool +- [D29] OpenAI. "Computer use" guide. https://developers.openai.com/api/docs/guides/tools-computer-use +- [D30] Airbyte. "Schema change management." https://docs.airbyte.com/platform/using-airbyte/schema-change-management + +**Tried and not opened (not used as evidence):** Ammons, Bodik, Larus, +"Mining specifications" (POPL 2002); Staats et al., ISSTA 2012, on how well +people judge generated invariants; Cobb et al., WODA 2011, on invariants +over databases; the Metanome demo text; Schuler and Zeller, "checked +coverage" (ICST 2011); the Science version of the reusable holdout; +ROBULA+ and Leotta's multi-locator paper; WATER; mabl's documentation; the +full ICST 2016 paper on why record/replay tests break. diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index 6250de30b9..c75c9769ab 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -1723,6 +1723,29 @@ def pin(self, alias: str, table: str, column: str, value: Optional[str]) -> str: return f"CASE WHEN {ref} = CAST({self.literal(value)} AS DATE) THEN '1' ELSE '0' END" return f"CASE WHEN {self.text(alias, table, column)} = {self.literal(value)} THEN '1' ELSE '0' END" + def now(self, alias: str, column: str, window_s: int) -> str: + """A flag: this date-time column holds a time within ``window_s`` + seconds before the read (or 5 minutes after it, for clock skew), by + the database's local clock or by UTC (an app may store either). + NULL is not a time, so it fails.""" + + ref = f"{alias}.{self.q(column)}" + w, ahead = int(window_s), 300 + if self.d == "mysql": + clocks = ("NOW()", "UTC_TIMESTAMP()") + spans = [ + f"{ref} BETWEEN {c} - INTERVAL {w} SECOND AND {c} + INTERVAL {ahead} SECOND" + for c in clocks + ] + else: + clocks = ("LOCALTIMESTAMP", "(NOW() AT TIME ZONE 'UTC')") + spans = [ + f"CAST({ref} AS timestamp) BETWEEN {c} - INTERVAL '{w} seconds' " + f"AND {c} + INTERVAL '{ahead} seconds'" + for c in clocks + ] + return f"CASE WHEN {' OR '.join(spans)} THEN '1' ELSE '0' END" + def empty(self, alias: str, table: str, column: str) -> str: col = self.schema.tables[table].column(column) ref = f"{alias}.{self.q(column)}" @@ -1835,6 +1858,18 @@ class Settings: #: Table -> columns of a new row the person named as not checked (they #: vary, are free text left empty, or are the write's time stamp). unchecked: dict[str, set[str]] = field(default_factory=dict) + #: ``now_checks`` answered true: a date-time column of the main record + #: that is the write's time stamp (named unchecked on a create, or the + #: target's excused bookkeeping on an update) must hold a time near the + #: read, instead of any value. + now_checks: bool = False + + @property + def now_window_s(self) -> int: + """How old the write's time stamp may be when it is read: the + execution window plus the recheck, plus a minute.""" + + return int(self.max_window_s + (self.recheck_after_s or 0) + 60) #: Roles of a new row's column that no rule checks: named by the person @@ -1915,6 +1950,18 @@ def build_contract( target_fields.update( c for c in plan.anchor_extra if c in excused.get(anchor, set()) ) + if settings.now_checks: + for c in sorted(plan.anchor_extra): + column = schema.tables[anchor].column(c) + if ( + c in excused.get(anchor, set()) + and column is not None + and column.kind == "datetime" + ): + alias = _add_column( + main, f"now__{_alias(c)}", b.now("t", c, settings.now_window_s) + ) + match[alias] = {"literal": "1"} if plan.mode == "create": # ``found`` pins the same constants as ``create`` (a soft-delete # flag, a status, a type): a trashed record with the task's values @@ -1933,6 +1980,19 @@ def build_contract( settings.actor_params, settings.unchecked, ) + column = schema.tables[anchor].column(col) + if ( + settings.now_checks + and flavor == "create" + and role.role == "timestamp" + and col in settings.unchecked.get(anchor, set()) + and column is not None + and column.kind == "datetime" + ): + alias = _add_column( + main, f"now__{_alias(col)}", b.now("t", col, settings.now_window_s) + ) + match[alias] = {"literal": "1"} if plan.mode == "create" and flavor == "create": support = _support(plan, variant) for p in plan.optional: @@ -3367,6 +3427,7 @@ def table_cols(key: str) -> dict[str, set[str]]: recheck or DEFAULT_RECHECK_S, env, unchecked=unchecked_named, + now_checks=_answer(answers, "now_checks") is True, ) # Tables whose name the SQL guard refuses: a view, or a refusal. diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py new file mode 100644 index 0000000000..5257b990f7 --- /dev/null +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -0,0 +1,914 @@ +"""Set up the write verifier for an app with no person decisions. + +The contract generator (:mod:`openadapt_flow.generate.verify_contract`) asks +a person what it can't decide alone. This module answers those questions +itself, and admits an answer only when a sandbox shows it works: + +1. **Sandbox.** A disposable copy of the app's database, beside it, made + with an admin account the deployer gives once (MySQL and MariaDB: a + database ``_oa_sandbox``; PostgreSQL: a schema ``oa_sandbox``). Faults + are planted only there, never in the app's own database. +2. **Answers.** Every answer comes from a fixed rule (below). None names a + log table or a changed column beyond the generator's own proposal. +3. **Validation.** Held-out demonstrations (correct writes the generator + never saw) are replayed in the sandbox, each between ``begin_write`` and + ``verify_write``: every one must be ``CONFIRMED``, at the recheck too. + Then faults are planted from each held-out write: a wrong value, a wrong + or look-alike record, a duplicate, a collateral change, a lost write, a + late change, a changed bookkeeping cell. Every fault must be stopped. +4. **Admission.** A kind is admitted only when all of its held-out writes + are confirmed and all of its faults are stopped. Anything else is not + admitted: its tasks stop (``choose`` finds no contract), and + ``EXCEPTIONS.md`` lists why. That list is what a person may look at; the + setup needs no answer from them. + +The faults are built from the held-out write's recorded diff, the schema +and the task's values only, never from the contracts under test, so the +planter doesn't share the generator's view of what matters. + +The rules for answers: + +- ``recheck_after_s``: the generator's default, 300 seconds (a policy, not a + judgment about the app). The sandbox uses a few seconds, so validation + doesn't wait. +- ``phones``: every parameter the generator lists. A wrong digit is a + planted fault, so a parameter that isn't a phone number fails validation. +- ``bookkeeping``: the generator's own proposal (``accept: true`` and its + proposed ``unchecked`` columns), with ``now_checks: true``: an unchecked + or excused time stamp of the main record must then hold a time near the + read. Nothing is added to the proposal. A cell left open is caught by the + bookkeeping faults, and its kind is not admitted. +- ``locations.``: each candidate place in turn; a candidate is kept + only when its kind validates, and only when exactly one does. + +Nothing here calls a model. Writes go only to the sandbox, through the admin +account; the contracts read through the read-only account, as in production. +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import importlib +import json +import os +import re +import sys +import time +import traceback +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +import yaml + +from openadapt_flow.generate import sql_store as store +from openadapt_flow.generate import verify_contract as gen +from openadapt_flow.generate.demo_analysis import Demo, value_text +from openadapt_flow.generate.sql_store import RowChange, Schema, Table + +#: The sandbox's name: a database (MySQL) or a schema (PostgreSQL). Only a +#: store with this name is ever dropped or written. +SANDBOX_SUFFIX = "_oa_sandbox" +PG_SANDBOX_SCHEMA = "oa_sandbox" +#: The recheck window in the sandbox, so validation doesn't wait minutes. +SANDBOX_RECHECK_S = 3 +#: A planted bookkeeping time: far from any write's own time. +FAR_TIME = "2001-02-03 04:05:06" +#: At most this many faults of one class per held-out write. +MAX_PER_CLASS = 12 + + +class ZeroTouchError(RuntimeError): + """The setup can't run as asked (no sandbox, a bad config).""" + + +# -- the sandbox -------------------------------------------------------------------- + + +class Sandbox: + """A copy of the app's tables, beside the app's own, that faults are + planted in. ``admin``: a read config (``sql_driver``, + ``sql_connect_args``, ``sql_password_env``) for an account that can + create a database or schema and grant on it. ``read``: the read-only + account's config, which the contracts use.""" + + def __init__(self, admin: Mapping[str, Any], read: Mapping[str, Any]) -> None: + self.admin = dict(admin) + self.read = dict(read) + self.dialect = store.dialect_of(read) + args = dict(read.get("sql_connect_args") or {}) + self.source = str(args.get("database") or args.get("dbname") or "") + if not self.source: + raise ZeroTouchError("the read config names no database") + self.name = ( + self.source + SANDBOX_SUFFIX + if self.dialect == "mysql" + else PG_SANDBOX_SCHEMA + ) + self.schema: Optional[Schema] = None + + # Connections -------------------------------------------------------------------- + + def _connect(self) -> Any: + mod = importlib.import_module(str(self.admin["sql_driver"])) + args = dict(self.admin.get("sql_connect_args") or {}) + env = self.admin.get("sql_password_env") + if env: + args["password"] = os.environ.get(str(env), "") + if self.dialect == "mysql": + args.pop("database", None) + conn = mod.connect(**args, autocommit=True) + else: + args.setdefault("dbname", self.source) + args.pop("database", None) + conn = mod.connect(**args) + conn.autocommit = True + return conn + + def _exec(self, sql: str, params: Sequence[Any] = ()) -> list[tuple[Any, ...]]: + conn = self._connect() + try: + cur = conn.cursor() + cur.execute(sql, tuple(params)) + try: + return list(cur.fetchall()) + except Exception: # noqa: BLE001 - a statement with no rows + return [] + finally: + conn.close() + + def q(self, name: str) -> str: + return store.quote(self.dialect, name) + + def t(self, table: str) -> str: + return f"{self.q(self.name)}.{self.q(table)}" + + def _guard(self) -> None: + ok = ( + self.name.endswith(SANDBOX_SUFFIX) + if self.dialect == "mysql" + else self.name == PG_SANDBOX_SCHEMA + ) + if not ok or self.name == self.source: + raise ZeroTouchError(f"refusing to write to {self.name!r}: not a sandbox") + + # Make and drop -------------------------------------------------------------------- + + def create(self, schema: Schema, reader: str) -> None: + """Copy every base table of ``schema`` into the sandbox, and let the + read account (``reader``: MySQL ``'user'@'host'``, PostgreSQL a role) + read it. An earlier sandbox of the same name is replaced.""" + + self._guard() + self.drop() + conn = self._connect() + try: + cur = conn.cursor() + if self.dialect == "mysql": + cur.execute(f"CREATE DATABASE {self.q(self.name)}") + src = self.q(self.source) + for name in sorted(schema.tables): + cur.execute(f"CREATE TABLE {self.t(name)} LIKE {src}.{self.q(name)}") + cur.execute( + f"INSERT INTO {self.t(name)} SELECT * FROM {src}.{self.q(name)}" + ) + user, _, host = reader.partition("@") + cur.execute( + f"GRANT SELECT, SHOW VIEW, TRIGGER ON {self.q(self.name)}.* " + "TO %s@%s", + (user.strip("'`\""), host.strip("'`\"") or "%"), + ) + else: + cur.execute(f"CREATE SCHEMA {self.q(self.name)}") + cur.execute("SELECT current_schema()") + src = self.q(str(cur.fetchone()[0])) + for name in sorted(schema.tables): + cur.execute( + f"CREATE TABLE {self.t(name)} (LIKE {src}.{self.q(name)} " + "INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES)" + ) + cur.execute( + f"INSERT INTO {self.t(name)} OVERRIDING SYSTEM VALUE " + f"SELECT * FROM {src}.{self.q(name)}" + ) + cur.execute(f"GRANT USAGE ON SCHEMA {self.q(self.name)} TO {self.q(reader)}") + cur.execute( + f"GRANT SELECT ON ALL TABLES IN SCHEMA {self.q(self.name)} " + f"TO {self.q(reader)}" + ) + finally: + conn.close() + self.schema = store.read_schema(store.connector(self.read_config()), self.dialect) + + def drop(self) -> None: + self._guard() + if self.dialect == "mysql": + self._exec(f"DROP DATABASE IF EXISTS {self.q(self.name)}") + else: + self._exec(f"DROP SCHEMA IF EXISTS {self.q(self.name)} CASCADE") + + def read_config(self) -> dict[str, Any]: + """The read-only account's config, pointed at the sandbox.""" + + out = json.loads(json.dumps(self.read)) + args = out.setdefault("sql_connect_args", {}) + if self.dialect == "mysql": + args["database"] = self.name + else: + args["options"] = f"-c search_path={self.name}" + return out + + # Rows --------------------------------------------------------------------------- + + def _table(self, name: str) -> Table: + assert self.schema is not None + return self.schema.tables[name] + + def _where(self, table: Table, row: Mapping[str, Optional[str]]) -> tuple[str, list[Any]]: + if not table.primary_key: + raise ZeroTouchError(f"{table.name} has no primary key; rows can't be planted") + parts, args = [], [] + for c in table.primary_key: + v = row.get(c) + if v is None: + raise ZeroTouchError(f"{table.name}: a key cell is NULL") + parts.append(f"{self.q(c)} = %s") + args.append(v) + return " AND ".join(parts), args + + def get(self, table: str, row: Mapping[str, Optional[str]]) -> Optional[dict[str, Optional[str]]]: + t = self._table(table) + where, args = self._where(t, row) + cols = t.column_names + rows = self._exec( + f"SELECT {', '.join(self.q(c) for c in cols)} FROM {self.t(table)} WHERE {where}", + args, + ) + if not rows: + return None + return {c: store.cell_text(v) for c, v in zip(cols, rows[0])} + + def rows(self, table: str, limit: int = 50) -> list[dict[str, Optional[str]]]: + t = self._table(table) + cols = t.column_names + order = ", ".join(self.q(c) for c in t.primary_key) or self.q(cols[0]) + got = self._exec( + f"SELECT {', '.join(self.q(c) for c in cols)} FROM {self.t(table)} " + f"ORDER BY {order} LIMIT {int(limit)}" + ) + return [{c: store.cell_text(v) for c, v in zip(cols, r)} for r in got] + + def max_key(self, table: str) -> Optional[int]: + t = self._table(table) + if len(t.primary_key) != 1: + return None + col = t.column(t.primary_key[0]) + if col is None or col.kind != "number": + return None + got = self._exec(f"SELECT MAX({self.q(col.name)}) FROM {self.t(table)}") + return int(got[0][0] or 0) + + def put(self, table: str, row: Optional[Mapping[str, Optional[str]]], key: Mapping[str, Optional[str]]) -> None: + """Make the row with ``key``'s primary key be ``row`` (None: absent).""" + + self._guard() + t = self._table(table) + where, args = self._where(t, key) + conn = self._connect() + try: + cur = conn.cursor() + cur.execute(f"DELETE FROM {self.t(table)} WHERE {where}", tuple(args)) + if row is not None: + cols = [c for c in t.column_names if c in row] + over = "" if self.dialect == "mysql" else " OVERRIDING SYSTEM VALUE" + cur.execute( + f"INSERT INTO {self.t(table)} ({', '.join(self.q(c) for c in cols)})" + f"{over} VALUES ({', '.join(['%s'] * len(cols))})", + tuple(_db_value(t, c, row.get(c)) for c in cols), + ) + finally: + conn.close() + + +def _db_value(table: Table, column: str, text: Optional[str]) -> Any: + """A snapshot cell back as a value the driver can bind.""" + + if text is None: + return None + col = table.column(column) + if col is not None and col.kind == "binary" and text.startswith("0x"): + return bytes.fromhex(text[2:]) + if col is not None and col.kind == "bool" and text in ("true", "false"): + return text == "true" + return text + + +# -- operations: a write as rows to put --------------------------------------------- + + +@dataclass +class Op: + """Make row ``key`` of ``table`` be ``row`` (None: absent).""" + + table: str + key: dict[str, Optional[str]] + row: Optional[dict[str, Optional[str]]] + + +def forward(diff_changes: Sequence[RowChange]) -> list[Op]: + out = [] + for ch in diff_changes: + ref = ch.after if ch.after is not None else ch.before + out.append(Op(ch.table, dict(ref or {}), dict(ch.after) if ch.after else None)) + return out + + +def backward(diff_changes: Sequence[RowChange]) -> list[Op]: + out = [] + for ch in diff_changes: + ref = ch.before if ch.before is not None else ch.after + out.append(Op(ch.table, dict(ref or {}), dict(ch.before) if ch.before else None)) + return out + + +def _parse_time(text: Optional[str]) -> Optional[_dt.datetime]: + if not text: + return None + raw = text.replace("T", " ").split("+")[0] + for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"): + try: + return _dt.datetime.strptime(raw, fmt) + except ValueError: + continue + return None + + +def shift_times(schema: Schema, ops: list[Op], demo: Demo, values_text: set[str]) -> list[Op]: + """Move the write's own time stamps to now. A date-time cell within a + day of when the demonstration was made, and not a task's value, moves by + the time since then, so a replay looks like a write made now (in + whatever time zone the app stores).""" + + start = _parse_time(demo.started_at) + if start is None: + return ops + delta = _dt.datetime.now(_dt.timezone.utc).replace(tzinfo=None) - start + lo, hi = start - _dt.timedelta(days=1), start + _dt.timedelta(days=1) + out = [] + for op in ops: + if op.row is None: + out.append(op) + continue + t = schema.tables[op.table] + row = dict(op.row) + for c in t.columns: + if c.kind != "datetime" or c.name in t.primary_key: + continue + when = _parse_time(row.get(c.name)) + if when is None or not (lo <= when <= hi) or row.get(c.name) in values_text: + continue + row[c.name] = (when + delta).isoformat(sep=" ", timespec="seconds") + out.append(Op(op.table, op.key, row)) + return out + + +# -- faults ---------------------------------------------------------------------- + + +@dataclass +class Case: + """One validation run on a held-out write. ``write`` is put after + ``begin_write``; ``late`` after a ``CONFIRMED`` and before the recheck. + ``correct``: the run must be confirmed; else it must be stopped.""" + + name: str + cls: str + write: list[Op] + late: list[Op] = field(default_factory=list) + correct: bool = False + + +def mutate(table: Table, column: str, text: Optional[str]) -> Optional[str]: + """Another value of the column's type, plainly different.""" + + col = table.column(column) + kind = col.kind if col else "text" + if kind == "datetime": + return FAR_TIME if text != FAR_TIME else "2002-03-04 05:06:07" + if kind == "date": + return "2001-02-03" if text != "2001-02-03" else "2002-03-04" + if kind == "bool": + return "false" if text in ("true", "1") else "true" + if kind == "number": + try: + n = float(text) if text is not None else 0.0 + except ValueError: + return None + return str(int(n) + 7) if n.is_integer() else str(n + 7) + if kind == "text": + return (text or "") + " (x)" + return None + + +def _text_values(values: Mapping[str, Any]) -> set[str]: + out: set[str] = set() + + def walk(v: Any) -> None: + if v is None: + return + if isinstance(v, Mapping): + for x in v.values(): + walk(x) + elif isinstance(v, list): + for x in v: + walk(x) + else: + out.add(value_text(v)) + + walk(values) + return out + + +def _is_value(text: Optional[str], values: set[str]) -> bool: + if text is None: + return False + if text in values: + return True + for v in values: + try: + if float(text) == float(v): + return True + except ValueError: + pass + if len(v) >= 10 and text.startswith(v): # a date stored as a date-time + return True + return False + + +def faults(schema: Schema, sb: Sandbox, demo: Demo, write: list[Op]) -> list[Case]: + """Faults planted from one held-out write: its diff, the schema and its + task's values. The contracts are never read here.""" + + values = _text_values(demo.values) + out: list[Case] = [Case("lost", "lost_write", [])] + touched = {op.table for op in write} + counts: dict[str, int] = {} + + def add(case: Case) -> None: + n = counts.get(case.cls, 0) + if n < MAX_PER_CLASS: + counts[case.cls] = n + 1 + out.append(case) + + for i, op in enumerate(write): + t = schema.tables[op.table] + if op.row is None: + continue + before = sb.get(op.table, op.key) + is_new = before is None + fks = {fk.column: fk for fk in t.foreign_keys} + for c in t.columns: + if c.name in t.primary_key: + continue + cur = op.row.get(c.name) + if not is_new and before is not None and before.get(c.name) == cur: + continue # not changed by the write: see collateral_target + if is_new and cur is None: + continue + new = None + if c.name in fks: + others = [ + r for r in sb.rows(fks[c.name].table, 20) + if r.get(fks[c.name].ref_column) not in (None, cur) + ] + new = others[0].get(fks[c.name].ref_column) if others else None + if new is None: + new = mutate(t, c.name, cur) + if new is None: + continue + cls = "wrong_value" if _is_value(cur, values) else "bookkeeping" + row = dict(op.row) + row[c.name] = new + add(Case(f"{cls}:{op.table}.{c.name}", cls, + write[:i] + [Op(op.table, op.key, row)] + write[i + 1:])) + if not is_new and before is not None: + # The write's changes landed on another row of the table. + changed = {c for c in t.column_names if before.get(c) != op.row.get(c)} + for other in sb.rows(op.table, 20): + if store.row_key(t, other) == store.row_key(t, op.key): + continue + moved = dict(other) + for c in changed: + moved[c] = op.row.get(c) + add(Case(f"wrong_record:{op.table}", "wrong_record", + write[:i] + write[i + 1:] + [Op(op.table, other, moved)])) + break + # A column the write didn't change, changed on the target. + for c in t.columns: + if c.name in t.primary_key or c.name in changed: + continue + new = mutate(t, c.name, op.row.get(c.name)) + if new is None: + continue + row = dict(op.row) + row[c.name] = new + add(Case(f"collateral_target:{op.table}.{c.name}", "collateral_target", + write[:i] + [Op(op.table, op.key, row)] + write[i + 1:])) + if is_new: + top = sb.max_key(op.table) + if top is not None: + pk = t.primary_key[0] + dup = dict(op.row) + dup[pk] = str(top + 1000) + add(Case(f"duplicate:{op.table}", "duplicate", + write + [Op(op.table, {pk: dup[pk]}, dup)])) + # The values landed on an existing look-alike instead. + for other in sb.rows(op.table, 20): + look = dict(op.row) + for c in t.primary_key: + look[c] = other.get(c) + add(Case(f"look_alike:{op.table}", "look_alike", + write[:i] + write[i + 1:] + [Op(op.table, other, look)])) + break + # Another row of a touched table, and a new row in an untouched one. + for name in sorted(touched): + t = schema.tables[name] + mine = {store.row_key(t, op.key) for op in write if op.table == name} + for other in sb.rows(name, 20): + if store.row_key(t, other) in mine: + continue + for c in t.columns: + if c.name in t.primary_key: + continue + new = mutate(t, c.name, other.get(c.name)) + if new is None: + continue + row = dict(other) + row[c.name] = new + add(Case(f"collateral_row:{name}.{c.name}", "collateral_row", + write + [Op(name, other, row)])) + add(Case(f"late:{name}.{c.name}", "late_write", + list(write), late=[Op(name, other, row)])) + break + break + for name in sorted(set(schema.tables) - touched): + top = sb.max_key(name) + sample = sb.rows(name, 1) + if top is None or not sample: + continue + row = dict(sample[0]) + pk = schema.tables[name].primary_key[0] + row[pk] = str(top + 1000) + add(Case(f"extra_row:{name}", "extra_row", write + [Op(name, {pk: row[pk]}, row)])) + if counts.get("extra_row", 0) >= 2: + break + return out + + +# -- running a case --------------------------------------------------------------- + + +@dataclass +class Outcome: + case: str + cls: str + correct: bool + outcome: str + passed: bool + reasons: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + """Confirmed when correct; stopped when a fault.""" + + return self.passed if self.correct else not self.passed + + +def _put_all(sb: Sandbox, ops: Sequence[Op]) -> None: + for op in ops: + sb.put(op.table, op.row, op.key) + + +def run_case( + sb: Sandbox, + manifest: Path, + demo: Demo, + case: Case, + actor_id: str, + extra: Mapping[str, Any], +) -> Outcome: + """Bring the held-out write's rows back to before it, then run the case + between ``begin_write`` and ``verify_write`` (and the recheck), then put + every touched row back.""" + + from openadapt_flow.verifier import Actor, begin_write, recheck, verify_write + + undo = backward(demo.diff.changes) + all_ops = undo + case.write + case.late + saved: dict[tuple[str, str], tuple[dict[str, Optional[str]], Optional[dict[str, Optional[str]]]]] = {} + assert sb.schema is not None + for op in all_ops: + k = (op.table, store.row_key(sb.schema.tables[op.table], op.key)) + if k not in saved: + saved[k] = (op.key, sb.get(op.table, op.key)) + try: + _put_all(sb, undo) + try: + contract, params = gen.choose(manifest, demo.kind, demo.values, extra) + except gen.GenerateError as exc: + return Outcome(case.name, case.cls, case.correct, "NO_CONTRACT", False, [str(exc)]) + actor = Actor("agent", actor_id) + pending = begin_write(contract, params, actor=actor, ledger=None, + write_key=f"zt-{time.time_ns()}") # fmt: skip + _put_all(sb, case.write) + r = verify_write(pending) + outcome = str(getattr(r.outcome, "value", r.outcome)) + reasons = list(r.reasons) + passed = outcome == "CONFIRMED" + if passed and getattr(r, "recheck", None) is not None: + _put_all(sb, case.late) + rc = recheck(contract, r, params=params, actor=actor, wait=True, ledger=None) + rc_out = str(getattr(rc.outcome, "value", rc.outcome)) + if rc_out != "CONFIRMED": + passed, outcome = False, f"RECHECK_{rc_out}" + reasons += list(rc.reasons) + elif passed and case.late: + passed, outcome = False, "NO_RECHECK" + reasons.append("the contract schedules no recheck, so a late write can't be seen") + # A contract with no recheck can't stop a late write: count it as + # a pass of the fault, which fails validation. + passed = True + return Outcome(case.name, case.cls, case.correct, outcome, passed, reasons[:6]) + except Exception as exc: # noqa: BLE001 - a case that can't run admits nothing + return Outcome(case.name, case.cls, case.correct, "ERROR", False, + [f"{type(exc).__name__}: {exc}", traceback.format_exc(limit=3)]) + finally: + for (table, _), (key, row) in saved.items(): + sb.put(table, row, key) + + +# -- answers ---------------------------------------------------------------------- + + +def auto_answers(report: Mapping[str, Any], picks: Mapping[str, Mapping[str, int]] = {}) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """The answers this module gives, and a ledger saying by which rule.""" + + answers: dict[str, Any] = {"now_checks": True} + ledger: list[dict[str, Any]] = [ + {"key": "now_checks", "answer": True, + "rule": "an unchecked or excused time stamp of the main record must hold a time near the read"}, + ] + for q in report.get("questions") or []: + key = str(q.get("key")) + proposed = q.get("proposal") + if key == "recheck_after_s": + answers[key] = gen.DEFAULT_RECHECK_S + ledger.append({"key": key, "answer": gen.DEFAULT_RECHECK_S, "rule": "policy default"}) + elif key == "phones": + answers[key] = list(proposed or []) + ledger.append({"key": key, "answer": answers[key], + "rule": "every listed parameter; wrong digits are planted faults"}) + elif key == "bookkeeping": + prop = dict(proposed or {}) if isinstance(proposed, Mapping) else {} + ans = {"accept": True} + if prop.get("unchecked"): + ans["unchecked"] = prop["unchecked"] + answers[key] = ans + ledger.append({"key": key, "answer": ans, + "rule": "the generator's own proposal; nothing added (no logs_add, no columns_add)"}) + elif key.startswith("locations."): + if key in picks: + answers[key] = dict(picks[key]) + ledger.append({"key": key, "answer": answers[key], + "rule": "the one candidate whose kind validated"}) + return answers, ledger + + +def location_options(report: Mapping[str, Any]) -> dict[str, dict[str, int]]: + """``locations.`` -> parameter -> how many candidate places.""" + + out: dict[str, dict[str, int]] = {} + for q in report.get("questions") or []: + key = str(q.get("key")) + if not key.startswith("locations."): + continue + for name, opts in re.findall(r"`([^`]+)` is in several places: (.*?)(?= `|$)", str(q.get("text"))): + out.setdefault(key, {})[name] = len(re.findall(r"\[\d+\]", opts)) + return out + + +# -- the whole setup ---------------------------------------------------------------- + + +def _load_yaml(path: Path) -> Any: + return yaml.safe_load(Path(path).read_text()) if path else None + + +def validate( + sb: Sandbox, + manifest: Path, + holdout: Sequence[Demo], + actor_id: str, + extra: Mapping[str, Any], + log: Callable[[str], None] = lambda s: None, +) -> dict[str, list[Outcome]]: + """Every held-out write, replayed and faulted, by kind.""" + + assert sb.schema is not None + by_kind: dict[str, list[Outcome]] = {} + for demo in holdout: + values = _text_values(demo.values) + write = shift_times(sb.schema, forward(demo.diff.changes), demo, values) + cases = [Case("correct", "correct", write, correct=True)] + # Faults are planted from the store as it was before the write. + undo = backward(demo.diff.changes) + saved = [(op, sb.get(op.table, op.key)) for op in undo] + _put_all(sb, undo) + try: + cases += faults(sb.schema, sb, demo, write) + finally: + for op, row in saved: + sb.put(op.table, row, op.key) + for case in cases: + res = run_case(sb, manifest, demo, case, actor_id, extra) + log(f"{demo.name} {case.name}: {res.outcome} {'ok' if res.ok else 'FAIL'}") + by_kind.setdefault(demo.kind, []).append(res) + return by_kind + + +def summarize(by_kind: Mapping[str, Sequence[Outcome]], kinds: Sequence[str]) -> dict[str, Any]: + out: dict[str, Any] = {} + for k in kinds: + rs = list(by_kind.get(k) or []) + correct = [r for r in rs if r.correct] + fl = [r for r in rs if not r.correct] + out[k] = { + "admitted": bool(correct) and all(r.ok for r in rs), + "correct_writes": len(correct), + "false_stops": [f"{r.outcome}: {'; '.join(r.reasons[:2])}" for r in correct if not r.ok], + "faults": len(fl), + "faults_stopped": sum(1 for r in fl if r.ok), + "faults_passed": [r.case for r in fl if not r.ok], + "by_class": { + c: f"{sum(1 for r in fl if r.cls == c and r.ok)}/{sum(1 for r in fl if r.cls == c)}" + for c in sorted({r.cls for r in fl}) + }, + } + if not rs: + out[k]["why"] = "no held-out write of this kind" + return out + + +def exceptions_md(summary: Mapping[str, Any], report: Mapping[str, Any]) -> str: + lines = ["# What setup could not validate", "", + "No person decision was asked for. The kinds below are not admitted, so", + "their tasks stop (no contract). Each line says why.", ""] + n = 0 + for k, s in summary.items(): + if s["admitted"]: + continue + n += 1 + lines.append(f"{n}. `{k}`:") + if s.get("why"): + lines.append(f" - {s['why']}") + for fs in s["false_stops"]: + lines.append(f" - a held-out correct write stopped: {fs}") + for f in s["faults_passed"]: + lines.append(f" - a planted fault passed: {f}") + for r in ((report.get("kinds") or {}).get(k) or {}).get("refused") or []: + lines.append(f" - generator: {r}") + if n == 0: + lines.append("Every kind is admitted.") + return "\n".join(lines) + "\n" + + +def onboard( + *, + read: dict[str, Any], + admin: dict[str, Any], + reader: str, + kinds_path: Path, + demo_dir: Path, + holdout_dir: Path, + out: Path, + actor_params: Mapping[str, str], + actor_id: str, + keep_sandbox: bool = False, + log: Callable[[str], None] = print, +) -> dict[str, Any]: + """Propose, answer, validate in the sandbox, and write the admitted + contracts for the app's own store to ``out/contracts``.""" + + t0 = time.time() + kinds = gen.load_kinds(kinds_path) + holdout = gen.load_demos(holdout_dir) + schema = store.read_schema(store.connector(read), store.dialect_of(read)) + sb = Sandbox(admin, read) + sb.create(schema, reader) + sread = sb.read_config() + work = out / "sandbox" + try: + report = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / "q", + actor_params=actor_params, answers={}) # fmt: skip + options = location_options(report) + trials: list[dict[str, dict[str, int]]] = [{}] + for key, params in options.items(): + for name, n in params.items(): + trials += [{key: {name: i}} for i in range(n)] + kind_names = [k.kind for k in kinds] + best: dict[str, Any] = {} + results: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = [] + for picks in trials: + answers, ledger = auto_answers(report, picks) + sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} + tag = "base" if not picks else "pick-" + "-".join( + f"{k}.{p}.{i}" for k, d in picks.items() for p, i in d.items()) + log(f"propose ({tag})") + rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / tag, + actor_params=actor_params, answers=sans) # fmt: skip + by_kind = validate(sb, work / tag / "manifest.yaml", holdout, actor_id, + dict(actor_params), log) # fmt: skip + summary = summarize(by_kind, kind_names) + results.append((picks, answers, summary)) + best = best or {"answers": answers, "ledger": ledger, "summary": summary, "report": rep} + # A location pick is kept when exactly one candidate admits its kind. + final_picks: dict[str, dict[str, int]] = {} + for key, params in options.items(): + kind = key.split(".", 1)[1] + for name in params: + ok = [p[key][name] for p, _, s in results + if key in p and name in p[key] and s[kind]["admitted"]] + if len(ok) == 1: + final_picks.setdefault(key, {})[name] = ok[0] + answers, ledger = auto_answers(report, final_picks) + if final_picks: + sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} + rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / "final", + actor_params=actor_params, answers=sans) # fmt: skip + summary = summarize(validate(sb, work / "final" / "manifest.yaml", holdout, + actor_id, dict(actor_params), log), kind_names) # fmt: skip + else: + summary, rep = best["summary"], best["report"] + finally: + if not keep_sandbox: + sb.drop() + # The production contracts: the same answers, against the app's own store. + prod = gen.propose(read=read, kinds=kinds, demo_dir=demo_dir, out=out / "contracts", + actor_params=actor_params, answers=answers) # fmt: skip + manifest_path = out / "contracts" / "manifest.yaml" + manifest = _load_yaml(manifest_path) + admitted = sorted(k for k, s in summary.items() if s["admitted"]) + for k in list((manifest.get("kinds") or {})): + if k not in admitted: + manifest["kinds"].pop(k) + manifest["zero_touch"] = {"admitted": admitted} + manifest_path.write_text(yaml.safe_dump(manifest, sort_keys=False)) + result = { + "minutes": round((time.time() - t0) / 60, 2), + "person_decisions": 0, + "answers": answers, + "ledger": ledger, + "admitted": admitted, + "kinds": summary, + "exceptions": sum(1 for s in summary.values() if not s["admitted"]), + } + (out / "ONBOARDING.json").write_text(json.dumps(result, indent=2, default=str)) + (out / "EXCEPTIONS.md").write_text(exceptions_md(summary, prod)) + return result + + +def main(argv: Optional[Sequence[str]] = None) -> int: + ap = argparse.ArgumentParser( + prog="python -m openadapt_flow.generate.zero_touch", + description="Set up verify contracts with no person decisions: answer the " + "generator's questions by rule, validate them in a sandbox copy with " + "held-out writes and planted faults, and admit only what validates.", + ) + ap.add_argument("--read", required=True, type=Path, help="read-only account (read.yaml)") + ap.add_argument("--admin", required=True, type=Path, + help="admin account that may create the sandbox beside the app's store") + ap.add_argument("--reader", required=True, + help="the read account to grant on the sandbox: MySQL 'user'@'host', PostgreSQL role") + ap.add_argument("--kinds", required=True, type=Path) + ap.add_argument("--demos", required=True, type=Path, help="demonstrations the generator learns from") + ap.add_argument("--holdout", required=True, type=Path, + help="other correct writes, recorded the same way, used only to validate") + ap.add_argument("--out", required=True, type=Path) + ap.add_argument("--actor-param", action="append", default=[], help="NAME=VALUE") + ap.add_argument("--actor-id", required=True, help="the agent's user id as the verifier's Actor") + ap.add_argument("--keep-sandbox", action="store_true") + args = ap.parse_args(argv) + result = onboard( + read=gen.load_read(args.read), admin=gen.load_read(args.admin), reader=args.reader, + kinds_path=args.kinds, demo_dir=args.demos, holdout_dir=args.holdout, out=args.out, + actor_params=gen._parse_actor(args.actor_param), actor_id=args.actor_id, + keep_sandbox=args.keep_sandbox, + ) # fmt: skip + print(json.dumps({k: result[k] for k in ("minutes", "person_decisions", "admitted", "exceptions")})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py new file mode 100644 index 0000000000..4a711f270b --- /dev/null +++ b/packages/flow/tests/test_zero_touch.py @@ -0,0 +1,160 @@ +"""Zero-touch setup on a real PostgreSQL: no person answers, a sandbox decides. + +The issue tracker from ``test_generate_verify_contract`` (names behind keys, +a journal, custom fields, look-alike projects). Demonstrations train the +generator; other correct writes are held out. ``zero_touch.onboard`` answers +every question by its rules, plants faults in a sandbox schema, and admits a +kind only when every held-out write is confirmed and every fault stopped. +The app's own tables must not change. Skipped without PostgreSQL. Records +are made up. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Callable + +import pytest +import yaml + +psycopg2 = pytest.importorskip("psycopg2") + +from openadapt_flow.generate import verify_contract as gen # noqa: E402 +from openadapt_flow.generate import zero_touch as zt # noqa: E402 +from tests.test_generate_verify_contract import BOT, KINDS, SCHEMA, App, pg # noqa: E402,F401 + +HOLDOUT = { + "create_issue": [ + {"project": "Help Desk", "subject": "Badge reader offline", "status": "In Progress", + "assignee": "Jon Park", "due_date": "2026-12-01", "customer_ref": None}, + ], + "update_issue": [ + {"project": "Help Desk", "subject": "VPN drops", "status": "Resolved", + "assignee": None, "due_date": None, "done_ratio": None, "notes": None}, + ], +} # fmt: skip + + +def _digest(app: App) -> str: + h = hashlib.sha256() + for (t,) in app.q( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema='public' ORDER BY table_name" + ): + h.update(json.dumps(app.q(f"SELECT * FROM {t} ORDER BY 1"), default=str).encode()) + return h.hexdigest() + + +@pytest.fixture(scope="module") +def onboarded(pg: Any) -> dict[str, Any]: # noqa: F811 + root, port = pg["root"], pg["port"] + admin = psycopg2.connect(host="127.0.0.1", port=port, user="admin", dbname="postgres") + admin.autocommit = True + admin.cursor().execute("CREATE DATABASE tracker_zt") + admin.close() + app = App(port, "tracker_zt") + with app.con() as c, c.cursor() as cur: + cur.execute(SCHEMA) + cur.execute("UPDATE issues SET lock_version = 4") + for seq in ("journals_id_seq", "journal_details_id_seq", "custom_values_id_seq"): + cur.execute(f"SELECT setval('{seq}', 100)") + os.environ["OA_ZT_TEST_PW"] = "reader-pw" + work = root / "zt" + work.mkdir() + read = {"sql_driver": "psycopg2", + "sql_connect_args": {"host": "127.0.0.1", "port": port, "user": "oa_reader", + "dbname": "tracker_zt"}, + "sql_password_env": "OA_ZT_TEST_PW"} # fmt: skip + admin_read = {"sql_driver": "psycopg2", + "sql_connect_args": {"host": "127.0.0.1", "port": port, "user": "admin", + "dbname": "tracker_zt"}} # fmt: skip + (work / "read.yaml").write_text(yaml.safe_dump(read)) + (work / "kinds.yaml").write_text(yaml.safe_dump(KINDS, allow_unicode=True)) + common = ["--read", str(work / "read.yaml")] + + def demo(folder: str, kind: str, values: dict[str, Any], write: Callable[[], Any]) -> None: + assert gen.main(["snapshot", *common, "--out", str(work / "before.json")]) == 0 + write() + assert gen.main(["record", *common, "--before", str(work / "before.json"), "--kind", kind, + "--values", json.dumps(values), "--demos", str(work / folder)]) == 0 # fmt: skip + + for ex in KINDS["kinds"][0]["examples"]: + demo("demos", "create_issue", ex["values"], lambda v=ex["values"]: app.create_issue(v)) + for ex in KINDS["kinds"][1]["examples"]: + v = ex["values"] + demo("demos", "update_issue", v, + lambda v=v: app.update_issue(app.find(v["project"], v["subject"]), v)) # fmt: skip + for v in HOLDOUT["create_issue"]: + demo("holdout", "create_issue", v, lambda v=v: app.create_issue(v)) + for v in HOLDOUT["update_issue"]: + demo("holdout", "update_issue", v, + lambda v=v: app.update_issue(app.find(v["project"], v["subject"]), v)) # fmt: skip + before = _digest(app) + lines: list[str] = [] + result = zt.onboard( + read=read, admin=admin_read, reader="oa_reader", kinds_path=work / "kinds.yaml", + demo_dir=work / "demos", holdout_dir=work / "holdout", out=work / "out", + actor_params={"bot_user_id": str(BOT)}, actor_id="ops-bot", log=lines.append, + ) # fmt: skip + return {"app": app, "result": result, "before": before, "out": work / "out", "log": lines} + + +def test_no_person_decision_and_a_ledger(onboarded: dict[str, Any]) -> None: + r = onboarded["result"] + assert r["person_decisions"] == 0 + keys = {e["key"] for e in r["ledger"]} + assert {"now_checks", "bookkeeping", "recheck_after_s"} <= keys + # Nothing beyond the generator's proposal: no log tables, no added columns. + book = r["answers"]["bookkeeping"] + assert "logs_add" not in book and "columns_add" not in book + assert (onboarded["out"] / "EXCEPTIONS.md").exists() + + +def test_the_apps_own_tables_did_not_change(onboarded: dict[str, Any]) -> None: + assert _digest(onboarded["app"]) == onboarded["before"] + schemas = {s for (s,) in onboarded["app"].q("SELECT schema_name FROM information_schema.schemata")} + assert zt.PG_SANDBOX_SCHEMA not in schemas + + +def test_an_admitted_kind_confirmed_every_held_out_write_and_stopped_every_fault( + onboarded: dict[str, Any], +) -> None: + kinds = onboarded["result"]["kinds"] + for name, s in kinds.items(): + if s["admitted"]: + assert s["correct_writes"] >= 1 and not s["false_stops"] + assert s["faults"] > 0 and s["faults_stopped"] == s["faults"], (name, s) + else: + assert s["false_stops"] or s["faults_passed"] or s.get("why"), (name, s) + + +def test_create_issue_is_admitted_with_every_fault_class(onboarded: dict[str, Any]) -> None: + s = onboarded["result"]["kinds"]["create_issue"] + assert s["admitted"], (s, "\n".join(onboarded["log"][-60:])) + for cls in ("lost_write", "wrong_value", "duplicate", "look_alike", "collateral_row", + "late_write", "extra_row", "bookkeeping"): # fmt: skip + assert cls in s["by_class"], s["by_class"] + + +def test_a_kind_not_admitted_has_no_production_contract(onboarded: dict[str, Any]) -> None: + r = onboarded["result"] + manifest = yaml.safe_load((onboarded["out"] / "contracts" / "manifest.yaml").read_text()) + assert sorted(manifest["kinds"]) == r["admitted"] + for name, s in r["kinds"].items(): + if not s["admitted"]: + with pytest.raises(gen.GenerateError): + gen.choose(onboarded["out"] / "contracts" / "manifest.yaml", name, + HOLDOUT[name][0], {"bot_user_id": str(BOT)}) # fmt: skip + + +def test_a_planted_time_stamp_fails_the_now_check() -> None: + from openadapt_flow.generate.sql_store import Column, Schema, Table + + schema = Schema("mysql", "d", {"t": Table("t", [Column("at", "datetime", True, None)], [])}) + b = gen.SqlBuilder(schema) + sql = b.now("t", "at", 1260) + assert "NOW() - INTERVAL 1260 SECOND" in sql and "UTC_TIMESTAMP()" in sql + assert zt.mutate(schema.tables["t"], "at", "2026-11-03 10:00:00") == zt.FAR_TIME From 9c96cece326a3eb0c61caf3958383bf89743793a Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 13:08:17 -0500 Subject: [PATCH 02/12] Zero-touch: fault_gate class names, #1520 decisions, now_checks docs, capability row --- CAPABILITIES.md | 1 + llms-full.txt | 1 + packages/flow/docs/GENERATE_CONTRACTS.md | 15 + .../flow/docs/research/ZERO_TOUCH_DESIGN.md | 43 ++- .../openadapt_flow/generate/zero_touch.py | 324 +++++++++++++----- packages/flow/tests/test_zero_touch.py | 101 +++++- 6 files changed, 390 insertions(+), 95 deletions(-) diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 1f9d0d7c2d..0791cd4d07 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -148,6 +148,7 @@ improvising or retrying when something is off. | Generate an ensure-record contract from three demonstrations, the server's CapabilityStatement and its SearchParameter definitions: field paths and `[system=…]` filters, comparison kinds, value maps, identity, search parameters, the closed record and the change feed. It refuses when unsure, asks a person to confirm an identity for a type with no reviewed template, and self-checks every demonstration plus a perturbation per field | On main ([#1653](https://github.com/OpenAdaptAI/OpenAdapt/pull/1653)) | Real app (Medplum 5.2.8: all 16 hand-written Patient lines regenerated; one blind write type, Immunization: 78 s of person time, 2 lines edited after the person declined the proposed identity; blind suite 23/24, no fault passed) | `openadapt_flow/generate/ensure_contract.py`, [RESULTS.md](packages/flow/benchmark/contract_autogen/results/RESULTS.md) | | A stop card for a stopped write: the exact record difference and who else wrote, with Undo or Fix that reads back, or Open in the app; cost modelled from published operators and timed machine steps, as pre-registered | On main ([#1653](https://github.com/OpenAdaptAI/OpenAdapt/pull/1653)) | Real app (Medplum 5.2.8, blind suite: 16 stops, 14 one-tap with no failed read-back; modelled 42 s per caught harm against 420; one wrong category on a false stop, gone in the re-measure with `void`: 24/24, 15 stops, no wrong category, no failed read-back) | `benchmark/stop_card/`, [PREREG_REVIEW.md](packages/flow/benchmark/stop_card/PREREG_REVIEW.md) | | Generate `openadapt.verify/v1` contracts from a MySQL, MariaDB or PostgreSQL schema, the kinds of write with their parameter names, and one demonstration write per example: one contract per field set (an update gets one for each set of fields it may change), and `choose` builds any other set of the kind's fields the same way from the same pinned inputs, list parameters by length, a `found` twin for a record already in place, the whole-database frame on every dialect (it compares the contract's own tables too, so an excused column holds only on the rows the write declares, and the recheck sees a late change to a look-alike row), rows the app adds on its own declared by count, a recheck window the person confirms, and `frame.versions` (a lost update) from a version column or the app's audit trail of saves. Equal ids are never taken for a join (a row's own key, the column a path arrived by), a join only equal ids support is a question when a supported place also fits, and a pin needs two rows that agree. Every undecided point is one numbered question; every contract is loaded and dry-read against the live store before it is written; a needed `TRIGGER` grant or view is applied with a flag or the kind is refused. Not yet: OpenAPI metadata | [#1672](https://github.com/OpenAdaptAI/OpenAdapt/pull/1672) | Real app (first study, instance copies; questions for a person: Dolibarr 4, Redmine 2, EspoCRM 3, against 11 hand decisions each; every in-scope planted fault stopped (22/22, 19/19, 21/21); correct writes confirmed 32/43, 32/38, 27/40; evidence in the private study repository) | `python -m openadapt_flow.generate.verify_contract`, [GENERATE_CONTRACTS.md](packages/flow/docs/GENERATE_CONTRACTS.md) | +| Set up generated contracts with no person answers: answer the generator's questions by fixed rules (its own bookkeeping proposal with time stamps checked against the clock, nothing added), validate them in a sandbox copy of the store beside the app's own (a database or schema the setup makes and drops) by replaying held-out correct writes and planting faults built from each write's recorded diff, never from the contract (missing, wrong value, wrong or look-alike record, duplicate, unrelated edit, a row in another table, a delayed change, a changed bookkeeping cell), and admit a kind only when every held-out write is confirmed and every fault stopped; a kind not admitted has no contract and is listed in `EXCEPTIONS.md`. Not yet: log tables, counters and derived columns, learned excuses, value maps | (this PR) | Unit (real PostgreSQL: one kind admitted with every planted fault stopped, one listed as an exception; a control without the clock check lets the time-stamp fault through). Development apps pending | `python -m openadapt_flow.generate.zero_touch`, [ZERO_TOUCH_DESIGN.md](packages/flow/docs/research/ZERO_TOUCH_DESIGN.md) | | `void` in an ensure-record contract: values that make a record assert nothing (`status="entered-in-error"`). A record holding one is never the person, is not a duplicate, and a created one needs review; it names values to leave out, never values to keep, and can't name an identity field | On main ([#1657](https://github.com/OpenAdaptAI/OpenAdapt/pull/1657)) | Real app (Medplum 5.2.8: 8 live cases written in advance, 8/8 with the line, 5/8 and one silent pass without it; frozen blind Immunization suite 24/24); adversarial review recorded | `openadapt_flow/ensure_record.py`, [RESULTS.md](packages/flow/benchmark/fhir_medplum/void/results/RESULTS.md), [REVIEW_VOID.md](packages/flow/benchmark/fhir_medplum/void/REVIEW_VOID.md) | | Check a patient registration on a commercial EHR through its public FHIR sandbox: Epic (open.epic, R4), read through FHIR search and read with SMART backend auth; the frame is the search set, since the sandbox has no AuditEvent or `_history` | On main ([#1656](https://github.com/OpenAdaptAI/OpenAdapt/pull/1656)) | Real app (Epic public sandbox, August 2026 release; blind round 12/13 with none passed as verified, the miss a false refusal fixed in the contract; caught Epic dropping the phone on a 201 create) | `benchmark/commercial_ehr/`, [FINDINGS.md](packages/flow/benchmark/commercial_ehr/FINDINGS.md) | | A public benchmark a vendor reruns with their own agent: `openadapt bench medplum --agent ` runs 37 written-for-it cases (Patient, AllergyIntolerance, Appointment) against a local Medplum through a fault proxy, and scores the agent's claim, a read-back and OpenAdapt's verifier against an end-state oracle | On main ([#1652](https://github.com/OpenAdaptAI/OpenAdapt/pull/1652)) | Real app (Medplum 5.2.8; Claude Sonnet 5.5: OpenAdapt missed 1 of 9 harms with 4 false stops, all corrections that changed the record's identity, against 8 missed by the claim and the read-back; cold run from clone to scorecard 5 min) | [README](packages/flow/benchmark/fhir_medplum/public/README.md), [RESULTS.md](packages/flow/benchmark/fhir_medplum/public/RESULTS.md) | diff --git a/llms-full.txt b/llms-full.txt index 46b17ec00a..39973f789f 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -545,6 +545,7 @@ improvising or retrying when something is off. | Generate an ensure-record contract from three demonstrations, the server's CapabilityStatement and its SearchParameter definitions: field paths and `[system=…]` filters, comparison kinds, value maps, identity, search parameters, the closed record and the change feed. It refuses when unsure, asks a person to confirm an identity for a type with no reviewed template, and self-checks every demonstration plus a perturbation per field | On main ([#1653](https://github.com/OpenAdaptAI/OpenAdapt/pull/1653)) | Real app (Medplum 5.2.8: all 16 hand-written Patient lines regenerated; one blind write type, Immunization: 78 s of person time, 2 lines edited after the person declined the proposed identity; blind suite 23/24, no fault passed) | `openadapt_flow/generate/ensure_contract.py`, [RESULTS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/contract_autogen/results/RESULTS.md) | | A stop card for a stopped write: the exact record difference and who else wrote, with Undo or Fix that reads back, or Open in the app; cost modelled from published operators and timed machine steps, as pre-registered | On main ([#1653](https://github.com/OpenAdaptAI/OpenAdapt/pull/1653)) | Real app (Medplum 5.2.8, blind suite: 16 stops, 14 one-tap with no failed read-back; modelled 42 s per caught harm against 420; one wrong category on a false stop, gone in the re-measure with `void`: 24/24, 15 stops, no wrong category, no failed read-back) | `benchmark/stop_card/`, [PREREG_REVIEW.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/stop_card/PREREG_REVIEW.md) | | Generate `openadapt.verify/v1` contracts from a MySQL, MariaDB or PostgreSQL schema, the kinds of write with their parameter names, and one demonstration write per example: one contract per field set (an update gets one for each set of fields it may change), and `choose` builds any other set of the kind's fields the same way from the same pinned inputs, list parameters by length, a `found` twin for a record already in place, the whole-database frame on every dialect (it compares the contract's own tables too, so an excused column holds only on the rows the write declares, and the recheck sees a late change to a look-alike row), rows the app adds on its own declared by count, a recheck window the person confirms, and `frame.versions` (a lost update) from a version column or the app's audit trail of saves. Equal ids are never taken for a join (a row's own key, the column a path arrived by), a join only equal ids support is a question when a supported place also fits, and a pin needs two rows that agree. Every undecided point is one numbered question; every contract is loaded and dry-read against the live store before it is written; a needed `TRIGGER` grant or view is applied with a flag or the kind is refused. Not yet: OpenAPI metadata | [#1672](https://github.com/OpenAdaptAI/OpenAdapt/pull/1672) | Real app (first study, instance copies; questions for a person: Dolibarr 4, Redmine 2, EspoCRM 3, against 11 hand decisions each; every in-scope planted fault stopped (22/22, 19/19, 21/21); correct writes confirmed 32/43, 32/38, 27/40; evidence in the private study repository) | `python -m openadapt_flow.generate.verify_contract`, [GENERATE_CONTRACTS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/GENERATE_CONTRACTS.md) | +| Set up generated contracts with no person answers: answer the generator's questions by fixed rules (its own bookkeeping proposal with time stamps checked against the clock, nothing added), validate them in a sandbox copy of the store beside the app's own (a database or schema the setup makes and drops) by replaying held-out correct writes and planting faults built from each write's recorded diff, never from the contract (missing, wrong value, wrong or look-alike record, duplicate, unrelated edit, a row in another table, a delayed change, a changed bookkeeping cell), and admit a kind only when every held-out write is confirmed and every fault stopped; a kind not admitted has no contract and is listed in `EXCEPTIONS.md`. Not yet: log tables, counters and derived columns, learned excuses, value maps | (this PR) | Unit (real PostgreSQL: one kind admitted with every planted fault stopped, one listed as an exception; a control without the clock check lets the time-stamp fault through). Development apps pending | `python -m openadapt_flow.generate.zero_touch`, [ZERO_TOUCH_DESIGN.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md) | | `void` in an ensure-record contract: values that make a record assert nothing (`status="entered-in-error"`). A record holding one is never the person, is not a duplicate, and a created one needs review; it names values to leave out, never values to keep, and can't name an identity field | On main ([#1657](https://github.com/OpenAdaptAI/OpenAdapt/pull/1657)) | Real app (Medplum 5.2.8: 8 live cases written in advance, 8/8 with the line, 5/8 and one silent pass without it; frozen blind Immunization suite 24/24); adversarial review recorded | `openadapt_flow/ensure_record.py`, [RESULTS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/fhir_medplum/void/results/RESULTS.md), [REVIEW_VOID.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/fhir_medplum/void/REVIEW_VOID.md) | | Check a patient registration on a commercial EHR through its public FHIR sandbox: Epic (open.epic, R4), read through FHIR search and read with SMART backend auth; the frame is the search set, since the sandbox has no AuditEvent or `_history` | On main ([#1656](https://github.com/OpenAdaptAI/OpenAdapt/pull/1656)) | Real app (Epic public sandbox, August 2026 release; blind round 12/13 with none passed as verified, the miss a false refusal fixed in the contract; caught Epic dropping the phone on a 201 create) | `benchmark/commercial_ehr/`, [FINDINGS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/commercial_ehr/FINDINGS.md) | | A public benchmark a vendor reruns with their own agent: `openadapt bench medplum --agent ` runs 37 written-for-it cases (Patient, AllergyIntolerance, Appointment) against a local Medplum through a fault proxy, and scores the agent's claim, a read-back and OpenAdapt's verifier against an end-state oracle | On main ([#1652](https://github.com/OpenAdaptAI/OpenAdapt/pull/1652)) | Real app (Medplum 5.2.8; Claude Sonnet 5.5: OpenAdapt missed 1 of 9 harms with 4 false stops, all corrections that changed the record's identity, against 8 missed by the claim and the read-back; cold run from clone to scorecard 5 min) | [README](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/fhir_medplum/public/README.md), [RESULTS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/fhir_medplum/public/RESULTS.md) | diff --git a/packages/flow/docs/GENERATE_CONTRACTS.md b/packages/flow/docs/GENERATE_CONTRACTS.md index 815b832b23..b745606461 100644 --- a/packages/flow/docs/GENERATE_CONTRACTS.md +++ b/packages/flow/docs/GENERATE_CONTRACTS.md @@ -351,6 +351,21 @@ version check, and `report.json` shows `versions: null`. If the app stops writing the trail (an agenda setting turned off), the agent's own save has no new version, and the verifier stops the write rather than pass it. +**Time stamps checked against the clock.** With `now_checks: true` in the +answers, a date-time column of the main record that the bookkeeping answer +leaves unchecked on a create (the write's own time stamp), or excuses on the +target of an update (its modified time), must hold a time near the read: no +older than the execution window plus the recheck window plus a minute, and +no more than five minutes ahead, by the database's local clock or by UTC. A +NULL fails. Without it, such a column may hold any value. It only adds a +check. + +**Setting up with no person answers.** `python -m +openadapt_flow.generate.zero_touch` answers these questions by fixed rules +and admits a kind only after a sandbox copy of the store confirms held-out +writes and stops planted faults; see +[research/ZERO_TOUCH_DESIGN.md](research/ZERO_TOUCH_DESIGN.md). + **The recheck window.** One question asks how long the app can still write on its own after a verified write. The answer is `frame.database.recheck_after_s`; the proposal is 300 seconds. diff --git a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md index 6ff4682612..dbf4926c40 100644 --- a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md +++ b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md @@ -79,18 +79,20 @@ schema + demonstrations ──► generator propose ──► questions Built only from the held-out write's recorded diff, the schema and the task's values. The planter never reads a contract. +Class names follow `openadapt_flow.fault_gate` where the class is the same. + | Class | What is planted | | --- | --- | -| `lost_write` | Nothing is written | +| `missing` | Nothing is written | | `wrong_value` | A cell holding a task value gets another value of its type (a key gets another existing row's key) | | `bookkeeping` | A cell the write changed that holds no task value gets a far value (a time stamp in 2001, a number plus 7, text plus " (x)") | | `wrong_record` | An update's changes land on another row of the same table | | `look_alike` | A create's values land on an existing row instead | | `duplicate` | Every added row is added twice | | `collateral_target` | A column of the target the write didn't change is changed | -| `collateral_row` | Another row of a table the write touched is changed | -| `extra_row` | A row is added to a table the write didn't touch | -| `late_write` | The write is correct; a collateral change lands after the check and before the recheck | +| `unrelated_edit` | Another row of a table the write touched is changed | +| `other_table` | A row is added to a table the write didn't touch | +| `delayed` | The write is correct; a change lands after the check and before the recheck | The `bookkeeping` class is what keeps excuses honest. Every cell the write changes is either a task value (checked), a value a rule checks (the acting @@ -119,6 +121,39 @@ The fault planter and the contract generator must not share blind spots An LLM fault generator can be added on top later, from a different model family and a separate prompt, but only above this floor [C12]. +## What this reuses, and what it doesn't + +**The fault gate** (`openadapt_flow/fault_gate.py`, #1445) runs a fixed +suite against a test copy and writes a coverage record that `verify-run` +reads under an `admission:` policy. Zero-touch uses its class names and its +two-sided rule (the clean save must pass, every fault must fail). It does +not call `run_gate`, for two reasons: + +- the gate plans its faults from the check's declared fields + (`_declared`), so it can't show that a check catches a change to a cell + the check doesn't declare. That is the blind spot an over-wide excuse + hides in [C5]; +- its test copy addresses one table slice by one key field; a generated + contract judges the whole database. + +A follow-up can let `run_gate` take a plan derived from a recorded diff and +write the same coverage record, so one admission format covers both. + +**Draft PR #1520 (`generate-parts`)** generated four workflow parts, each +admitted by its own check, then approved by a person. What carries over: + +| #1520 part | Here | +| --- | --- | +| The floor (`baseline.py`): generation adds checks, never removes one | Adopted as a rule: zero-touch only answers the generator's questions with its own proposal and adds `now_checks`; it never names a log or an extra column | +| Read channel from OpenAPI or a CapabilityStatement, admitted by a channel fault test (401, 500, a truncated or non-JSON body, a failed next page all read as unreadable) | The admission rule for read access to an app with no database account (REST, FHIR). Not needed for the SQL development apps; the code stays on the branch for that step. Its pagination fix already landed in #1538 | +| Record contract from diffs plus the fault gate | Superseded for SQL stores by the contract generator and this sandbox | +| Input types: a code table learned one entry per demonstrated value, any other value halts, admitted by a round trip | The admission rule for value mappings (`high` stored as 3): an exact table from the demonstrations, an unknown value stops, every held-out write must round-trip | +| Identity onto a reviewed template, with a perturbation suite where each slot has a case only it can catch | The model for record identity of people (patients, customers): the template is reviewed once per domain, never per app; the mapping is generated and admitted by perturbation | +| A person's approval bound to the manifest digest | Replaced by the exception report. Nothing is approved | + +#1520 itself is closed in favour of this work; its branch stays for the +channel and identity code when those steps are built. + ## What stays general No app-specific code or tests. Everything per app is data: diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index 5257b990f7..a2258fb4e6 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -13,9 +13,11 @@ 3. **Validation.** Held-out demonstrations (correct writes the generator never saw) are replayed in the sandbox, each between ``begin_write`` and ``verify_write``: every one must be ``CONFIRMED``, at the recheck too. - Then faults are planted from each held-out write: a wrong value, a wrong - or look-alike record, a duplicate, a collateral change, a lost write, a - late change, a changed bookkeeping cell. Every fault must be stopped. + Then faults are planted from each held-out write, named as in + :mod:`openadapt_flow.fault_gate` where the class is the same: a wrong + value, a wrong or look-alike record, a duplicate, an unrelated edit, a + row in another table, a missing write, a delayed change, a changed + bookkeeping cell. Every fault must be stopped. 4. **Admission.** A kind is admitted only when all of its held-out writes are confirmed and all of its faults are stopped. Anything else is not admitted: its tasks stop (``choose`` finds no contract), and @@ -24,7 +26,11 @@ The faults are built from the held-out write's recorded diff, the schema and the task's values only, never from the contracts under test, so the -planter doesn't share the generator's view of what matters. +planter doesn't share the generator's view of what matters. That is why +this doesn't call :func:`openadapt_flow.fault_gate.run_gate`: the gate plans +its faults from the check's declared fields, on one table slice, while a +whole-database contract must also be shown to catch changes to cells it +doesn't declare. The rules for answers: @@ -170,7 +176,9 @@ def create(self, schema: Schema, reader: str) -> None: cur.execute(f"CREATE DATABASE {self.q(self.name)}") src = self.q(self.source) for name in sorted(schema.tables): - cur.execute(f"CREATE TABLE {self.t(name)} LIKE {src}.{self.q(name)}") + cur.execute( + f"CREATE TABLE {self.t(name)} LIKE {src}.{self.q(name)}" + ) cur.execute( f"INSERT INTO {self.t(name)} SELECT * FROM {src}.{self.q(name)}" ) @@ -193,14 +201,18 @@ def create(self, schema: Schema, reader: str) -> None: f"INSERT INTO {self.t(name)} OVERRIDING SYSTEM VALUE " f"SELECT * FROM {src}.{self.q(name)}" ) - cur.execute(f"GRANT USAGE ON SCHEMA {self.q(self.name)} TO {self.q(reader)}") + cur.execute( + f"GRANT USAGE ON SCHEMA {self.q(self.name)} TO {self.q(reader)}" + ) cur.execute( f"GRANT SELECT ON ALL TABLES IN SCHEMA {self.q(self.name)} " f"TO {self.q(reader)}" ) finally: conn.close() - self.schema = store.read_schema(store.connector(self.read_config()), self.dialect) + self.schema = store.read_schema( + store.connector(self.read_config()), self.dialect + ) def drop(self) -> None: self._guard() @@ -226,9 +238,13 @@ def _table(self, name: str) -> Table: assert self.schema is not None return self.schema.tables[name] - def _where(self, table: Table, row: Mapping[str, Optional[str]]) -> tuple[str, list[Any]]: + def _where( + self, table: Table, row: Mapping[str, Optional[str]] + ) -> tuple[str, list[Any]]: if not table.primary_key: - raise ZeroTouchError(f"{table.name} has no primary key; rows can't be planted") + raise ZeroTouchError( + f"{table.name} has no primary key; rows can't be planted" + ) parts, args = [], [] for c in table.primary_key: v = row.get(c) @@ -238,7 +254,9 @@ def _where(self, table: Table, row: Mapping[str, Optional[str]]) -> tuple[str, l args.append(v) return " AND ".join(parts), args - def get(self, table: str, row: Mapping[str, Optional[str]]) -> Optional[dict[str, Optional[str]]]: + def get( + self, table: str, row: Mapping[str, Optional[str]] + ) -> Optional[dict[str, Optional[str]]]: t = self._table(table) where, args = self._where(t, row) cols = t.column_names @@ -270,7 +288,12 @@ def max_key(self, table: str) -> Optional[int]: got = self._exec(f"SELECT MAX({self.q(col.name)}) FROM {self.t(table)}") return int(got[0][0] or 0) - def put(self, table: str, row: Optional[Mapping[str, Optional[str]]], key: Mapping[str, Optional[str]]) -> None: + def put( + self, + table: str, + row: Optional[Mapping[str, Optional[str]]], + key: Mapping[str, Optional[str]], + ) -> None: """Make the row with ``key``'s primary key be ``row`` (None: absent).""" self._guard() @@ -329,7 +352,9 @@ def backward(diff_changes: Sequence[RowChange]) -> list[Op]: out = [] for ch in diff_changes: ref = ch.before if ch.before is not None else ch.after - out.append(Op(ch.table, dict(ref or {}), dict(ch.before) if ch.before else None)) + out.append( + Op(ch.table, dict(ref or {}), dict(ch.before) if ch.before else None) + ) return out @@ -345,7 +370,9 @@ def _parse_time(text: Optional[str]) -> Optional[_dt.datetime]: return None -def shift_times(schema: Schema, ops: list[Op], demo: Demo, values_text: set[str]) -> list[Op]: +def shift_times( + schema: Schema, ops: list[Op], demo: Demo, values_text: set[str] +) -> list[Op]: """Move the write's own time stamps to now. A date-time cell within a day of when the demonstration was made, and not a task's value, moves by the time since then, so a replay looks like a write made now (in @@ -367,7 +394,11 @@ def shift_times(schema: Schema, ops: list[Op], demo: Demo, values_text: set[str] if c.kind != "datetime" or c.name in t.primary_key: continue when = _parse_time(row.get(c.name)) - if when is None or not (lo <= when <= hi) or row.get(c.name) in values_text: + if ( + when is None + or not (lo <= when <= hi) + or _is_value(row.get(c.name), values_text) + ): continue row[c.name] = (when + delta).isoformat(sep=" ", timespec="seconds") out.append(Op(op.table, op.key, row)) @@ -452,7 +483,7 @@ def faults(schema: Schema, sb: Sandbox, demo: Demo, write: list[Op]) -> list[Cas task's values. The contracts are never read here.""" values = _text_values(demo.values) - out: list[Case] = [Case("lost", "lost_write", [])] + out: list[Case] = [Case("missing", "missing", [])] touched = {op.table for op in write} counts: dict[str, int] = {} @@ -480,7 +511,8 @@ def add(case: Case) -> None: new = None if c.name in fks: others = [ - r for r in sb.rows(fks[c.name].table, 20) + r + for r in sb.rows(fks[c.name].table, 20) if r.get(fks[c.name].ref_column) not in (None, cur) ] new = others[0].get(fks[c.name].ref_column) if others else None @@ -491,8 +523,13 @@ def add(case: Case) -> None: cls = "wrong_value" if _is_value(cur, values) else "bookkeeping" row = dict(op.row) row[c.name] = new - add(Case(f"{cls}:{op.table}.{c.name}", cls, - write[:i] + [Op(op.table, op.key, row)] + write[i + 1:])) + add( + Case( + f"{cls}:{op.table}.{c.name}", + cls, + write[:i] + [Op(op.table, op.key, row)] + write[i + 1 :], + ) + ) if not is_new and before is not None: # The write's changes landed on another row of the table. changed = {c for c in t.column_names if before.get(c) != op.row.get(c)} @@ -502,8 +539,13 @@ def add(case: Case) -> None: moved = dict(other) for c in changed: moved[c] = op.row.get(c) - add(Case(f"wrong_record:{op.table}", "wrong_record", - write[:i] + write[i + 1:] + [Op(op.table, other, moved)])) + add( + Case( + f"wrong_record:{op.table}", + "wrong_record", + write[:i] + write[i + 1 :] + [Op(op.table, other, moved)], + ) + ) break # A column the write didn't change, changed on the target. for c in t.columns: @@ -514,23 +556,38 @@ def add(case: Case) -> None: continue row = dict(op.row) row[c.name] = new - add(Case(f"collateral_target:{op.table}.{c.name}", "collateral_target", - write[:i] + [Op(op.table, op.key, row)] + write[i + 1:])) + add( + Case( + f"collateral_target:{op.table}.{c.name}", + "collateral_target", + write[:i] + [Op(op.table, op.key, row)] + write[i + 1 :], + ) + ) if is_new: top = sb.max_key(op.table) if top is not None: pk = t.primary_key[0] dup = dict(op.row) dup[pk] = str(top + 1000) - add(Case(f"duplicate:{op.table}", "duplicate", - write + [Op(op.table, {pk: dup[pk]}, dup)])) + add( + Case( + f"duplicate:{op.table}", + "duplicate", + write + [Op(op.table, {pk: dup[pk]}, dup)], + ) + ) # The values landed on an existing look-alike instead. for other in sb.rows(op.table, 20): look = dict(op.row) for c in t.primary_key: look[c] = other.get(c) - add(Case(f"look_alike:{op.table}", "look_alike", - write[:i] + write[i + 1:] + [Op(op.table, other, look)])) + add( + Case( + f"look_alike:{op.table}", + "look_alike", + write[:i] + write[i + 1 :] + [Op(op.table, other, look)], + ) + ) break # Another row of a touched table, and a new row in an untouched one. for name in sorted(touched): @@ -547,10 +604,21 @@ def add(case: Case) -> None: continue row = dict(other) row[c.name] = new - add(Case(f"collateral_row:{name}.{c.name}", "collateral_row", - write + [Op(name, other, row)])) - add(Case(f"late:{name}.{c.name}", "late_write", - list(write), late=[Op(name, other, row)])) + add( + Case( + f"unrelated_edit:{name}.{c.name}", + "unrelated_edit", + write + [Op(name, other, row)], + ) + ) + add( + Case( + f"delayed:{name}.{c.name}", + "delayed", + list(write), + late=[Op(name, other, row)], + ) + ) break break for name in sorted(set(schema.tables) - touched): @@ -561,8 +629,14 @@ def add(case: Case) -> None: row = dict(sample[0]) pk = schema.tables[name].primary_key[0] row[pk] = str(top + 1000) - add(Case(f"extra_row:{name}", "extra_row", write + [Op(name, {pk: row[pk]}, row)])) - if counts.get("extra_row", 0) >= 2: + add( + Case( + f"other_table:{name}", + "other_table", + write + [Op(name, {pk: row[pk]}, row)], + ) + ) + if counts.get("other_table", 0) >= 2: break return out @@ -598,6 +672,7 @@ def run_case( case: Case, actor_id: str, extra: Mapping[str, Any], + workdir: Optional[Path] = None, ) -> Outcome: """Bring the held-out write's rows back to before it, then run the case between ``begin_write`` and ``verify_write`` (and the recheck), then put @@ -607,7 +682,10 @@ def run_case( undo = backward(demo.diff.changes) all_ops = undo + case.write + case.late - saved: dict[tuple[str, str], tuple[dict[str, Optional[str]], Optional[dict[str, Optional[str]]]]] = {} + saved: dict[ + tuple[str, str], + tuple[dict[str, Optional[str]], Optional[dict[str, Optional[str]]]], + ] = {} assert sb.schema is not None for op in all_ops: k = (op.table, store.row_key(sb.schema.tables[op.table], op.key)) @@ -618,10 +696,12 @@ def run_case( try: contract, params = gen.choose(manifest, demo.kind, demo.values, extra) except gen.GenerateError as exc: - return Outcome(case.name, case.cls, case.correct, "NO_CONTRACT", False, [str(exc)]) + return Outcome( + case.name, case.cls, case.correct, "NO_CONTRACT", False, [str(exc)] + ) actor = Actor("agent", actor_id) pending = begin_write(contract, params, actor=actor, ledger=None, - write_key=f"zt-{time.time_ns()}") # fmt: skip + write_key=f"zt-{time.time_ns()}", workdir=workdir) # fmt: skip _put_all(sb, case.write) r = verify_write(pending) outcome = str(getattr(r.outcome, "value", r.outcome)) @@ -629,21 +709,30 @@ def run_case( passed = outcome == "CONFIRMED" if passed and getattr(r, "recheck", None) is not None: _put_all(sb, case.late) - rc = recheck(contract, r, params=params, actor=actor, wait=True, ledger=None) + rc = recheck( + contract, r, params=params, actor=actor, wait=True, ledger=None + ) rc_out = str(getattr(rc.outcome, "value", rc.outcome)) if rc_out != "CONFIRMED": passed, outcome = False, f"RECHECK_{rc_out}" reasons += list(rc.reasons) elif passed and case.late: - passed, outcome = False, "NO_RECHECK" - reasons.append("the contract schedules no recheck, so a late write can't be seen") - # A contract with no recheck can't stop a late write: count it as - # a pass of the fault, which fails validation. - passed = True + # No recheck is scheduled, so the late change is never read: the + # fault passes, which fails validation. + outcome = "CONFIRMED_NO_RECHECK" + reasons.append( + "the contract schedules no recheck, so a late write can't be seen" + ) return Outcome(case.name, case.cls, case.correct, outcome, passed, reasons[:6]) except Exception as exc: # noqa: BLE001 - a case that can't run admits nothing - return Outcome(case.name, case.cls, case.correct, "ERROR", False, - [f"{type(exc).__name__}: {exc}", traceback.format_exc(limit=3)]) + return Outcome( + case.name, + case.cls, + case.correct, + "ERROR", + False, + [f"{type(exc).__name__}: {exc}", traceback.format_exc(limit=3)], + ) finally: for (table, _), (key, row) in saved.items(): sb.put(table, row, key) @@ -652,37 +741,59 @@ def run_case( # -- answers ---------------------------------------------------------------------- -def auto_answers(report: Mapping[str, Any], picks: Mapping[str, Mapping[str, int]] = {}) -> tuple[dict[str, Any], list[dict[str, Any]]]: +def auto_answers( + report: Mapping[str, Any], picks: Mapping[str, Mapping[str, int]] = {} +) -> tuple[dict[str, Any], list[dict[str, Any]]]: """The answers this module gives, and a ledger saying by which rule.""" answers: dict[str, Any] = {"now_checks": True} ledger: list[dict[str, Any]] = [ - {"key": "now_checks", "answer": True, - "rule": "an unchecked or excused time stamp of the main record must hold a time near the read"}, + { + "key": "now_checks", + "answer": True, + "rule": "an unchecked or excused time stamp of the main record must hold a time near the read", + }, ] for q in report.get("questions") or []: key = str(q.get("key")) proposed = q.get("proposal") if key == "recheck_after_s": answers[key] = gen.DEFAULT_RECHECK_S - ledger.append({"key": key, "answer": gen.DEFAULT_RECHECK_S, "rule": "policy default"}) + ledger.append( + {"key": key, "answer": gen.DEFAULT_RECHECK_S, "rule": "policy default"} + ) elif key == "phones": answers[key] = list(proposed or []) - ledger.append({"key": key, "answer": answers[key], - "rule": "every listed parameter; wrong digits are planted faults"}) + ledger.append( + { + "key": key, + "answer": answers[key], + "rule": "every listed parameter; wrong digits are planted faults", + } + ) elif key == "bookkeeping": prop = dict(proposed or {}) if isinstance(proposed, Mapping) else {} ans = {"accept": True} if prop.get("unchecked"): ans["unchecked"] = prop["unchecked"] answers[key] = ans - ledger.append({"key": key, "answer": ans, - "rule": "the generator's own proposal; nothing added (no logs_add, no columns_add)"}) + ledger.append( + { + "key": key, + "answer": ans, + "rule": "the generator's own proposal; nothing added (no logs_add, no columns_add)", + } + ) elif key.startswith("locations."): if key in picks: answers[key] = dict(picks[key]) - ledger.append({"key": key, "answer": answers[key], - "rule": "the one candidate whose kind validated"}) + ledger.append( + { + "key": key, + "answer": answers[key], + "rule": "the one candidate whose kind validated", + } + ) return answers, ledger @@ -694,7 +805,9 @@ def location_options(report: Mapping[str, Any]) -> dict[str, dict[str, int]]: key = str(q.get("key")) if not key.startswith("locations."): continue - for name, opts in re.findall(r"`([^`]+)` is in several places: (.*?)(?= `|$)", str(q.get("text"))): + for name, opts in re.findall( + r"`([^`]+)` is in several places: (.*?)(?= `|$)", str(q.get("text")) + ): out.setdefault(key, {})[name] = len(re.findall(r"\[\d+\]", opts)) return out @@ -713,6 +826,7 @@ def validate( actor_id: str, extra: Mapping[str, Any], log: Callable[[str], None] = lambda s: None, + record: Optional[list[dict[str, Any]]] = None, ) -> dict[str, list[Outcome]]: """Every held-out write, replayed and faulted, by kind.""" @@ -732,13 +846,21 @@ def validate( for op, row in saved: sb.put(op.table, row, op.key) for case in cases: - res = run_case(sb, manifest, demo, case, actor_id, extra) + res = run_case( + sb, manifest, demo, case, actor_id, extra, manifest.parent / "state" + ) log(f"{demo.name} {case.name}: {res.outcome} {'ok' if res.ok else 'FAIL'}") + if record is not None: + record.append({"demo": demo.name, "kind": demo.kind, "case": res.case, + "class": res.cls, "correct": res.correct, "outcome": res.outcome, + "ok": res.ok, "reasons": [r[:300] for r in res.reasons[:3]]}) # fmt: skip by_kind.setdefault(demo.kind, []).append(res) return by_kind -def summarize(by_kind: Mapping[str, Sequence[Outcome]], kinds: Sequence[str]) -> dict[str, Any]: +def summarize( + by_kind: Mapping[str, Sequence[Outcome]], kinds: Sequence[str] +) -> dict[str, Any]: out: dict[str, Any] = {} for k in kinds: rs = list(by_kind.get(k) or []) @@ -747,7 +869,9 @@ def summarize(by_kind: Mapping[str, Sequence[Outcome]], kinds: Sequence[str]) -> out[k] = { "admitted": bool(correct) and all(r.ok for r in rs), "correct_writes": len(correct), - "false_stops": [f"{r.outcome}: {'; '.join(r.reasons[:2])}" for r in correct if not r.ok], + "false_stops": [ + f"{r.outcome}: {'; '.join(r.reasons[:2])}" for r in correct if not r.ok + ], "faults": len(fl), "faults_stopped": sum(1 for r in fl if r.ok), "faults_passed": [r.case for r in fl if not r.ok], @@ -762,9 +886,13 @@ def summarize(by_kind: Mapping[str, Sequence[Outcome]], kinds: Sequence[str]) -> def exceptions_md(summary: Mapping[str, Any], report: Mapping[str, Any]) -> str: - lines = ["# What setup could not validate", "", - "No person decision was asked for. The kinds below are not admitted, so", - "their tasks stop (no contract). Each line says why.", ""] + lines = [ + "# What setup could not validate", + "", + "No person decision was asked for. The kinds below are not admitted, so", + "their tasks stop (no contract). Each line says why.", + "", + ] n = 0 for k, s in summary.items(): if s["admitted"]: @@ -823,23 +951,34 @@ def onboard( for picks in trials: answers, ledger = auto_answers(report, picks) sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} - tag = "base" if not picks else "pick-" + "-".join( - f"{k}.{p}.{i}" for k, d in picks.items() for p, i in d.items()) + tag = ( + "base" + if not picks + else "pick-" + + "-".join( + f"{k}.{p}.{i}" for k, d in picks.items() for p, i in d.items() + ) + ) log(f"propose ({tag})") rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / tag, actor_params=actor_params, answers=sans) # fmt: skip + cases: list[dict[str, Any]] = [] by_kind = validate(sb, work / tag / "manifest.yaml", holdout, actor_id, - dict(actor_params), log) # fmt: skip + dict(actor_params), log, cases) # fmt: skip summary = summarize(by_kind, kind_names) results.append((picks, answers, summary)) - best = best or {"answers": answers, "ledger": ledger, "summary": summary, "report": rep} + best = best or {"answers": answers, "ledger": ledger, "summary": summary, + "report": rep, "cases": cases} # fmt: skip # A location pick is kept when exactly one candidate admits its kind. final_picks: dict[str, dict[str, int]] = {} for key, params in options.items(): kind = key.split(".", 1)[1] for name in params: - ok = [p[key][name] for p, _, s in results - if key in p and name in p[key] and s[kind]["admitted"]] + ok = [ + p[key][name] + for p, _, s in results + if key in p and name in p[key] and s[kind]["admitted"] + ] if len(ok) == 1: final_picks.setdefault(key, {})[name] = ok[0] answers, ledger = auto_answers(report, final_picks) @@ -847,10 +986,11 @@ def onboard( sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / "final", actor_params=actor_params, answers=sans) # fmt: skip + cases = [] summary = summarize(validate(sb, work / "final" / "manifest.yaml", holdout, - actor_id, dict(actor_params), log), kind_names) # fmt: skip + actor_id, dict(actor_params), log, cases), kind_names) # fmt: skip else: - summary, rep = best["summary"], best["report"] + summary, rep, cases = best["summary"], best["report"], best["cases"] finally: if not keep_sandbox: sb.drop() @@ -876,6 +1016,7 @@ def onboard( } (out / "ONBOARDING.json").write_text(json.dumps(result, indent=2, default=str)) (out / "EXCEPTIONS.md").write_text(exceptions_md(summary, prod)) + (out / "VALIDATION.jsonl").write_text("".join(json.dumps(c) + "\n" for c in cases)) return result @@ -886,18 +1027,38 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "generator's questions by rule, validate them in a sandbox copy with " "held-out writes and planted faults, and admit only what validates.", ) - ap.add_argument("--read", required=True, type=Path, help="read-only account (read.yaml)") - ap.add_argument("--admin", required=True, type=Path, - help="admin account that may create the sandbox beside the app's store") - ap.add_argument("--reader", required=True, - help="the read account to grant on the sandbox: MySQL 'user'@'host', PostgreSQL role") + ap.add_argument( + "--read", required=True, type=Path, help="read-only account (read.yaml)" + ) + ap.add_argument( + "--admin", + required=True, + type=Path, + help="admin account that may create the sandbox beside the app's store", + ) + ap.add_argument( + "--reader", + required=True, + help="the read account to grant on the sandbox: MySQL 'user'@'host', PostgreSQL role", + ) ap.add_argument("--kinds", required=True, type=Path) - ap.add_argument("--demos", required=True, type=Path, help="demonstrations the generator learns from") - ap.add_argument("--holdout", required=True, type=Path, - help="other correct writes, recorded the same way, used only to validate") + ap.add_argument( + "--demos", + required=True, + type=Path, + help="demonstrations the generator learns from", + ) + ap.add_argument( + "--holdout", + required=True, + type=Path, + help="other correct writes, recorded the same way, used only to validate", + ) ap.add_argument("--out", required=True, type=Path) ap.add_argument("--actor-param", action="append", default=[], help="NAME=VALUE") - ap.add_argument("--actor-id", required=True, help="the agent's user id as the verifier's Actor") + ap.add_argument( + "--actor-id", required=True, help="the agent's user id as the verifier's Actor" + ) ap.add_argument("--keep-sandbox", action="store_true") args = ap.parse_args(argv) result = onboard( @@ -906,7 +1067,14 @@ def main(argv: Optional[Sequence[str]] = None) -> int: actor_params=gen._parse_actor(args.actor_param), actor_id=args.actor_id, keep_sandbox=args.keep_sandbox, ) # fmt: skip - print(json.dumps({k: result[k] for k in ("minutes", "person_decisions", "admitted", "exceptions")})) + print( + json.dumps( + { + k: result[k] + for k in ("minutes", "person_decisions", "admitted", "exceptions") + } + ) + ) return 0 diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py index 4a711f270b..5a211de027 100644 --- a/packages/flow/tests/test_zero_touch.py +++ b/packages/flow/tests/test_zero_touch.py @@ -44,14 +44,18 @@ def _digest(app: App) -> str: "SELECT table_name FROM information_schema.tables " "WHERE table_schema='public' ORDER BY table_name" ): - h.update(json.dumps(app.q(f"SELECT * FROM {t} ORDER BY 1"), default=str).encode()) + h.update( + json.dumps(app.q(f"SELECT * FROM {t} ORDER BY 1"), default=str).encode() + ) return h.hexdigest() @pytest.fixture(scope="module") def onboarded(pg: Any) -> dict[str, Any]: # noqa: F811 root, port = pg["root"], pg["port"] - admin = psycopg2.connect(host="127.0.0.1", port=port, user="admin", dbname="postgres") + admin = psycopg2.connect( + host="127.0.0.1", port=port, user="admin", dbname="postgres" + ) admin.autocommit = True admin.cursor().execute("CREATE DATABASE tracker_zt") admin.close() @@ -59,7 +63,11 @@ def onboarded(pg: Any) -> dict[str, Any]: # noqa: F811 with app.con() as c, c.cursor() as cur: cur.execute(SCHEMA) cur.execute("UPDATE issues SET lock_version = 4") - for seq in ("journals_id_seq", "journal_details_id_seq", "custom_values_id_seq"): + for seq in ( + "journals_id_seq", + "journal_details_id_seq", + "custom_values_id_seq", + ): cur.execute(f"SELECT setval('{seq}', 100)") os.environ["OA_ZT_TEST_PW"] = "reader-pw" work = root / "zt" @@ -75,14 +83,21 @@ def onboarded(pg: Any) -> dict[str, Any]: # noqa: F811 (work / "kinds.yaml").write_text(yaml.safe_dump(KINDS, allow_unicode=True)) common = ["--read", str(work / "read.yaml")] - def demo(folder: str, kind: str, values: dict[str, Any], write: Callable[[], Any]) -> None: + def demo( + folder: str, kind: str, values: dict[str, Any], write: Callable[[], Any] + ) -> None: assert gen.main(["snapshot", *common, "--out", str(work / "before.json")]) == 0 write() assert gen.main(["record", *common, "--before", str(work / "before.json"), "--kind", kind, "--values", json.dumps(values), "--demos", str(work / folder)]) == 0 # fmt: skip for ex in KINDS["kinds"][0]["examples"]: - demo("demos", "create_issue", ex["values"], lambda v=ex["values"]: app.create_issue(v)) + demo( + "demos", + "create_issue", + ex["values"], + lambda v=ex["values"]: app.create_issue(v), + ) for ex in KINDS["kinds"][1]["examples"]: v = ex["values"] demo("demos", "update_issue", v, @@ -98,8 +113,31 @@ def demo(folder: str, kind: str, values: dict[str, Any], write: Callable[[], Any read=read, admin=admin_read, reader="oa_reader", kinds_path=work / "kinds.yaml", demo_dir=work / "demos", holdout_dir=work / "holdout", out=work / "out", actor_params={"bot_user_id": str(BOT)}, actor_id="ops-bot", log=lines.append, + keep_sandbox=True, ) # fmt: skip - return {"app": app, "result": result, "before": before, "out": work / "out", "log": lines} + # The control: the same answers without the time check, in the same + # sandbox. The planted time-stamp fault must then pass. + sb = zt.Sandbox(admin_read, read) + sb.schema = zt.store.read_schema(zt.store.connector(sb.read_config()), "postgresql") + try: + answers = {**result["answers"], "recheck_after_s": zt.SANDBOX_RECHECK_S} + answers.pop("now_checks") + gen.propose(read=sb.read_config(), kinds=gen.load_kinds(work / "kinds.yaml"), + demo_dir=work / "demos", out=work / "control", + actor_params={"bot_user_id": str(BOT)}, answers=answers) # fmt: skip + control: list[dict[str, Any]] = [] + zt.validate(sb, work / "control" / "manifest.yaml", + [d for d in gen.load_demos(work / "holdout") if d.kind == "create_issue"], + "ops-bot", {"bot_user_id": str(BOT)}, record=control) # fmt: skip + finally: + sb.drop() + if os.environ.get("OA_ZT_TEST_KEEP"): + import shutil + + shutil.copytree(work, os.environ["OA_ZT_TEST_KEEP"], dirs_exist_ok=True) + (Path(os.environ["OA_ZT_TEST_KEEP"]) / "log.txt").write_text("\n".join(lines)) + return {"app": app, "result": result, "before": before, "out": work / "out", "log": lines, + "control": control} # fmt: skip def test_no_person_decision_and_a_ledger(onboarded: dict[str, Any]) -> None: @@ -115,7 +153,12 @@ def test_no_person_decision_and_a_ledger(onboarded: dict[str, Any]) -> None: def test_the_apps_own_tables_did_not_change(onboarded: dict[str, Any]) -> None: assert _digest(onboarded["app"]) == onboarded["before"] - schemas = {s for (s,) in onboarded["app"].q("SELECT schema_name FROM information_schema.schemata")} + schemas = { + s + for (s,) in onboarded["app"].q( + "SELECT schema_name FROM information_schema.schemata" + ) + } assert zt.PG_SANDBOX_SCHEMA not in schemas @@ -131,17 +174,23 @@ def test_an_admitted_kind_confirmed_every_held_out_write_and_stopped_every_fault assert s["false_stops"] or s["faults_passed"] or s.get("why"), (name, s) -def test_create_issue_is_admitted_with_every_fault_class(onboarded: dict[str, Any]) -> None: +def test_create_issue_is_admitted_with_every_fault_class( + onboarded: dict[str, Any], +) -> None: s = onboarded["result"]["kinds"]["create_issue"] assert s["admitted"], (s, "\n".join(onboarded["log"][-60:])) - for cls in ("lost_write", "wrong_value", "duplicate", "look_alike", "collateral_row", - "late_write", "extra_row", "bookkeeping"): # fmt: skip + for cls in ("missing", "wrong_value", "duplicate", "look_alike", "unrelated_edit", + "delayed", "other_table", "bookkeeping"): # fmt: skip assert cls in s["by_class"], s["by_class"] -def test_a_kind_not_admitted_has_no_production_contract(onboarded: dict[str, Any]) -> None: +def test_a_kind_not_admitted_has_no_production_contract( + onboarded: dict[str, Any], +) -> None: r = onboarded["result"] - manifest = yaml.safe_load((onboarded["out"] / "contracts" / "manifest.yaml").read_text()) + manifest = yaml.safe_load( + (onboarded["out"] / "contracts" / "manifest.yaml").read_text() + ) assert sorted(manifest["kinds"]) == r["admitted"] for name, s in r["kinds"].items(): if not s["admitted"]: @@ -153,8 +202,34 @@ def test_a_kind_not_admitted_has_no_production_contract(onboarded: dict[str, Any def test_a_planted_time_stamp_fails_the_now_check() -> None: from openadapt_flow.generate.sql_store import Column, Schema, Table - schema = Schema("mysql", "d", {"t": Table("t", [Column("at", "datetime", True, None)], [])}) + schema = Schema( + "mysql", "d", {"t": Table("t", [Column("at", "datetime", True, None)], [])} + ) b = gen.SqlBuilder(schema) sql = b.now("t", "at", 1260) assert "NOW() - INTERVAL 1260 SECOND" in sql and "UTC_TIMESTAMP()" in sql assert zt.mutate(schema.tables["t"], "at", "2026-11-03 10:00:00") == zt.FAR_TIME + + +def test_the_time_stamp_fault_is_caught_by_the_time_check( + onboarded: dict[str, Any], +) -> None: + rows = [ + json.loads(x) + for x in (onboarded["out"] / "VALIDATION.jsonl").read_text().splitlines() + ] + hit = [r for r in rows if r["case"] == "bookkeeping:issues.created_on"] + assert hit and all(not r["correct"] and r["ok"] for r in hit), hit + # The main record no longer matches its contract (the time check is one + # of its columns); the control below shows the same fault passes + # without the time check. + assert any("record_written was refuted" in " ".join(r["reasons"]) for r in hit), hit + + +def test_without_the_time_check_the_time_stamp_fault_passes( + onboarded: dict[str, Any], +) -> None: + hit = [ + r for r in onboarded["control"] if r["case"] == "bookkeeping:issues.created_on" + ] + assert hit and not any(r["ok"] for r in hit), hit From 00b6503da5e609c03086f81e0d5e346581e7189a Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 15:33:20 -0500 Subject: [PATCH 03/12] Zero-touch gate: fail closed on unplanted faults, fault every column, require core classes, strict pins, production equals validated --- .../flow/docs/research/ZERO_TOUCH_DESIGN.md | 45 +- .../openadapt_flow/generate/demo_analysis.py | 19 +- .../generate/verify_contract.py | 62 +- .../openadapt_flow/generate/zero_touch.py | 636 +++++++++++++----- packages/flow/tests/test_zero_touch.py | 179 +++++ 5 files changed, 737 insertions(+), 204 deletions(-) diff --git a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md index dbf4926c40..f0dc148310 100644 --- a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md +++ b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md @@ -84,15 +84,46 @@ Class names follow `openadapt_flow.fault_gate` where the class is the same. | Class | What is planted | | --- | --- | | `missing` | Nothing is written | -| `wrong_value` | A cell holding a task value gets another value of its type (a key gets another existing row's key) | -| `bookkeeping` | A cell the write changed that holds no task value gets a far value (a time stamp in 2001, a number plus 7, text plus " (x)") | -| `wrong_record` | An update's changes land on another row of the same table | -| `look_alike` | A create's values land on an existing row instead | -| `duplicate` | Every added row is added twice | +| `partial` | One row of a multi-row write is lost (a child row, a journal entry) | +| `wrong_value` | A cell holding a task value gets another value | +| `bookkeeping` | A cell holding no task value gets another value, NULL cells included | | `collateral_target` | A column of the target the write didn't change is changed | -| `unrelated_edit` | Another row of a table the write touched is changed | +| `wrong_record` | An update's changes land on another row of the same table (a copy of the target when the table has no other row) | +| `look_alike` | A create's values land on an existing row instead | +| `duplicate` | Each added row is added again under a new key | +| `unrelated_edit` | Each column the update changed is changed the same way on another row; and one other row of each touched table is edited | | `other_table` | A row is added to a table the write didn't touch | -| `delayed` | The write is correct; a change lands after the check and before the recheck | +| `delayed` | The write is correct; a change to the target, or to another row, lands after the check and before the recheck | + +**Every column, no cap.** Every column of every row the write adds or +changes gets its own fault. The value planted is, in order: another row's +key for a declared foreign key; a value the column already holds in another +row (so the store accepts it); or a value made by the column's type (a date +in 2001, a number plus one, text of the same length with its last character +changed, valid JSON, a new UUID, a flipped byte, another time). A column +whose type has no generator gets an `unplantable` case. + +**Only a verdict counts.** A fault counts as stopped only when the verifier +read the planted state and returned `REFUTED` or `UNCERTAIN` (at the check +or the recheck). A plant the store rejects, a case that errors, a fault +with no contract and an unplantable column all count as not stopped, so the +kind is not admitted. + +**Required classes.** A kind is admitted only if its held-out writes +produced `missing`, `unrelated_edit`, `delayed` and `other_table` faults, +plus `duplicate` and `look_alike` for a create and `wrong_record` for an +update. A class that couldn't be planted is listed in `EXCEPTIONS.md`. + +**Held out means held out.** A held-out write with the same kind and values +as a demonstration, or the same diff, is refused, and so is a kind with no +held-out write. + +**Production equals validated.** The sandbox copies the declared foreign +keys, because the generator reads them. The production contracts are +proposed again against the app's own store; a kind is kept only if its +contracts and manifest entry match the validated ones once connections and +the recheck window are set aside. The manifest is filtered (entries, pinned +digests and files) in a staging folder and moved into place in one rename. The `bookkeeping` class is what keeps excuses honest. Every cell the write changes is either a task value (checked), a value a rule checks (the acting diff --git a/packages/flow/openadapt_flow/generate/demo_analysis.py b/packages/flow/openadapt_flow/generate/demo_analysis.py index db250ea0bd..67d4d2268c 100644 --- a/packages/flow/openadapt_flow/generate/demo_analysis.py +++ b/packages/flow/openadapt_flow/generate/demo_analysis.py @@ -672,13 +672,14 @@ def column_roles( if copy is not None else "a link to a row the write added" ) - out[name] = ColumnRole( - "varies", - detail={ - "why": f"the acting user's id or {other}: the " - "demonstrations can't tell" - }, - ) + detail: dict[str, Any] = { + "why": f"the acting user's id or {other}: the " + "demonstrations can't tell" + } + if copy is not None: + # Both fit every row: ``strict_pins`` checks both. + detail["strict"] = {"actor": actor[0], "copied": copy} + out[name] = ColumnRole("varies", detail=detail) continue out[name] = ColumnRole("actor", detail={"param": actor[0]}) continue @@ -706,7 +707,9 @@ def column_roles( "varies", detail={ "why": f"a constant or a copy of {copied['table']}." - f"{copied['column']}: the demonstrations can't tell" + f"{copied['column']}: the demonstrations can't tell", + # Both fit every row: ``strict_pins`` checks both. + "strict": {"constant": values[0], "copied": copied}, }, ) continue diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index c75c9769ab..3138a9284a 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -89,6 +89,41 @@ def _computed(role: ColumnRole) -> bool: return why.startswith("the ") and "of a date the task gives" in why +def _strict(roles: Mapping[str, ColumnRole], on: bool) -> dict[str, ColumnRole]: + """``roles`` with each column two readings fit made a ``strict`` role + (checked both ways) when ``strict_pins`` is on; else unchanged.""" + + if not on: + return dict(roles) + return { + c: ColumnRole("strict", detail=dict(r.detail["strict"])) + if r.role == "varies" and r.detail.get("strict") + else r + for c, r in roles.items() + } + + +def _strict_pins( + b: "SqlBuilder", surf: "Surface", table: str, col: str, detail: Mapping[str, Any], + match: dict[str, Any], +) -> None: # fmt: skip + """Both checks of a ``strict`` role: the constant (or the acting user), + and the copy of the column its key points to.""" + + if "constant" in detail: + alias = _add_column( + surf, f"pin__{_alias(col)}", b.pin("t", table, col, detail["constant"]) + ) + match[alias] = {"literal": "1"} + if "actor" in detail: + match.setdefault(col, {"param": detail["actor"]}) + if detail.get("copied"): + alias = _add_column( + surf, f"copy__{_alias(col)}", b.copied("t", detail["copied"], table, col) + ) + match[alias] = {"literal": "1"} + + class GenerateError(RuntimeError): """The generator can't go on; the message says why.""" @@ -1863,6 +1898,11 @@ class Settings: #: target's excused bookkeeping on an update) must hold a time near the #: read, instead of any value. now_checks: bool = False + #: ``strict_pins`` answered true: a new-row column two readings fit in + #: every demonstration (a constant and a copy of a column a key points + #: to, or the acting user and such a copy) is checked both ways, instead + #: of refusing the kind. + strict_pins: bool = False @property def now_window_s(self) -> int: @@ -1966,7 +2006,10 @@ def build_contract( # ``found`` pins the same constants as ``create`` (a soft-delete # flag, a status, a type): a trashed record with the task's values # is not "already there". Who created it may differ. - roles = _anchor_roles(schema, plan, variant, settings.actor_params) + roles = _strict( + _anchor_roles(schema, plan, variant, settings.actor_params), + settings.strict_pins, + ) for col, role in sorted(roles.items()): if flavor == "found" and role.role in ("actor", "copied"): continue @@ -2098,7 +2141,7 @@ def build_contract( m[key] = {"literal": "1"} else: m[key] = _value_expr(pname, loc.rule) - for col, role in sorted(g.roles.items()): + for col, role in sorted(_strict(g.roles, settings.strict_pins).items()): if role.role == "link" or (found and role.role in ("actor", "copied")): continue _pin( @@ -2201,7 +2244,8 @@ def build_contract( if found or (side.unless is not None and side.unless in fields): continue m = {} - for col, role in sorted(side.roles.items()): + side_roles = _strict(side.roles, settings.strict_pins) + for col, role in sorted(side_roles.items()): if role.role == "constant" and role.value is not None: m[col] = {"literal": role.value} elif role.role == "actor": @@ -2215,7 +2259,7 @@ def build_contract( side.table, m, side.count, - side.roles, + side_roles, f"{side.count} row(s) the app adds to {side.table}", ) ) @@ -2331,6 +2375,9 @@ def _pin( actor_params: Mapping[str, str], unchecked: Mapping[str, set[str]] = {}, ) -> None: + if role.role == "strict": + _strict_pins(b, surf, table, col, role.detail, match) + return if ( role.role == "varies" and col not in unchecked.get(table, set()) @@ -2435,6 +2482,10 @@ def _raw_effect( role = roles.get(c.name) if role is not None and role.role == "constant" and role.value is None: continue + if role is not None and role.role == "strict": + _strict_pins(b, surf, table, c.name, role.detail, m) + nrf.add(c.name) + continue if ( role is None or role.role != "constant" @@ -3253,6 +3304,8 @@ def table_cols(key: str) -> dict[str, set[str]]: (g.table, g.roles) for g in [*p.children.values(), *p.lists.values()] ] groups += [(sd.table, sd.roles) for sd in p.sides] + strict_on = _answer(answers, "strict_pins") is True + groups = [(t, _strict(r, strict_on)) for t, r in groups] for table, group_roles in groups: for col, role in group_roles.items(): if role.role in OPEN_ROLES: @@ -3428,6 +3481,7 @@ def table_cols(key: str) -> dict[str, set[str]]: env, unchecked=unchecked_named, now_checks=_answer(answers, "now_checks") is True, + strict_pins=_answer(answers, "strict_pins") is True, ) # Tables whose name the SQL guard refuses: a view, or a refusal. diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index a2258fb4e6..d0e8cb257c 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -82,8 +82,6 @@ SANDBOX_RECHECK_S = 3 #: A planted bookkeeping time: far from any write's own time. FAR_TIME = "2001-02-03 04:05:06" -#: At most this many faults of one class per held-out write. -MAX_PER_CLASS = 12 class ZeroTouchError(RuntimeError): @@ -114,6 +112,9 @@ def __init__(self, admin: Mapping[str, Any], read: Mapping[str, Any]) -> None: else PG_SANDBOX_SCHEMA ) self.schema: Optional[Schema] = None + #: Foreign keys the sandbox couldn't copy. Contracts built without + #: them differ from production ones, and are then not admitted. + self.fk_errors: list[str] = [] # Connections -------------------------------------------------------------------- @@ -182,6 +183,19 @@ def create(self, schema: Schema, reader: str) -> None: cur.execute( f"INSERT INTO {self.t(name)} SELECT * FROM {src}.{self.q(name)}" ) + # Declared foreign keys too: the generator reads them, so the + # sandbox's contracts must see the same schema as the app's. + cur.execute("SET FOREIGN_KEY_CHECKS=0") + for name, t in sorted(schema.tables.items()): + for fk in t.foreign_keys: + try: + cur.execute( + f"ALTER TABLE {self.t(name)} ADD FOREIGN KEY " + f"({self.q(fk.column)}) REFERENCES " + f"{self.t(fk.table)} ({self.q(fk.ref_column)})" + ) + except Exception as exc: # noqa: BLE001 - recorded + self.fk_errors.append(f"{name}.{fk.column}: {exc}") user, _, host = reader.partition("@") cur.execute( f"GRANT SELECT, SHOW VIEW, TRIGGER ON {self.q(self.name)}.* " @@ -201,6 +215,16 @@ def create(self, schema: Schema, reader: str) -> None: f"INSERT INTO {self.t(name)} OVERRIDING SYSTEM VALUE " f"SELECT * FROM {src}.{self.q(name)}" ) + for name, t in sorted(schema.tables.items()): + for fk in t.foreign_keys: + try: + cur.execute( + f"ALTER TABLE {self.t(name)} ADD FOREIGN KEY " + f"({self.q(fk.column)}) REFERENCES " + f"{self.t(fk.table)} ({self.q(fk.ref_column)}) NOT VALID" + ) + except Exception as exc: # noqa: BLE001 - recorded + self.fk_errors.append(f"{name}.{fk.column}: {exc}") cur.execute( f"GRANT USAGE ON SCHEMA {self.q(self.name)} TO {self.q(reader)}" ) @@ -278,6 +302,15 @@ def rows(self, table: str, limit: int = 50) -> list[dict[str, Optional[str]]]: ) return [{c: store.cell_text(v) for c, v in zip(cols, r)} for r in got] + def distinct(self, table: str, column: str, limit: int = 5) -> list[str]: + """Up to ``limit`` values the column holds, as text.""" + + got = self._exec( + f"SELECT DISTINCT {self.q(column)} FROM {self.t(table)} " + f"WHERE {self.q(column)} IS NOT NULL LIMIT {int(limit)}" + ) + return [v for v in (store.cell_text(r[0]) for r in got) if v is not None] + def max_key(self, table: str) -> Optional[int]: t = self._table(table) if len(t.primary_key) != 1: @@ -302,6 +335,15 @@ def put( conn = self._connect() try: cur = conn.cursor() + # A planted fault may break a foreign key on purpose (a row the + # write points to is gone): the store must take it. + if self.dialect == "mysql": + cur.execute("SET FOREIGN_KEY_CHECKS=0") + else: + try: # a superuser only; else a key-breaking fault fails closed + cur.execute("SET session_replication_role = replica") + except Exception: # noqa: BLE001 + pass cur.execute(f"DELETE FROM {self.t(table)} WHERE {where}", tuple(args)) if row is not None: cols = [c for c in t.column_names if c in row] @@ -412,24 +454,53 @@ def shift_times( class Case: """One validation run on a held-out write. ``write`` is put after ``begin_write``; ``late`` after a ``CONFIRMED`` and before the recheck. - ``correct``: the run must be confirmed; else it must be stopped.""" + ``correct``: the run must be confirmed; else it must be stopped. + ``unplantable``: why no fault could be planted here (the case fails).""" name: str cls: str write: list[Op] late: list[Op] = field(default_factory=list) correct: bool = False + unplantable: str = "" + + +#: Fault classes every kind's held-out writes must have produced before the +#: kind can be admitted. A create also needs ``CREATE_CLASSES``, an update +#: ``UPDATE_CLASSES``. +REQUIRED_CLASSES = ("missing", "unrelated_edit", "delayed", "other_table") +CREATE_CLASSES = ("duplicate", "look_alike") +UPDATE_CLASSES = ("wrong_record",) + +_UUID = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def _fresh_uuid() -> str: + import uuid + + return str(uuid.uuid4()) def mutate(table: Table, column: str, text: Optional[str]) -> Optional[str]: - """Another value of the column's type, plainly different.""" + """Another value of the column's type, plainly different, or None when + the type has no generator. Text keeps its length (a length limit or a + fixed-width column still takes it).""" col = table.column(column) kind = col.kind if col else "text" + dtype = (col.data_type if col else "").lower() + if dtype in ("json", "jsonb"): + return '{"oa_fault": 1}' if text != '{"oa_fault": 1}' else '{"oa_fault": 2}' + if dtype == "uuid" or (text is not None and _UUID.match(text)): + return _fresh_uuid() if kind == "datetime": return FAR_TIME if text != FAR_TIME else "2002-03-04 05:06:07" if kind == "date": return "2001-02-03" if text != "2001-02-03" else "2002-03-04" + if kind == "time": + return "01:02:03" if text != "01:02:03" else "02:03:04" if kind == "bool": return "false" if text in ("true", "1") else "true" if kind == "number": @@ -437,12 +508,38 @@ def mutate(table: Table, column: str, text: Optional[str]) -> Optional[str]: n = float(text) if text is not None else 0.0 except ValueError: return None - return str(int(n) + 7) if n.is_integer() else str(n + 7) + return str(int(n) + 1) if n.is_integer() else str(n + 1) + if kind == "binary": + if text and text.startswith("0x") and len(text) > 2: + last = int(text[-2:], 16) ^ 0x01 + return text[:-2] + f"{last:02x}" + return "0x01" if kind == "text": - return (text or "") + " (x)" + if not text: + return "x" + return text[:-1] + ("x" if text[-1] != "x" else "y") return None +def plant_value( + sb: "Sandbox", table: Table, column: str, current: Optional[str] +) -> Optional[str]: + """A value to plant in ``table.column`` instead of ``current``: another + row's key for a declared foreign key, else a value the column already + holds in another row (the store accepts it), else :func:`mutate`.""" + + fk = next((f for f in table.foreign_keys if f.column == column), None) + if fk is not None: + for r in sb.rows(fk.table, 20): + v = r.get(fk.ref_column) + if v is not None and v != current: + return v + for v in sb.distinct(table.name, column, 5): + if v is not None and v != current: + return v + return mutate(table, column, current) + + def _text_values(values: Mapping[str, Any]) -> set[str]: out: set[str] = set() @@ -478,171 +575,180 @@ def _is_value(text: Optional[str], values: set[str]) -> bool: return False -def faults(schema: Schema, sb: Sandbox, demo: Demo, write: list[Op]) -> list[Case]: +def _new_key( + sb: "Sandbox", t: Table, row: Mapping[str, Optional[str]] +) -> Optional[dict[str, Optional[str]]]: + """A key for a copy of ``row`` that no row holds yet, or None.""" + + if len(t.primary_key) != 1: + return None + pk = t.primary_key[0] + top = sb.max_key(t.name) + if top is not None: + return {pk: str(top + 1000)} + current = row.get(pk) + if current is not None and _UUID.match(current): + return {pk: _fresh_uuid()} + return None + + +def _copy_row( + t: Table, row: Mapping[str, Optional[str]], key: Mapping[str, Optional[str]] +) -> dict[str, Optional[str]]: + """``row`` under ``key``, with every UUID-shaped cell made new (a unique + token copied twice is refused by the store, not by the contract).""" + + out = dict(row) + out.update(key) + for c, v in list(out.items()): + if c not in key and v is not None and _UUID.match(v): + out[c] = _fresh_uuid() + return out + + +def faults(schema: Schema, sb: "Sandbox", demo: Demo, write: list[Op]) -> list[Case]: """Faults planted from one held-out write: its diff, the schema and its - task's values. The contracts are never read here.""" + task's values. The contracts are never read here. + + Every column of every row the write adds or changes gets its own fault, + NULL cells included; a column no value can be planted in gets an + ``unplantable`` case, which fails admission.""" values = _text_values(demo.values) out: list[Case] = [Case("missing", "missing", [])] touched = {op.table for op in write} - counts: dict[str, int] = {} - def add(case: Case) -> None: - n = counts.get(case.cls, 0) - if n < MAX_PER_CLASS: - counts[case.cls] = n + 1 - out.append(case) + def swap(i: int, op: Op, row: dict[str, Optional[str]]) -> list[Op]: + return write[:i] + [Op(op.table, op.key, row)] + write[i + 1 :] + + def column_fault(cls: str, i: int, op: Op, col: str) -> None: + t = schema.tables[op.table] + assert op.row is not None + new = plant_value(sb, t, col, op.row.get(col)) + name = f"{cls}:{op.table}.{col}" + if new is None: + out.append(Case(name, cls, [], unplantable=f"no value of type " + f"{t.column(col).data_type if t.column(col) else '?'} can be planted")) # fmt: skip + return + row = dict(op.row) + row[col] = new + out.append(Case(name, cls, swap(i, op, row))) for i, op in enumerate(write): t = schema.tables[op.table] + if len(write) > 1: + # Part of the write is lost (a child row, a journal entry). + out.append( + Case(f"partial:{op.table}", "partial", write[:i] + write[i + 1 :]) + ) if op.row is None: continue before = sb.get(op.table, op.key) is_new = before is None - fks = {fk.column: fk for fk in t.foreign_keys} - for c in t.columns: - if c.name in t.primary_key: - continue - cur = op.row.get(c.name) - if not is_new and before is not None and before.get(c.name) == cur: - continue # not changed by the write: see collateral_target - if is_new and cur is None: - continue - new = None - if c.name in fks: - others = [ - r - for r in sb.rows(fks[c.name].table, 20) - if r.get(fks[c.name].ref_column) not in (None, cur) - ] - new = others[0].get(fks[c.name].ref_column) if others else None - if new is None: - new = mutate(t, c.name, cur) - if new is None: - continue - cls = "wrong_value" if _is_value(cur, values) else "bookkeeping" - row = dict(op.row) - row[c.name] = new - add( - Case( - f"{cls}:{op.table}.{c.name}", - cls, - write[:i] + [Op(op.table, op.key, row)] + write[i + 1 :], - ) - ) + changed = [ + c.name + for c in t.columns + if c.name not in t.primary_key + and (is_new or before is None or before.get(c.name) != op.row.get(c.name)) + ] + for col in changed: + cls = "wrong_value" if _is_value(op.row.get(col), values) else "bookkeeping" + column_fault(cls, i, op, col) if not is_new and before is not None: - # The write's changes landed on another row of the table. - changed = {c for c in t.column_names if before.get(c) != op.row.get(c)} - for other in sb.rows(op.table, 20): - if store.row_key(t, other) == store.row_key(t, op.key): - continue + for c in t.columns: + if c.name not in t.primary_key and c.name not in changed: + column_fault("collateral_target", i, op, c.name) + # The write's changes land on another row of the table: an + # existing one, or a copy of the target when there is none. + other = next((r for r in sb.rows(op.table, 20) + if store.row_key(t, r) != store.row_key(t, op.key)), None) # fmt: skip + if other is None: + key = _new_key(sb, t, before) + other = _copy_row(t, before, key) if key is not None else None + if other is not None: moved = dict(other) for c in changed: moved[c] = op.row.get(c) - add( - Case( - f"wrong_record:{op.table}", - "wrong_record", - write[:i] + write[i + 1 :] + [Op(op.table, other, moved)], - ) - ) - break - # A column the write didn't change, changed on the target. - for c in t.columns: - if c.name in t.primary_key or c.name in changed: - continue - new = mutate(t, c.name, op.row.get(c.name)) - if new is None: - continue - row = dict(op.row) - row[c.name] = new - add( - Case( - f"collateral_target:{op.table}.{c.name}", - "collateral_target", - write[:i] + [Op(op.table, op.key, row)] + write[i + 1 :], - ) - ) + out.append(Case(f"wrong_record:{op.table}", "wrong_record", + write[:i] + write[i + 1 :] + [Op(op.table, other, moved)])) # fmt: skip + # The same change on another row: an excuse must hold only + # on the rows the write declares. + for c in changed: + if other.get(c) == op.row.get(c): + continue + edit = dict(other) + edit[c] = op.row.get(c) + out.append(Case(f"unrelated_edit:{op.table}.{c}", "unrelated_edit", + write + [Op(op.table, other, edit)])) # fmt: skip + # A late change to the target itself: its first changed column. + if changed: + late = dict(op.row) + v = plant_value(sb, t, changed[0], op.row.get(changed[0])) + if v is not None: + late[changed[0]] = v + out.append(Case(f"delayed:{op.table}.{changed[0]}", "delayed", + list(write), late=[Op(op.table, op.key, late)])) # fmt: skip if is_new: - top = sb.max_key(op.table) - if top is not None: - pk = t.primary_key[0] - dup = dict(op.row) - dup[pk] = str(top + 1000) - add( - Case( - f"duplicate:{op.table}", - "duplicate", - write + [Op(op.table, {pk: dup[pk]}, dup)], - ) - ) - # The values landed on an existing look-alike instead. + key = _new_key(sb, t, op.row) + if key is None: + out.append(Case(f"duplicate:{op.table}", "duplicate", [], + unplantable="no new key can be made for a copy")) # fmt: skip + else: + dup = _copy_row(t, op.row, key) + out.append(Case(f"duplicate:{op.table}", "duplicate", + write + [Op(op.table, key, dup)])) # fmt: skip + # The values land on an existing look-alike instead. for other in sb.rows(op.table, 20): look = dict(op.row) for c in t.primary_key: look[c] = other.get(c) - add( - Case( - f"look_alike:{op.table}", - "look_alike", - write[:i] + write[i + 1 :] + [Op(op.table, other, look)], - ) - ) + out.append(Case(f"look_alike:{op.table}", "look_alike", + write[:i] + write[i + 1 :] + [Op(op.table, other, look)])) # fmt: skip break - # Another row of a touched table, and a new row in an untouched one. + # Another row of each touched table, now and late. for name in sorted(touched): t = schema.tables[name] mine = {store.row_key(t, op.key) for op in write if op.table == name} - for other in sb.rows(name, 20): - if store.row_key(t, other) in mine: + other = next( + (r for r in sb.rows(name, 20) if store.row_key(t, r) not in mine), None + ) + if other is None: + continue + for c in t.columns: + if c.name in t.primary_key: continue - for c in t.columns: - if c.name in t.primary_key: - continue - new = mutate(t, c.name, other.get(c.name)) - if new is None: - continue - row = dict(other) - row[c.name] = new - add( - Case( - f"unrelated_edit:{name}.{c.name}", - "unrelated_edit", - write + [Op(name, other, row)], - ) - ) - add( - Case( - f"delayed:{name}.{c.name}", - "delayed", - list(write), - late=[Op(name, other, row)], - ) - ) - break + v = plant_value(sb, t, c.name, other.get(c.name)) + if v is None: + continue + row = dict(other) + row[c.name] = v + out.append(Case(f"unrelated_edit:{name}.{c.name}", "unrelated_edit", + write + [Op(name, other, row)])) # fmt: skip + out.append(Case(f"delayed:{name}.{c.name}", "delayed", list(write), + late=[Op(name, other, row)])) # fmt: skip break + planted = 0 for name in sorted(set(schema.tables) - touched): - top = sb.max_key(name) + t = schema.tables[name] sample = sb.rows(name, 1) - if top is None or not sample: + key = _new_key(sb, t, sample[0]) if sample else None + if key is None: continue - row = dict(sample[0]) - pk = schema.tables[name].primary_key[0] - row[pk] = str(top + 1000) - add( - Case( - f"other_table:{name}", - "other_table", - write + [Op(name, {pk: row[pk]}, row)], - ) - ) - if counts.get("other_table", 0) >= 2: + out.append(Case(f"other_table:{name}", "other_table", + write + [Op(name, key, _copy_row(t, sample[0], key))])) # fmt: skip + planted += 1 + if planted >= 2: break return out # -- running a case --------------------------------------------------------------- +#: Verdicts the verifier gives for a state it read and didn't pass. Only +#: these count as a stopped fault: an error, a missing contract or a fault +#: that couldn't be planted never does. +STOP_VERDICTS = ("REFUTED", "UNCERTAIN", "RECHECK_REFUTED", "RECHECK_UNCERTAIN") + @dataclass class Outcome: @@ -655,9 +761,12 @@ class Outcome: @property def ok(self) -> bool: - """Confirmed when correct; stopped when a fault.""" + """Confirmed when correct; for a fault, a verifier verdict that + stops it, on a state that was planted.""" - return self.passed if self.correct else not self.passed + if self.correct: + return self.passed + return not self.passed and self.outcome in STOP_VERDICTS def _put_all(sb: Sandbox, ops: Sequence[Op]) -> None: @@ -680,6 +789,9 @@ def run_case( from openadapt_flow.verifier import Actor, begin_write, recheck, verify_write + if case.unplantable: + return Outcome(case.name, case.cls, case.correct, "NOT_PLANTED", False, + [case.unplantable]) # fmt: skip undo = backward(demo.diff.changes) all_ops = undo + case.write + case.late saved: dict[ @@ -746,7 +858,7 @@ def auto_answers( ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """The answers this module gives, and a ledger saying by which rule.""" - answers: dict[str, Any] = {"now_checks": True} + answers: dict[str, Any] = {"now_checks": True, "strict_pins": True} ledger: list[dict[str, Any]] = [ { "key": "now_checks", @@ -754,6 +866,11 @@ def auto_answers( "rule": "an unchecked or excused time stamp of the main record must hold a time near the read", }, ] + ledger.append({ + "key": "strict_pins", + "answer": True, + "rule": "a new-row column two readings fit in every demonstration is checked both ways", + }) # fmt: skip for q in report.get("questions") or []: key = str(q.get("key")) proposed = q.get("proposal") @@ -827,12 +944,21 @@ def validate( extra: Mapping[str, Any], log: Callable[[str], None] = lambda s: None, record: Optional[list[dict[str, Any]]] = None, + modes: Optional[dict[str, set[str]]] = None, ) -> dict[str, list[Outcome]]: - """Every held-out write, replayed and faulted, by kind.""" + """Every held-out write, replayed and faulted, by kind. ``modes`` is + filled with which rows each kind's held-out writes added or changed.""" + + if modes is None: + modes = {} assert sb.schema is not None by_kind: dict[str, list[Outcome]] = {} for demo in holdout: + for ch in demo.diff.changes: + modes.setdefault(demo.kind, set()).add( + "create" if ch.kind == "added" else "update" + ) values = _text_values(demo.values) write = shift_times(sb.schema, forward(demo.diff.changes), demo, values) cases = [Case("correct", "correct", write, correct=True)] @@ -859,22 +985,37 @@ def validate( def summarize( - by_kind: Mapping[str, Sequence[Outcome]], kinds: Sequence[str] + by_kind: Mapping[str, Sequence[Outcome]], + kinds: Sequence[str], + modes: Mapping[str, set[str]] = {}, ) -> dict[str, Any]: + """Per kind: admitted only when it has a held-out write, every held-out + write was confirmed, every fault was planted and stopped, and every + required class was planted. ``modes``: kind -> ``{"create", "update"}``, + which rows its held-out writes added or changed.""" + out: dict[str, Any] = {} for k in kinds: rs = list(by_kind.get(k) or []) correct = [r for r in rs if r.correct] fl = [r for r in rs if not r.correct] + need = list(REQUIRED_CLASSES) + if "create" in modes.get(k, set()): + need += CREATE_CLASSES + if "update" in modes.get(k, set()): + need += UPDATE_CLASSES + planted = {r.cls for r in fl if r.outcome != "NOT_PLANTED"} + missing = [c for c in need if c not in planted] out[k] = { - "admitted": bool(correct) and all(r.ok for r in rs), + "admitted": bool(correct) and all(r.ok for r in rs) and not missing, "correct_writes": len(correct), "false_stops": [ f"{r.outcome}: {'; '.join(r.reasons[:2])}" for r in correct if not r.ok ], "faults": len(fl), "faults_stopped": sum(1 for r in fl if r.ok), - "faults_passed": [r.case for r in fl if not r.ok], + "faults_passed": [f"{r.case} ({r.outcome})" for r in fl if not r.ok], + "classes_not_planted": missing, "by_class": { c: f"{sum(1 for r in fl if r.cls == c and r.ok)}/{sum(1 for r in fl if r.cls == c)}" for c in sorted({r.cls for r in fl}) @@ -904,7 +1045,9 @@ def exceptions_md(summary: Mapping[str, Any], report: Mapping[str, Any]) -> str: for fs in s["false_stops"]: lines.append(f" - a held-out correct write stopped: {fs}") for f in s["faults_passed"]: - lines.append(f" - a planted fault passed: {f}") + lines.append(f" - a fault was not planted and stopped: {f}") + for c in s.get("classes_not_planted") or []: + lines.append(f" - no `{c}` fault could be planted") for r in ((report.get("kinds") or {}).get(k) or {}).get("refused") or []: lines.append(f" - generator: {r}") if n == 0: @@ -912,6 +1055,111 @@ def exceptions_md(summary: Mapping[str, Any], report: Mapping[str, Any]) -> str: return "\n".join(lines) + "\n" +def check_holdout( + demos: Sequence[Demo], holdout: Sequence[Demo], kinds: Sequence[str] +) -> None: + """Refuse a held-out set that isn't held out: a held-out write with the + same kind and values as a demonstration, or the same diff, or a kind + with no held-out write.""" + + def key(d: Demo) -> str: + return json.dumps([d.kind, d.values], sort_keys=True, default=str) + + def diff_key(d: Demo) -> str: + return json.dumps(d.diff.to_json(), sort_keys=True, default=str) + + seen = {key(d) for d in demos} + seen_diffs = {diff_key(d) for d in demos} + for d in holdout: + if key(d) in seen or diff_key(d) in seen_diffs: + raise ZeroTouchError( + f"held-out write {d.name} repeats a demonstration; a held-out " + "write must have values the generator never saw" + ) + missing = sorted(set(kinds) - {d.kind for d in holdout}) + if missing: + raise ZeroTouchError( + "no held-out write for kind(s): " + + ", ".join(missing) + + "; every kind needs at least one to be validated" + ) + + +_INTERVAL = re.compile(r"INTERVAL (?:'\d+ seconds'|\d+ SECOND)") + + +def _normal(doc: Any) -> Any: + """A contract with what differs between the sandbox and the app's own + store taken out: connections, the recheck window and the time-check + window that follows from it.""" + + if isinstance(doc, Mapping): + return { + k: _normal(v) + for k, v in doc.items() + if k + not in ( + "sql_connect_args", + "sql_password_env", + "recheck_after_s", + "environment_id", + ) # fmt: skip + } + if isinstance(doc, list): + return [_normal(v) for v in doc] + if isinstance(doc, str): + return _INTERVAL.sub("INTERVAL ", doc) + return doc + + +def fingerprint(manifest_dir: Path, kind: str) -> str: + """A digest of one kind's listed contracts and manifest entry, as + :func:`_normal` sees them.""" + + import hashlib + + manifest = _load_yaml(manifest_dir / "manifest.yaml") or {} + entry = (manifest.get("kinds") or {}).get(kind) + files: dict[str, Any] = {} + for var in (entry or {}).get("variants") or []: + for k, f in var.items(): + if k in ("create", "update", "found") and isinstance(f, str): + files[f] = _normal(_load_yaml(manifest_dir / f)) + body = json.dumps({"entry": _normal(entry), "files": files}, sort_keys=True, + default=str) # fmt: skip + return hashlib.sha256(body.encode()).hexdigest() + + +def install(staged: Path, final: Path, admitted: Sequence[str]) -> None: + """Keep only the admitted kinds in ``staged`` (manifest entries, pinned + digests and contract files), then move it to ``final`` in one rename.""" + + import shutil + + path = staged / "manifest.yaml" + manifest = _load_yaml(path) or {} + dropped = [k for k in list(manifest.get("kinds") or {}) if k not in admitted] + for k in dropped: + manifest["kinds"].pop(k) + for sub in ("contracts", "field_sets"): + shutil.rmtree(staged / sub / k, ignore_errors=True) + listed = (manifest.get("field_sets") or {}).get("listed_sha256") or {} + for rel in list(listed): + if not (staged / rel).exists(): + listed.pop(rel) + manifest["zero_touch"] = {"admitted": sorted(admitted)} + path.write_text(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True), + encoding="utf-8") # fmt: skip + if final.exists(): + old = final.with_name(final.name + ".old") + shutil.rmtree(old, ignore_errors=True) + final.rename(old) + staged.rename(final) + shutil.rmtree(old, ignore_errors=True) + else: + staged.rename(final) + + def onboard( *, read: dict[str, Any], @@ -931,7 +1179,11 @@ def onboard( t0 = time.time() kinds = gen.load_kinds(kinds_path) + kind_names = [k.kind for k in kinds] + if Path(holdout_dir).resolve() == Path(demo_dir).resolve(): + raise ZeroTouchError("the held-out folder is the demonstrations folder") holdout = gen.load_demos(holdout_dir) + check_holdout(gen.load_demos(demo_dir), holdout, kind_names) schema = store.read_schema(store.connector(read), store.dialect_of(read)) sb = Sandbox(admin, read) sb.create(schema, reader) @@ -945,66 +1197,80 @@ def onboard( for key, params in options.items(): for name, n in params.items(): trials += [{key: {name: i}} for i in range(n)] - kind_names = [k.kind for k in kinds] best: dict[str, Any] = {} results: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = [] - for picks in trials: - answers, ledger = auto_answers(report, picks) - sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} - tag = ( - "base" - if not picks - else "pick-" - + "-".join( - f"{k}.{p}.{i}" for k, d in picks.items() for p, i in d.items() - ) - ) + + def answer(picks: Mapping[str, Mapping[str, int]], tag: str) -> tuple[Any, Any]: + # The bookkeeping proposal depends on where values are placed: ask + # with the picks first, then answer with that run's own proposal. + first = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, + out=work / tag / "q", actor_params=actor_params, + answers={**{k: dict(v) for k, v in picks.items()}, + "recheck_after_s": SANDBOX_RECHECK_S}) # fmt: skip + return auto_answers(first, picks) + + def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: log(f"propose ({tag})") - rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / tag, - actor_params=actor_params, answers=sans) # fmt: skip + try: + answers, ledger = answer(picks, tag) + sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} + rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, + out=work / tag, actor_params=actor_params, + answers=sans) # fmt: skip + except gen.GenerateError as exc: + # These answers don't fit together: nothing is admitted. + why = f"the generator refused these answers: {exc}" + return {"answers": {}, "ledger": [], "report": {}, "cases": [], + "dir": work / tag, + "summary": {k: {"admitted": False, "why": why, "correct_writes": 0, + "false_stops": [], "faults": 0, "faults_stopped": 0, + "faults_passed": [], "classes_not_planted": [], + "by_class": {}} for k in kind_names}} # fmt: skip cases: list[dict[str, Any]] = [] + modes: dict[str, set[str]] = {} by_kind = validate(sb, work / tag / "manifest.yaml", holdout, actor_id, - dict(actor_params), log, cases) # fmt: skip - summary = summarize(by_kind, kind_names) - results.append((picks, answers, summary)) - best = best or {"answers": answers, "ledger": ledger, "summary": summary, - "report": rep, "cases": cases} # fmt: skip + dict(actor_params), log, cases, modes) # fmt: skip + return {"answers": answers, "ledger": ledger, "report": rep, "cases": cases, + "dir": work / tag, "summary": summarize(by_kind, kind_names, modes)} # fmt: skip + + for picks in trials: + tag = "base" if not picks else "pick-" + "-".join( + f"{k}.{p}.{i}" for k, d in picks.items() for p, i in d.items()) # fmt: skip + got = trial(picks, tag) + results.append((picks, got["answers"], got["summary"])) + best = best or got # A location pick is kept when exactly one candidate admits its kind. final_picks: dict[str, dict[str, int]] = {} for key, params in options.items(): kind = key.split(".", 1)[1] for name in params: - ok = [ - p[key][name] - for p, _, s in results - if key in p and name in p[key] and s[kind]["admitted"] - ] + ok = [p[key][name] for p, _, s in results + if key in p and name in p[key] and s[kind]["admitted"]] # fmt: skip if len(ok) == 1: final_picks.setdefault(key, {})[name] = ok[0] - answers, ledger = auto_answers(report, final_picks) - if final_picks: - sans = {**answers, "recheck_after_s": SANDBOX_RECHECK_S} - rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / "final", - actor_params=actor_params, answers=sans) # fmt: skip - cases = [] - summary = summarize(validate(sb, work / "final" / "manifest.yaml", holdout, - actor_id, dict(actor_params), log, cases), kind_names) # fmt: skip - else: - summary, rep, cases = best["summary"], best["report"], best["cases"] + chosen = trial(final_picks, "final") if final_picks else best + answers, ledger = chosen["answers"], chosen["ledger"] + summary, cases = chosen["summary"], chosen["cases"] + validated = chosen["dir"] finally: if not keep_sandbox: sb.drop() - # The production contracts: the same answers, against the app's own store. - prod = gen.propose(read=read, kinds=kinds, demo_dir=demo_dir, out=out / "contracts", + # The production contracts: the same answers, against the app's own + # store, staged beside the final folder. A kind is kept only when its + # contracts are the validated ones, connections and windows aside. + staged = out / ".contracts.staged" + import shutil + + shutil.rmtree(staged, ignore_errors=True) + prod = gen.propose(read=read, kinds=kinds, demo_dir=demo_dir, out=staged, actor_params=actor_params, answers=answers) # fmt: skip - manifest_path = out / "contracts" / "manifest.yaml" - manifest = _load_yaml(manifest_path) + for k, s in summary.items(): + if s["admitted"] and fingerprint(staged, k) != fingerprint(validated, k): + s["admitted"] = False + s["why"] = ("the contracts proposed against the app's own store differ " + "from the ones validated in the sandbox") # fmt: skip admitted = sorted(k for k, s in summary.items() if s["admitted"]) - for k in list((manifest.get("kinds") or {})): - if k not in admitted: - manifest["kinds"].pop(k) - manifest["zero_touch"] = {"admitted": admitted} - manifest_path.write_text(yaml.safe_dump(manifest, sort_keys=False)) + install(staged, out / "contracts", admitted) result = { "minutes": round((time.time() - t0) / 60, 2), "person_decisions": 0, diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py index 5a211de027..5d45158b31 100644 --- a/packages/flow/tests/test_zero_touch.py +++ b/packages/flow/tests/test_zero_touch.py @@ -233,3 +233,182 @@ def test_without_the_time_check_the_time_stamp_fault_passes( r for r in onboarded["control"] if r["case"] == "bookkeeping:issues.created_on" ] assert hit and not any(r["ok"] for r in hit), hit + + +# -- the admission gate, without a database (review round 1, B1-B5, S7) ------- + + +class _FakeSandbox: + """Rows as a sandbox would read them; nothing is written.""" + + def __init__(self, rows: dict[str, list[dict[str, Any]]]) -> None: + self._rows = rows + + def get(self, table: str, key: Any) -> Any: + return None + + def rows(self, table: str, limit: int = 50) -> list[dict[str, Any]]: + return self._rows.get(table, [])[:limit] + + def max_key(self, table: str) -> int: + return 1 + + def distinct(self, table: str, column: str, limit: int = 5) -> list[str]: + return [] + + +def _wide() -> tuple[Any, Any, Any, list[Any]]: + from openadapt_flow.generate.sql_store import Column, Schema, Table + + cols = [Column("id", "int", False, None, auto=True)] + cols += [Column(f"c{i:02d}", "varchar", True, None) for i in range(20)] + cols += [Column("meta", "json", True, None), Column("blob", "blob", True, None), + Column("sex", "enum", True, None), Column("note", "varchar", True, None), + Column("t", "time", True, None), Column("u", "uuid", True, None)] # fmt: skip + patient = Table("patient", cols, ["id"]) + other = Table( + "other", + [Column("id", "int", False, None), Column("x", "int", True, None)], + ["id"], + ) + schema = Schema("mysql", "app", {"patient": patient, "other": other}) + sb = _FakeSandbox({"other": [{"id": "1", "x": "5"}], + "patient": [{"id": "1", **{c.name: "old" for c in cols[1:]}}]}) # fmt: skip + + class D: + values = {"name": "Ann"} + + row = {"id": "2", **{f"c{i:02d}": f"v{i}" for i in range(20)}, "meta": '{"a": 1}', + "blob": "0xff", "sex": "F", "note": None, "t": "10:00:00", + "u": "123e4567-e89b-12d3-a456-426614174000"} # fmt: skip + return schema, sb, D(), [zt.Op("patient", {"id": "2"}, row)] + + +def test_b2_b3_b4_every_column_of_a_wide_row_gets_a_planted_fault() -> None: + schema, sb, demo, write = _wide() + cases = zt.faults(schema, sb, demo, write) + faulted = {c.name.split(".")[-1] for c in cases + if c.cls in ("bookkeeping", "wrong_value") and not c.unplantable} # fmt: skip + assert faulted == set(write[0].row) - {"id"} # note (NULL) included + note = next(c for c in cases if c.name == "bookkeeping:patient.note") + assert note.write[0].row["note"] is not None # B4: a non-NULL plant + meta = next(c for c in cases if c.name == "bookkeeping:patient.meta") + assert json.loads(meta.write[0].row["meta"]) != {"a": 1} # B3: valid JSON + sex = next(c for c in cases if c.name == "bookkeeping:patient.sex") + assert len(sex.write[0].row["sex"]) == 1 # text keeps its length + + +def test_b3_a_type_with_no_generator_blocks_admission() -> None: + from openadapt_flow.generate.sql_store import Column, Schema, Table + + t = Table( + "p", + [Column("id", "int", False, None), Column("geo", "geometry", True, None)], + ["id"], + ) + schema = Schema("mysql", "app", {"p": t}) + cases = zt.faults(schema, _FakeSandbox({}), type("D", (), {"values": {}})(), + [zt.Op("p", {"id": "2"}, {"id": "2", "geo": "0101"})]) # fmt: skip + geo = [c for c in cases if c.name.endswith("p.geo")] + assert geo and geo[0].unplantable + out = zt.run_case(None, Path("."), None, geo[0], "a", {}) # type: ignore[arg-type] + assert out.outcome == "NOT_PLANTED" and not out.ok + + +def test_b1_a_fault_that_errors_is_not_stopped() -> None: + correct = zt.Outcome("correct", "correct", True, "CONFIRMED", True) + for verdict in ("ERROR", "NO_CONTRACT", "NOT_PLANTED"): + bad = zt.Outcome("bookkeeping:p.sex", "bookkeeping", False, verdict, False) + assert not bad.ok + s = zt.summarize({"k": [correct, bad]}, ["k"])["k"] + assert not s["admitted"], verdict + assert zt.Outcome("x", "missing", False, "REFUTED", False).ok + assert zt.Outcome("x", "delayed", False, "RECHECK_REFUTED", False).ok + + +def test_b5_a_kind_without_the_core_fault_classes_is_not_admitted() -> None: + correct = zt.Outcome("correct", "correct", True, "CONFIRMED", True) + missing = zt.Outcome("missing", "missing", False, "UNCERTAIN", False) + s = zt.summarize({"k": [correct, missing]}, ["k"], {"k": {"create"}})["k"] + assert not s["admitted"] + assert { + "unrelated_edit", + "delayed", + "other_table", + "duplicate", + "look_alike", + } <= set(s["classes_not_planted"]) + full = [correct] + [zt.Outcome(c, c, False, "REFUTED", False) + for c in (*zt.REQUIRED_CLASSES, *zt.CREATE_CLASSES)] # fmt: skip + assert zt.summarize({"k": full}, ["k"], {"k": {"create"}})["k"]["admitted"] + + +def test_s7_a_held_out_write_that_repeats_a_demonstration_is_refused() -> None: + from openadapt_flow.generate.demo_analysis import Demo + from openadapt_flow.generate.sql_store import Diff + + a = Demo("k", {"x": 1}, Diff([]), {}, name="a") + b = Demo("k", {"x": 2}, Diff([]), {}, name="b") + with pytest.raises(zt.ZeroTouchError): + zt.check_holdout([a], [Demo("k", {"x": 1}, Diff([]), {}, name="c")], ["k"]) + with pytest.raises(zt.ZeroTouchError): + zt.check_holdout([a], [b], ["k", "other_kind"]) + + +def test_s1_s2_install_keeps_only_admitted_kinds_and_fingerprints_ignore_connections( + tmp_path: Path, +) -> None: + staged = tmp_path / "staged" + for k in ("a", "b"): + (staged / "contracts" / k).mkdir(parents=True) + (staged / "contracts" / k / "x.create.yaml").write_text(yaml.safe_dump( + {"verifier": {"sql_connect_args": {"port": 1}, "sql_query": "SELECT 1"}, + "frame": {"database": {"recheck_after_s": 3}}})) # fmt: skip + manifest = {"kinds": {k: {"mode": "create", "variants": [ + {"fields": ["x"], "create": f"contracts/{k}/x.create.yaml"}]} for k in ("a", "b")}, + "field_sets": {"listed_sha256": {f"contracts/{k}/x.create.yaml": "d" for k in ("a", "b")}}} # fmt: skip + (staged / "manifest.yaml").write_text(yaml.safe_dump(manifest)) + other = tmp_path / "validated" + shutil_copy = __import__("shutil").copytree + shutil_copy(staged, other) + doc = yaml.safe_load((other / "contracts/a/x.create.yaml").read_text()) + doc["verifier"]["sql_connect_args"]["port"] = 2 + doc["frame"]["database"]["recheck_after_s"] = 300 + (other / "contracts/a/x.create.yaml").write_text(yaml.safe_dump(doc)) + assert zt.fingerprint(staged, "a") == zt.fingerprint(other, "a") + doc["verifier"]["sql_query"] = "SELECT 2" + (other / "contracts/a/x.create.yaml").write_text(yaml.safe_dump(doc)) + assert zt.fingerprint(staged, "a") != zt.fingerprint(other, "a") + zt.install(staged, tmp_path / "contracts", ["a"]) + got = yaml.safe_load((tmp_path / "contracts" / "manifest.yaml").read_text()) + assert list(got["kinds"]) == ["a"] + assert list(got["field_sets"]["listed_sha256"]) == ["contracts/a/x.create.yaml"] + assert not (tmp_path / "contracts" / "contracts" / "b").exists() + assert not staged.exists() + + +def test_strict_pins_checks_both_readings_and_is_off_by_default() -> None: + from openadapt_flow.generate.demo_analysis import ColumnRole + from openadapt_flow.generate.sql_store import Column, ForeignKey, Schema, Table + + item = Table("item", [Column("id", "int", False, None), Column("cur", "varchar", True, None), + Column("prod", "int", True, None)], ["id"], + [ForeignKey("prod", "product", "id")]) # fmt: skip + product = Table("product", [Column("id", "int", False, None), + Column("cur", "varchar", True, None)], ["id"]) # fmt: skip + schema = Schema("mysql", "d", {"item": item, "product": product}) + copied = {"table": "product", "column": "cur", "ref_column": "id", "via": "prod"} + role = ColumnRole("varies", detail={"why": "a constant or a copy: can't tell", + "strict": {"constant": "EUR", "copied": copied}}) # fmt: skip + assert gen._strict({"cur": role}, False)["cur"].role == "varies" + strict = gen._strict({"cur": role}, True)["cur"] + assert strict.role == "strict" + b = gen.SqlBuilder(schema) + surf = gen.Surface("record", "item", []) + match: dict[str, Any] = {} + gen._pin(b, surf, "item", "cur", strict, match, {}) + exprs = dict(surf.columns) + assert match == {"pin__cur": {"literal": "1"}, "copy__cur": {"literal": "1"}} + assert "'EUR'" in exprs["pin__cur"] and "`product`" in exprs["copy__cur"] + with pytest.raises(gen.Unpinned): + gen._pin(b, gen.Surface("record", "item", []), "item", "cur", role, {}, {}) From 551c3b5332da839992a012fa3fedce612c92fcc8 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 15:49:34 -0500 Subject: [PATCH 04/12] Zero-touch: type-made plants for typed columns; strict_pins pins a one-demonstration constant; docs --- packages/flow/docs/GENERATE_CONTRACTS.md | 8 ++++++++ .../flow/openadapt_flow/generate/demo_analysis.py | 9 ++++++--- packages/flow/openadapt_flow/generate/zero_touch.py | 12 ++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/flow/docs/GENERATE_CONTRACTS.md b/packages/flow/docs/GENERATE_CONTRACTS.md index b745606461..0f79058570 100644 --- a/packages/flow/docs/GENERATE_CONTRACTS.md +++ b/packages/flow/docs/GENERATE_CONTRACTS.md @@ -360,6 +360,14 @@ no more than five minutes ahead, by the database's local clock or by UTC. A NULL fails. Without it, such a column may hold any value. It only adds a check. +**Checking both readings.** With `strict_pins: true`, a new-row column the +demonstrations can't settle is checked instead of refusing the kind, in +three cases: it holds one value that is also a copy of a column a key points +to (both are checked), it holds the acting user's id and such a copy (both +are checked), or its field set was seen in one demonstration only with one +value (that value is pinned). Each only adds a check: a wrong guess stops +correct writes, never passes a wrong one. + **Setting up with no person answers.** `python -m openadapt_flow.generate.zero_touch` answers these questions by fixed rules and admits a kind only after a sandbox copy of the store confirms held-out diff --git a/packages/flow/openadapt_flow/generate/demo_analysis.py b/packages/flow/openadapt_flow/generate/demo_analysis.py index 67d4d2268c..3c419311d9 100644 --- a/packages/flow/openadapt_flow/generate/demo_analysis.py +++ b/packages/flow/openadapt_flow/generate/demo_analysis.py @@ -732,9 +732,12 @@ def column_roles( out[name] = ColumnRole("free", detail={"empty": empty}) continue if one: - out[name] = ColumnRole( - "varies", detail={"why": "seen in one demonstration only"} - ) + detail = {"why": "seen in one demonstration only"} + if len(distinct) == 1: + # ``strict_pins`` pins the one value seen: too strict at + # worst, which a held-out write shows. + detail["strict"] = {"constant": values[0]} + out[name] = ColumnRole("varies", detail=detail) continue if len(distinct) == 1: v = values[0] diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index d0e8cb257c..7df068c656 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -525,8 +525,11 @@ def plant_value( sb: "Sandbox", table: Table, column: str, current: Optional[str] ) -> Optional[str]: """A value to plant in ``table.column`` instead of ``current``: another - row's key for a declared foreign key, else a value the column already - holds in another row (the store accepts it), else :func:`mutate`.""" + row's key for a declared foreign key; for a type the store takes any + value of (a date or time, a flag, JSON, a UUID, bytes), a value made by + :func:`mutate`, far from anything a write would store; for text and + numbers, a value the column already holds in another row (a length or an + enum the schema doesn't show still accepts it), else :func:`mutate`.""" fk = next((f for f in table.foreign_keys if f.column == column), None) if fk is not None: @@ -534,6 +537,11 @@ def plant_value( v = r.get(fk.ref_column) if v is not None and v != current: return v + col = table.column(column) + if col is not None and col.kind not in ("text", "number"): + made = mutate(table, column, current) + if made is not None: + return made for v in sb.distinct(table.name, column, 5): if v is not None and v != current: return v From d46829a0bbcd0a61c76f24687158898be1428277 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 16:39:09 -0500 Subject: [PATCH 05/12] Zero-touch: two-phase sandbox restore; a duplicate the store's unique key refuses is not applicable (local, for review) --- .../openadapt_flow/generate/zero_touch.py | 74 ++++++++++++++----- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index 7df068c656..01b9ec1531 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -321,6 +321,20 @@ def max_key(self, table: str) -> Optional[int]: got = self._exec(f"SELECT MAX({self.q(col.name)}) FROM {self.t(table)}") return int(got[0][0] or 0) + def put_many(self, ops: Sequence["Op"]) -> None: + """Put each op's row, in two phases: every key is deleted first, then + every row inserted, so rows swapping a unique key don't collide on + the way. The last op for a key wins.""" + + last: dict[tuple[str, str], Op] = {} + for op in ops: + last[(op.table, store.row_key(self._table(op.table), op.key))] = op + for op in last.values(): + self.put(op.table, None, op.key) + for op in last.values(): + if op.row is not None: + self.put(op.table, op.row, op.key) + def put( self, table: str, @@ -778,8 +792,18 @@ def ok(self) -> bool: def _put_all(sb: Sandbox, ops: Sequence[Op]) -> None: - for op in ops: - sb.put(op.table, op.row, op.key) + sb.put_many(ops) + + +def _unique_violation(exc: BaseException) -> bool: + """The store refused a row for a duplicate unique key (MySQL 1062, + PostgreSQL 23505).""" + + code = getattr(exc, "pgcode", None) + if code == "23505": + return True + args = getattr(exc, "args", ()) + return bool(args) and args[0] == 1062 def run_case( @@ -845,6 +869,11 @@ def run_case( ) return Outcome(case.name, case.cls, case.correct, outcome, passed, reasons[:6]) except Exception as exc: # noqa: BLE001 - a case that can't run admits nothing + if case.cls == "duplicate" and _unique_violation(exc): + # The store's own unique key refuses this copy: such a duplicate + # can't exist, so it is neither stopped nor passed. + return Outcome(case.name, case.cls, case.correct, "NOT_APPLICABLE", + False, [f"{type(exc).__name__}: {exc}"]) # fmt: skip return Outcome( case.name, case.cls, @@ -854,8 +883,12 @@ def run_case( [f"{type(exc).__name__}: {exc}", traceback.format_exc(limit=3)], ) finally: - for (table, _), (key, row) in saved.items(): - sb.put(table, row, key) + try: + sb.put_many( + [Op(table, key, row) for (table, _), (key, row) in saved.items()] + ) + except Exception as exc: # noqa: BLE001 - the sandbox is no longer known + raise ZeroTouchError(f"the sandbox could not be restored: {exc}") from exc # -- answers ---------------------------------------------------------------------- @@ -977,8 +1010,7 @@ def validate( try: cases += faults(sb.schema, sb, demo, write) finally: - for op, row in saved: - sb.put(op.table, row, op.key) + sb.put_many([Op(op.table, op.key, row) for op, row in saved]) for case in cases: res = run_case( sb, manifest, demo, case, actor_id, extra, manifest.parent / "state" @@ -1006,7 +1038,7 @@ def summarize( for k in kinds: rs = list(by_kind.get(k) or []) correct = [r for r in rs if r.correct] - fl = [r for r in rs if not r.correct] + fl = [r for r in rs if not r.correct and r.outcome != "NOT_APPLICABLE"] need = list(REQUIRED_CLASSES) if "create" in modes.get(k, set()): need += CREATE_CLASSES @@ -1217,6 +1249,13 @@ def answer(picks: Mapping[str, Mapping[str, int]], tag: str) -> tuple[Any, Any]: "recheck_after_s": SANDBOX_RECHECK_S}) # fmt: skip return auto_answers(first, picks) + def failed(tag: str, why: str) -> dict[str, Any]: + blank = {"admitted": False, "why": why, "correct_writes": 0, + "false_stops": [], "faults": 0, "faults_stopped": 0, + "faults_passed": [], "classes_not_planted": [], "by_class": {}} # fmt: skip + return {"answers": {}, "ledger": [], "report": {}, "cases": [], + "dir": work / tag, "summary": {k: dict(blank) for k in kind_names}} # fmt: skip + def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: log(f"propose ({tag})") try: @@ -1225,19 +1264,14 @@ def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: rep = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / tag, actor_params=actor_params, answers=sans) # fmt: skip - except gen.GenerateError as exc: - # These answers don't fit together: nothing is admitted. - why = f"the generator refused these answers: {exc}" - return {"answers": {}, "ledger": [], "report": {}, "cases": [], - "dir": work / tag, - "summary": {k: {"admitted": False, "why": why, "correct_writes": 0, - "false_stops": [], "faults": 0, "faults_stopped": 0, - "faults_passed": [], "classes_not_planted": [], - "by_class": {}} for k in kind_names}} # fmt: skip - cases: list[dict[str, Any]] = [] - modes: dict[str, set[str]] = {} - by_kind = validate(sb, work / tag / "manifest.yaml", holdout, actor_id, - dict(actor_params), log, cases, modes) # fmt: skip + cases: list[dict[str, Any]] = [] + modes: dict[str, set[str]] = {} + by_kind = validate(sb, work / tag / "manifest.yaml", holdout, actor_id, + dict(actor_params), log, cases, modes) # fmt: skip + except (gen.GenerateError, ZeroTouchError) as exc: + # The answers don't fit together, or the sandbox broke: + # nothing is admitted. + return failed(tag, f"not validated: {exc}") return {"answers": answers, "ledger": ledger, "report": rep, "cases": cases, "dir": work / tag, "summary": summarize(by_kind, kind_names, modes)} # fmt: skip From ea4f102cdb006580bc7f50e6c4deddd2ebb12b6a Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 17:09:58 -0500 Subject: [PATCH 06/12] Zero-touch: a duplicate is not applicable only when a declared unique index covering unchanged columns refuses it; recorded per kind --- .../openadapt_flow/generate/zero_touch.py | 107 +++++++++++++++--- packages/flow/tests/test_zero_touch.py | 80 +++++++++++++ 2 files changed, 174 insertions(+), 13 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index 01b9ec1531..f23d276cd8 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -302,6 +302,34 @@ def rows(self, table: str, limit: int = 50) -> list[dict[str, Optional[str]]]: ) return [{c: store.cell_text(v) for c, v in zip(cols, r)} for r in got] + def unique_indexes(self, table: str) -> dict[str, list[str]]: + """Unique indexes of a sandbox table other than its primary key: + name -> columns in index order, from the catalog.""" + + if self.dialect == "mysql": + got = self._exec( + "SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS " + "WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND NON_UNIQUE = 0 " + "AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX", + (self.name, table), + ) + else: + got = self._exec( + "SELECT i.relname, a.attname FROM pg_index x " + "JOIN pg_class c ON c.oid = x.indrelid " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "JOIN pg_class i ON i.oid = x.indexrelid " + "JOIN LATERAL unnest(x.indkey) WITH ORDINALITY k(attnum, ord) ON true " + "JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum " + "WHERE n.nspname = %s AND c.relname = %s AND x.indisunique " + "AND NOT x.indisprimary ORDER BY i.relname, k.ord", + (self.name, table), + ) + out: dict[str, list[str]] = {} + for name, col in got: + out.setdefault(str(name), []).append(str(col)) + return out + def distinct(self, table: str, column: str, limit: int = 5) -> list[str]: """Up to ``limit`` values the column holds, as text.""" @@ -477,6 +505,11 @@ class Case: late: list[Op] = field(default_factory=list) correct: bool = False unplantable: str = "" + #: For a ``duplicate``: the table, and the row copied with the copy, so a + #: refusal by a unique key can be checked against the schema. + copy_of: Optional[ + tuple[str, dict[str, Optional[str]], dict[str, Optional[str]]] + ] = None #: Fault classes every kind's held-out writes must have produced before the @@ -718,7 +751,8 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: else: dup = _copy_row(t, op.row, key) out.append(Case(f"duplicate:{op.table}", "duplicate", - write + [Op(op.table, key, dup)])) # fmt: skip + write + [Op(op.table, key, dup)], + copy_of=(op.table, dict(op.row), dup))) # fmt: skip # The values land on an existing look-alike instead. for other in sb.rows(op.table, 20): look = dict(op.row) @@ -795,15 +829,47 @@ def _put_all(sb: Sandbox, ops: Sequence[Op]) -> None: sb.put_many(ops) -def _unique_violation(exc: BaseException) -> bool: - """The store refused a row for a duplicate unique key (MySQL 1062, - PostgreSQL 23505).""" +def _unique_violation(exc: BaseException) -> Optional[str]: + """The constraint name when the store refused a row for a duplicate + unique key (MySQL 1062, PostgreSQL 23505, SQLite UNIQUE), else None. + An empty string: a unique violation whose constraint isn't named.""" - code = getattr(exc, "pgcode", None) - if code == "23505": - return True + if getattr(exc, "pgcode", None) == "23505": + diag = getattr(exc, "diag", None) + return str(getattr(diag, "constraint_name", "") or "") args = getattr(exc, "args", ()) - return bool(args) and args[0] == 1062 + if args and args[0] == 1062: + m = re.search(r"for key '([^']+)'", str(args[-1])) + return m.group(1).split(".")[-1] if m else "" + text = str(exc) + if text.startswith("UNIQUE constraint failed"): + return "" + return None + + +def blocked_by_constraint( + sb: "Sandbox", case: "Case", exc: BaseException +) -> Optional[str]: + """The unique index that makes ``case``'s duplicate impossible, or None. + + Only when the store refused the copy for a unique violation, the + violated index is one the schema declares unique on that table (not the + primary key), and every column of that index holds the same value in the + copy as in the row it copies: then no duplicate of that row can be + stored at all.""" + + if case.cls != "duplicate" or case.copy_of is None: + return None + name = _unique_violation(exc) + if not name: + return None + table, row, copy = case.copy_of + cols = sb.unique_indexes(table).get(name) + if not cols: + return None + if all(row.get(c) == copy.get(c) for c in cols): + return f"{table}.{name} ({', '.join(cols)})" + return None def run_case( @@ -869,11 +935,12 @@ def run_case( ) return Outcome(case.name, case.cls, case.correct, outcome, passed, reasons[:6]) except Exception as exc: # noqa: BLE001 - a case that can't run admits nothing - if case.cls == "duplicate" and _unique_violation(exc): - # The store's own unique key refuses this copy: such a duplicate - # can't exist, so it is neither stopped nor passed. + blocked = blocked_by_constraint(sb, case, exc) + if blocked: + # The store's own unique index refuses this copy: such a + # duplicate can't exist, so it is neither stopped nor passed. return Outcome(case.name, case.cls, case.correct, "NOT_APPLICABLE", - False, [f"{type(exc).__name__}: {exc}"]) # fmt: skip + False, [f"impossible by constraint {blocked}"]) # fmt: skip return Outcome( case.name, case.cls, @@ -1045,9 +1112,22 @@ def summarize( if "update" in modes.get(k, set()): need += UPDATE_CLASSES planted = {r.cls for r in fl if r.outcome != "NOT_PLANTED"} + blocked = [r for r in rs if r.outcome == "NOT_APPLICABLE"] + impossible = sorted({r.reasons[0] for r in blocked if r.reasons}) + if ( + "duplicate" in need + and "duplicate" not in planted + and blocked + and all(r.cls == "duplicate" for r in blocked) + ): + # Every duplicate the kind's writes could make is refused by a + # unique index the schema declares: recorded, not required. + need.remove("duplicate") missing = [c for c in need if c not in planted] out[k] = { - "admitted": bool(correct) and all(r.ok for r in rs) and not missing, + "admitted": bool(correct) + and all(r.ok for r in rs if r.outcome != "NOT_APPLICABLE") + and not missing, "correct_writes": len(correct), "false_stops": [ f"{r.outcome}: {'; '.join(r.reasons[:2])}" for r in correct if not r.ok @@ -1056,6 +1136,7 @@ def summarize( "faults_stopped": sum(1 for r in fl if r.ok), "faults_passed": [f"{r.case} ({r.outcome})" for r in fl if not r.ok], "classes_not_planted": missing, + "duplicates_impossible_by_constraint": impossible, "by_class": { c: f"{sum(1 for r in fl if r.cls == c and r.ok)}/{sum(1 for r in fl if r.cls == c)}" for c in sorted({r.cls for r in fl}) diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py index 5d45158b31..7ba22302ef 100644 --- a/packages/flow/tests/test_zero_touch.py +++ b/packages/flow/tests/test_zero_touch.py @@ -412,3 +412,83 @@ def test_strict_pins_checks_both_readings_and_is_off_by_default() -> None: assert "'EUR'" in exprs["pin__cur"] and "`product`" in exprs["copy__cur"] with pytest.raises(gen.Unpinned): gen._pin(b, gen.Surface("record", "item", []), "item", "cur", role, {}, {}) + + +# -- a duplicate the store's own unique index refuses (review round 2) ------- + + +class _UniqueSandbox(_FakeSandbox): + def unique_indexes(self, table: str) -> dict[str, list[str]]: + return {"uk_res": ["fk_event", "kind", "fk_element"]} + + +def _dup_case(copy_changes: dict[str, Any]) -> Any: + row = {"id": "5", "fk_event": "46", "kind": "user", "fk_element": "2", "note": "a"} + copy = {**row, "id": "1005", **copy_changes} + return zt.Case("duplicate:res", "duplicate", [], copy_of=("res", row, copy)) + + +class _Mysql1062(Exception): + pass + + +def test_a_duplicate_refused_by_a_declared_unique_index_is_not_applicable() -> None: + exc = _Mysql1062(1062, "Duplicate entry '46-user-2' for key 'uk_res'") + got = zt.blocked_by_constraint(_UniqueSandbox({}), _dup_case({}), exc) + assert got and "uk_res" in got + + +def test_any_other_refusal_of_a_duplicate_is_still_a_failure() -> None: + sb = _UniqueSandbox({}) + unique = _Mysql1062(1062, "Duplicate entry '46-user-2' for key 'uk_res'") + # The copy changed a column of the index: the index doesn't prove it. + assert zt.blocked_by_constraint(sb, _dup_case({"fk_element": "3"}), unique) is None + # An index the schema doesn't declare. + other = _Mysql1062(1062, "Duplicate entry 'x' for key 'uk_other'") + assert zt.blocked_by_constraint(sb, _dup_case({}), other) is None + # Not a unique violation (a NOT NULL refusal). + assert ( + zt.blocked_by_constraint(sb, _dup_case({}), _Mysql1062(1048, "cannot be null")) + is None + ) + # Not a duplicate fault. + case = _dup_case({}) + case.cls = "bookkeeping" + assert zt.blocked_by_constraint(sb, case, unique) is None + + +def test_duplicates_impossible_by_constraint_are_recorded_not_required() -> None: + correct = zt.Outcome("correct", "correct", True, "CONFIRMED", True) + rest = [zt.Outcome(c, c, False, "REFUTED", False) + for c in (*zt.REQUIRED_CLASSES, "look_alike")] # fmt: skip + na = zt.Outcome("duplicate:res", "duplicate", False, "NOT_APPLICABLE", False, + ["impossible by constraint res.uk_res (fk_event, kind, fk_element)"]) # fmt: skip + s = zt.summarize({"k": [correct, *rest, na]}, ["k"], {"k": {"create"}})["k"] + assert s["admitted"] and s["duplicates_impossible_by_constraint"] + # One blocked, one that errored: not admitted. + err = zt.Outcome("duplicate:main", "duplicate", False, "ERROR", False) + s = zt.summarize({"k": [correct, *rest, na, err]}, ["k"], {"k": {"create"}})["k"] + assert not s["admitted"] + + +def test_unique_indexes_reads_the_catalog_on_postgresql(pg: Any) -> None: # noqa: F811 + admin = psycopg2.connect( + host="127.0.0.1", port=pg["port"], user="admin", dbname="postgres" + ) + admin.autocommit = True + admin.cursor().execute("CREATE DATABASE uq_probe") + admin.close() + c = psycopg2.connect( + host="127.0.0.1", port=pg["port"], user="admin", dbname="uq_probe" + ) + c.autocommit = True + c.cursor().execute( + "CREATE SCHEMA oa_sandbox; CREATE TABLE oa_sandbox.res (id int PRIMARY KEY, " + "a int, b text, c int); CREATE UNIQUE INDEX uk_res ON oa_sandbox.res (b, a); " + "CREATE INDEX plain ON oa_sandbox.res (c)" + ) + c.close() + read = {"sql_driver": "psycopg2", "sql_connect_args": {"host": "127.0.0.1", "port": pg["port"], + "user": "admin", "dbname": "uq_probe"}} # fmt: skip + sb = zt.Sandbox(read, read) + assert sb.unique_indexes("res") == {"uk_res": ["b", "a"]} From bf98b1bed2185d1da20659be5b94b299cb16185f Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 19:17:41 -0500 Subject: [PATCH 07/12] Zero-touch round 2: failed reads never count as stopped; found keeps only constants of strict roles; validated field sets only; key and unrelated-edit faults on creates --- .../generate/verify_contract.py | 27 ++- .../openadapt_flow/generate/zero_touch.py | 165 ++++++++++++++++-- packages/flow/tests/test_zero_touch.py | 146 +++++++++++++++- 3 files changed, 319 insertions(+), 19 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index 3138a9284a..29de6a9a5b 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -103,6 +103,21 @@ def _strict(roles: Mapping[str, ColumnRole], on: bool) -> dict[str, ColumnRole]: } +def _found_role(role: ColumnRole) -> Optional[ColumnRole]: + """The part of a role a ``found`` contract checks. Who made the record and + what it copied may differ for a record already there (a person entered + it), so ``actor`` and ``copied`` are left out, and a ``strict`` role keeps + only its constant half. None: nothing to check.""" + + if role.role in ("actor", "copied"): + return None + if role.role == "strict": + if "constant" not in role.detail: + return None + return ColumnRole("strict", detail={"constant": role.detail["constant"]}) + return role + + def _strict_pins( b: "SqlBuilder", surf: "Surface", table: str, col: str, detail: Mapping[str, Any], match: dict[str, Any], @@ -2011,8 +2026,10 @@ def build_contract( settings.strict_pins, ) for col, role in sorted(roles.items()): - if flavor == "found" and role.role in ("actor", "copied"): - continue + if flavor == "found": + role = _found_role(role) + if role is None: + continue _pin( b, main, @@ -2142,8 +2159,12 @@ def build_contract( else: m[key] = _value_expr(pname, loc.rule) for col, role in sorted(_strict(g.roles, settings.strict_pins).items()): - if role.role == "link" or (found and role.role in ("actor", "copied")): + if role.role == "link": continue + if found: + role = _found_role(role) + if role is None: + continue _pin( b, surf, diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index f23d276cd8..500aaa42db 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -709,6 +709,41 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: for col in changed: cls = "wrong_value" if _is_value(op.row.get(col), values) else "bookkeeping" column_fault(cls, i, op, col) + if is_new: + # A new row's own key, when the store doesn't fill it (a code, + # a user name): a wrong one is planted like any other value. + for col in t.primary_key: + c = t.column(col) + if c is None or c.auto: + continue + new = plant_value(sb, t, col, op.row.get(col)) + cls = ( + "wrong_value" + if _is_value(op.row.get(col), values) + else "bookkeeping" + ) + if new is None: + out.append(Case(f"{cls}:{op.table}.{col}", cls, [], + unplantable="no other key value can be planted")) # fmt: skip + continue + row = dict(op.row) + row[col] = new + key = {k: row.get(k) for k in t.primary_key} + out.append(Case(f"{cls}:{op.table}.{col}", cls, + write[:i] + [Op(op.table, key, row)] + write[i + 1 :])) # fmt: skip + # Each value the new row holds, set the same way on another row + # of the table: an excuse must hold only on the rows the write + # declares. + other = next((r for r in sb.rows(op.table, 20) + if store.row_key(t, r) != store.row_key(t, op.key)), None) # fmt: skip + if other is not None: + for col in changed: + if other.get(col) == op.row.get(col): + continue + edit = dict(other) + edit[col] = op.row.get(col) + out.append(Case(f"unrelated_edit:{op.table}.{col}", "unrelated_edit", + write + [Op(op.table, other, edit)])) # fmt: skip if not is_new and before is not None: for c in t.columns: if c.name not in t.primary_key and c.name not in changed: @@ -801,9 +836,22 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: # -- running a case --------------------------------------------------------------- #: Verdicts the verifier gives for a state it read and didn't pass. Only -#: these count as a stopped fault: an error, a missing contract or a fault -#: that couldn't be planted never does. +#: these count as a stopped fault: an error, a missing contract, a fault +#: that couldn't be planted or a read that failed (``READ_FAILED``) never +#: does. STOP_VERDICTS = ("REFUTED", "UNCERTAIN", "RECHECK_REFUTED", "RECHECK_UNCERTAIN") +#: A case whose verdict came from a failed read is run this many times in +#: all before it counts as not stopped. +READ_ATTEMPTS = 3 +_READ_FAILURE = re.compile(r"could(?:n't| not) be read|can't be read|still moving") + + +def read_failed(reasons: Sequence[str]) -> bool: + """The verifier gave its verdict because it couldn't read the store (a + timeout, a lost connection, rows still moving), not from the state it + read.""" + + return any(_READ_FAILURE.search(r) for r in reasons) @dataclass @@ -1044,6 +1092,29 @@ def _load_yaml(path: Path) -> Any: return yaml.safe_load(Path(path).read_text()) if path else None +def run_verdict( + sb: Sandbox, + manifest: Path, + demo: Demo, + case: Case, + actor_id: str, + extra: Mapping[str, Any], +) -> Outcome: + """:func:`run_case`, run again while its verdict came from a failed read, + up to :data:`READ_ATTEMPTS` times. A read that keeps failing gives + ``READ_FAILED``: the planted state was never judged, so it isn't + stopped.""" + + for _ in range(READ_ATTEMPTS): + res = run_case(sb, manifest, demo, case, actor_id, extra, + manifest.parent / "state") # fmt: skip + if not (res.outcome in STOP_VERDICTS and read_failed(res.reasons)): + return res + res.reasons = [f"read failed {READ_ATTEMPTS} times"] + res.reasons + res.outcome = "READ_FAILED" + return res + + def validate( sb: Sandbox, manifest: Path, @@ -1079,9 +1150,7 @@ def validate( finally: sb.put_many([Op(op.table, op.key, row) for op, row in saved]) for case in cases: - res = run_case( - sb, manifest, demo, case, actor_id, extra, manifest.parent / "state" - ) + res = run_verdict(sb, manifest, demo, case, actor_id, extra) log(f"{demo.name} {case.name}: {res.outcome} {'ok' if res.ok else 'FAIL'}") if record is not None: record.append({"demo": demo.name, "kind": demo.kind, "case": res.case, @@ -1251,23 +1320,79 @@ def fingerprint(manifest_dir: Path, kind: str) -> str: return hashlib.sha256(body.encode()).hexdigest() -def install(staged: Path, final: Path, admitted: Sequence[str]) -> None: +def field_set(values: Mapping[str, Any]) -> tuple[list[str], dict[str, int]]: + """A task's field set as the manifest writes it: its non-null scalar + fields, and its lists by length.""" + + given = sorted( + k for k, v in values.items() if v is not None and not isinstance(v, list) + ) + lists = {k: len(v) for k, v in values.items() if isinstance(v, list)} + return given, lists + + +def install( + staged: Path, + final: Path, + admitted: Sequence[str], + validated: Optional[ + Mapping[str, Sequence[tuple[list[str], dict[str, int]]]] + ] = None, +) -> None: """Keep only the admitted kinds in ``staged`` (manifest entries, pinned - digests and contract files), then move it to ``final`` in one rename.""" + digests and contract files), and of each kind only the field sets a + held-out write validated (``validated``: kind -> field sets; None keeps + every listed one). Contracts are never built at run time from a + zero-touch manifest: ``choose`` stops a task whose field set isn't + listed. Then move it to ``final`` in one rename.""" import shutil path = staged / "manifest.yaml" manifest = _load_yaml(path) or {} dropped = [k for k in list(manifest.get("kinds") or {}) if k not in admitted] + if validated is not None: + for k, entry in (manifest.get("kinds") or {}).items(): + if k in dropped: + continue + listed = [(sorted(v.get("fields") or []), dict(v.get("lists") or {})) + for v in entry.get("variants") or []] # fmt: skip + for given, lists in validated.get(k, []): + if (sorted(given), dict(lists)) in listed: + continue + # Validated through a contract built on demand: build the + # same one here, from the same pinned inputs, and list it. + try: + files = gen._field_set( + path, manifest, k, entry, set(given), dict(lists) + ) + except gen.GenerateError: + files = None + if files: + var: dict[str, Any] = {"fields": sorted(given), **files} + if lists: + var["lists"] = dict(lists) + entry.setdefault("variants", []).append(var) + listed.append((sorted(given), dict(lists))) + keep = [] + for var in entry.get("variants") or []: + fs = (sorted(var.get("fields") or []), dict(var.get("lists") or {})) + if fs in [(sorted(f), dict(lst)) for f, lst in validated.get(k, [])]: + keep.append(var) + else: + for flavor in ("create", "update", "found"): + if isinstance(var.get(flavor), str): + (staged / var[flavor]).unlink(missing_ok=True) + entry["variants"] = keep + # Dropped kinds go only now: building a field set checks every pinned + # file of the manifest as ``propose`` wrote it. for k in dropped: manifest["kinds"].pop(k) - for sub in ("contracts", "field_sets"): - shutil.rmtree(staged / sub / k, ignore_errors=True) - listed = (manifest.get("field_sets") or {}).get("listed_sha256") or {} - for rel in list(listed): - if not (staged / rel).exists(): - listed.pop(rel) + shutil.rmtree(staged / "contracts" / k, ignore_errors=True) + # No run-time builds: they would be contracts no sandbox validated. + manifest.pop("field_sets", None) + for k in dropped: + shutil.rmtree(staged / "field_sets" / k, ignore_errors=True) manifest["zero_touch"] = {"admitted": sorted(admitted)} path.write_text(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True), encoding="utf-8") # fmt: skip @@ -1393,7 +1518,10 @@ def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: s["why"] = ("the contracts proposed against the app's own store differ " "from the ones validated in the sandbox") # fmt: skip admitted = sorted(k for k, s in summary.items() if s["admitted"]) - install(staged, out / "contracts", admitted) + validated_sets: dict[str, list[tuple[list[str], dict[str, int]]]] = {} + for d in holdout: + validated_sets.setdefault(d.kind, []).append(field_set(d.values)) + install(staged, out / "contracts", admitted, validated_sets) result = { "minutes": round((time.time() - t0) / 60, 2), "person_decisions": 0, @@ -1402,9 +1530,16 @@ def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: "admitted": admitted, "kinds": summary, "exceptions": sum(1 for s in summary.values() if not s["admitted"]), + "validated_field_sets": validated_sets, + "sandbox_fk_errors": list(sb.fk_errors), } (out / "ONBOARDING.json").write_text(json.dumps(result, indent=2, default=str)) - (out / "EXCEPTIONS.md").write_text(exceptions_md(summary, prod)) + text = exceptions_md(summary, prod) + if sb.fk_errors: + text += ("\nForeign keys the sandbox couldn't copy (contracts that depend " + "on them differ from production and are not admitted):\n" + + "".join(f"- {e}\n" for e in sb.fk_errors)) # fmt: skip + (out / "EXCEPTIONS.md").write_text(text) (out / "VALIDATION.jsonl").write_text("".join(json.dumps(c) + "\n" for c in cases)) return result diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py index 7ba22302ef..5a432a902a 100644 --- a/packages/flow/tests/test_zero_touch.py +++ b/packages/flow/tests/test_zero_touch.py @@ -382,7 +382,7 @@ def test_s1_s2_install_keeps_only_admitted_kinds_and_fingerprints_ignore_connect zt.install(staged, tmp_path / "contracts", ["a"]) got = yaml.safe_load((tmp_path / "contracts" / "manifest.yaml").read_text()) assert list(got["kinds"]) == ["a"] - assert list(got["field_sets"]["listed_sha256"]) == ["contracts/a/x.create.yaml"] + assert "field_sets" not in got # no contracts built at run time assert not (tmp_path / "contracts" / "contracts" / "b").exists() assert not staged.exists() @@ -492,3 +492,147 @@ def test_unique_indexes_reads_the_catalog_on_postgresql(pg: Any) -> None: # noq "user": "admin", "dbname": "uq_probe"}} # fmt: skip sb = zt.Sandbox(read, read) assert sb.unique_indexes("res") == {"uk_res": ["b", "a"]} + + +# -- review round 2: R2-B1, R2-B2, S8-S10 ------------------------------------- + +READ_BACK_FAILED = ( + "the system of record could not be read back (OperationalError); " + "the write may still land" +) + + +def test_r2b1_a_verdict_from_a_failed_read_is_retried_then_fails( + monkeypatch: Any, +) -> None: + calls: list[int] = [] + + def fails_to_read(*a: Any, **k: Any) -> Any: + calls.append(1) + return zt.Outcome("bookkeeping:p.c15", "bookkeeping", False, "UNCERTAIN", False, + [READ_BACK_FAILED]) # fmt: skip + + monkeypatch.setattr(zt, "run_case", fails_to_read) + case = zt.Case("bookkeeping:p.c15", "bookkeeping", []) + res = zt.run_verdict(None, Path("m/manifest.yaml"), None, case, "a", {}) # type: ignore[arg-type] + assert len(calls) == zt.READ_ATTEMPTS + assert res.outcome == "READ_FAILED" and not res.ok + correct = zt.Outcome("correct", "correct", True, "CONFIRMED", True) + assert not zt.summarize({"k": [correct, res]}, ["k"])["k"]["admitted"] + + +def test_r2b1_a_read_that_recovers_gives_its_real_verdict(monkeypatch: Any) -> None: + seq = iter([ + zt.Outcome("x", "bookkeeping", False, "UNCERTAIN", False, [READ_BACK_FAILED]), + zt.Outcome("x", "bookkeeping", False, "REFUTED", False, ["record_written was refuted"]), + ]) # fmt: skip + monkeypatch.setattr(zt, "run_case", lambda *a, **k: next(seq)) + res = zt.run_verdict(None, Path("m/manifest.yaml"), None, # type: ignore[arg-type] + zt.Case("x", "bookkeeping", []), "a", {}) # fmt: skip + assert res.outcome == "REFUTED" and res.ok + + +def test_r2b2_found_keeps_only_the_constant_half_of_a_strict_role() -> None: + from openadapt_flow.generate.demo_analysis import ColumnRole + + copied = {"table": "q", "column": "owner", "ref_column": "id", "via": "q_id"} + assert ( + gen._found_role(ColumnRole("strict", detail={"actor": "bot", "copied": copied})) + is None + ) + kept = gen._found_role( + ColumnRole("strict", detail={"constant": "1", "copied": copied}) + ) + assert kept is not None and kept.detail == {"constant": "1"} + assert gen._found_role(ColumnRole("actor", detail={"param": "bot"})) is None + + +TICKETS = """ +CREATE TABLE queues (id serial PRIMARY KEY, name text NOT NULL, owner_id integer); +CREATE TABLE tickets (id serial PRIMARY KEY, queue_id integer REFERENCES queues(id), + subject text NOT NULL, owner_id integer); +INSERT INTO queues (name, owner_id) VALUES ('Ops', 1), ('Billing', 1), ('Desk', 1); +SELECT setval('tickets_id_seq', 100); +CREATE ROLE oa_t LOGIN PASSWORD 'reader-pw'; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO oa_t; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO oa_t; +""" + + +def test_r2b2_a_record_another_user_made_is_found_under_strict_pins( + pg: Any, # noqa: F811 + tmp_path: Path, + monkeypatch: Any, +) -> None: + """A ticket's owner is the acting user and a copy of its queue's owner in + every demonstration, so ``strict_pins`` checks both on a create. A ticket + a colleague already made (owner 2) must still be found, so the task + writes nothing and a second create is refuted.""" + + admin = psycopg2.connect( + host="127.0.0.1", port=pg["port"], user="admin", dbname="postgres" + ) + admin.autocommit = True + admin.cursor().execute("CREATE DATABASE tickets") + admin.close() + app = App(pg["port"], "tickets") + with app.con() as c, c.cursor() as cur: + cur.execute(TICKETS) + os.environ["OA_ZT_T_PW"] = "reader-pw" + read = {"sql_driver": "psycopg2", "sql_connect_args": {"host": "127.0.0.1", + "port": pg["port"], "user": "oa_t", "dbname": "tickets"}, + "sql_password_env": "OA_ZT_T_PW"} # fmt: skip + (tmp_path / "read.yaml").write_text(yaml.safe_dump(read)) + kinds = {"kinds": [{"kind": "create_ticket", "parameters": ["queue", "subject"], + "examples": [{"values": {"queue": "Ops", "subject": "a"}}]}]} # fmt: skip + (tmp_path / "kinds.yaml").write_text(yaml.safe_dump(kinds)) + common = ["--read", str(tmp_path / "read.yaml")] + for queue, subject in ( + ("Ops", "Badge reader"), + ("Billing", "Refund"), + ("Desk", "VPN"), + ): + assert ( + gen.main(["snapshot", *common, "--out", str(tmp_path / "before.json")]) == 0 + ) + app.q("INSERT INTO tickets (queue_id, subject, owner_id) SELECT id, %s, 1 " + "FROM queues WHERE name = %s", (subject, queue)) # fmt: skip + assert gen.main(["record", *common, "--before", str(tmp_path / "before.json"), + "--kind", "create_ticket", "--values", + json.dumps({"queue": queue, "subject": subject}), + "--demos", str(tmp_path / "demos")]) == 0 # fmt: skip + answers = {"strict_pins": True, "bookkeeping": "accept", "recheck_after_s": 300} + gen.propose(read=read, kinds=gen.load_kinds(tmp_path / "kinds.yaml"), + demo_dir=tmp_path / "demos", out=tmp_path / "out", + actor_params={"bot_user_id": "1"}, answers=answers) # fmt: skip + create = next((tmp_path / "out" / "contracts").rglob("*.create.yaml")).read_text() + assert "copy__owner_id" in create # strict: the copy is checked on a create + # A colleague (user 2) already made this ticket. + app.q("INSERT INTO tickets (queue_id, subject, owner_id) SELECT id, 'Printer', 2 " + "FROM queues WHERE name = 'Ops'") # fmt: skip + loaded: list[Path] = [] + real = gen._load_for + + def record(path: Path, params: Any) -> Any: + loaded.append(Path(path)) + return real(path, params) + + monkeypatch.setattr(gen, "_load_for", record) + gen.choose(tmp_path / "out" / "manifest.yaml", "create_ticket", + {"queue": "Ops", "subject": "Printer"}, {"bot_user_id": "1"}) # fmt: skip + # The found contract is read first; it must also be the one returned. + assert loaded and loaded[-1].name.endswith(".found.yaml"), loaded + + +def test_s9_s10_a_new_rows_own_key_and_each_value_on_another_row_are_faulted() -> None: + from openadapt_flow.generate.sql_store import Column, Schema, Table + + t = Table("codes", [Column("code", "varchar", False, None), Column("label", "varchar", True, None)], + ["code"]) # fmt: skip + schema = Schema("mysql", "app", {"codes": t}) + sb = _FakeSandbox({"codes": [{"code": "AA", "label": "old"}]}) + cases = zt.faults(schema, sb, type("D", (), {"values": {"code": "BB"}})(), + [zt.Op("codes", {"code": "BB"}, {"code": "BB", "label": "new"})]) # fmt: skip + names = {c.name for c in cases} + assert "wrong_value:codes.code" in names # S9 + assert "unrelated_edit:codes.label" in names # S10 From 29886dd2d0f8477a4b67f407a92850969d1885e7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 19:20:49 -0500 Subject: [PATCH 08/12] Zero-touch: plants avoid declared unique keys; failing cases log their reason --- .../openadapt_flow/generate/zero_touch.py | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index 500aaa42db..f522186967 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -304,8 +304,15 @@ def rows(self, table: str, limit: int = 50) -> list[dict[str, Optional[str]]]: def unique_indexes(self, table: str) -> dict[str, list[str]]: """Unique indexes of a sandbox table other than its primary key: - name -> columns in index order, from the catalog.""" + name -> columns in index order, from the catalog (read once per + table).""" + cache = self.__dict__.setdefault("_unique", {}) + if table not in cache: + cache[table] = self._read_unique(table) + return cache[table] + + def _read_unique(self, table: str) -> dict[str, list[str]]: if self.dialect == "mysql": got = self._exec( "SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS " @@ -595,6 +602,28 @@ def plant_value( return mutate(table, column, current) +def unique_columns(sb: "Sandbox", table: str) -> set[str]: + """Columns in any unique index of ``table`` other than its primary key + (none when the sandbox can't say).""" + + get = getattr(sb, "unique_indexes", None) + if get is None: + return set() + return {c for cols in get(table).values() for c in cols} + + +def edit_value( + sb: "Sandbox", t: Table, col: str, current: Optional[str], written: Optional[str] +) -> Optional[str]: + """The value an unrelated edit sets on another row: what the write set + (the same change elsewhere), except in a column of a unique index, where + that value would make the store refuse the edit; there, a planted value.""" + + if col in unique_columns(sb, t.name): + return plant_value(sb, t, col, current) + return written + + def _text_values(values: Mapping[str, Any]) -> set[str]: out: set[str] = set() @@ -738,10 +767,11 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: if store.row_key(t, r) != store.row_key(t, op.key)), None) # fmt: skip if other is not None: for col in changed: - if other.get(col) == op.row.get(col): + v = edit_value(sb, t, col, other.get(col), op.row.get(col)) + if v is None or other.get(col) == v: continue edit = dict(other) - edit[col] = op.row.get(col) + edit[col] = v out.append(Case(f"unrelated_edit:{op.table}.{col}", "unrelated_edit", write + [Op(op.table, other, edit)])) # fmt: skip if not is_new and before is not None: @@ -764,10 +794,11 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: # The same change on another row: an excuse must hold only # on the rows the write declares. for c in changed: - if other.get(c) == op.row.get(c): + v = edit_value(sb, t, c, other.get(c), op.row.get(c)) + if v is None or other.get(c) == v: continue edit = dict(other) - edit[c] = op.row.get(c) + edit[c] = v out.append(Case(f"unrelated_edit:{op.table}.{c}", "unrelated_edit", write + [Op(op.table, other, edit)])) # fmt: skip # A late change to the target itself: its first changed column. @@ -825,8 +856,13 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: key = _new_key(sb, t, sample[0]) if sample else None if key is None: continue + row = _copy_row(t, sample[0], key) + for col in unique_columns(sb, name) - set(key): + v = plant_value(sb, t, col, row.get(col)) + if v is not None: + row[col] = v # a copied unique code would be refused out.append(Case(f"other_table:{name}", "other_table", - write + [Op(name, key, _copy_row(t, sample[0], key))])) # fmt: skip + write + [Op(name, key, row)])) # fmt: skip planted += 1 if planted >= 2: break @@ -1151,7 +1187,8 @@ def validate( sb.put_many([Op(op.table, op.key, row) for op, row in saved]) for case in cases: res = run_verdict(sb, manifest, demo, case, actor_id, extra) - log(f"{demo.name} {case.name}: {res.outcome} {'ok' if res.ok else 'FAIL'}") + log(f"{demo.name} {case.name}: {res.outcome} {'ok' if res.ok else 'FAIL'}" + + ("" if res.ok or not res.reasons else f" | {res.reasons[0][:200]}")) # fmt: skip if record is not None: record.append({"demo": demo.name, "kind": demo.kind, "case": res.case, "class": res.cls, "correct": res.correct, "outcome": res.outcome, From 619e627a38f02848a11422d4246f66b6d602ac83 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 20:17:51 -0500 Subject: [PATCH 09/12] Zero-touch design: failed reads, forbidden duplicates, validated field sets, strict pins --- .../flow/docs/research/ZERO_TOUCH_DESIGN.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md index f0dc148310..f89f7d3065 100644 --- a/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md +++ b/packages/flow/docs/research/ZERO_TOUCH_DESIGN.md @@ -109,6 +109,31 @@ or the recheck). A plant the store rejects, a case that errors, a fault with no contract and an unplantable column all count as not stopped, so the kind is not admitted. +**A failed read is not a verdict.** An `UNCERTAIN` that came from a read +the verifier couldn't make (a timeout, a lost connection, rows still +moving) is run again, up to three times; if the read keeps failing, the +case is `READ_FAILED` and doesn't count as stopped. + +**A duplicate the store forbids.** A `duplicate` the store refuses counts +as not applicable only when the refusal is a unique violation of an index +the catalog declares unique (not the primary key) and every column of that +index holds the same value in the copy as in the row copied. Any other +refusal is a failure. A kind whose every duplicate is refused this way +records "duplicates impossible by constraint" with the index names instead +of a planted duplicate. Other plants avoid declared unique indexes, so the +store takes them. + +**Only validated field sets.** A zero-touch manifest lists only the field +sets a held-out write validated (built during setup when needed) and builds +none at run time: a task with another field set stops. + +**Two readings, both checked.** The setup answers `strict_pins: true`, so a +new-row column two readings fit in every demonstration (a constant and a +copy, the acting user and a copy, or one demonstration's one value) is +checked both ways instead of refusing the kind. A `found` contract (the +record is already there) keeps only the constant half: who made the record +may differ. + **Required classes.** A kind is admitted only if its held-out writes produced `missing`, `unrelated_edit`, `delayed` and `other_table` faults, plus `duplicate` and `look_alike` for a create and `wrong_record` for an From 209700223c71e57ca79333c7c34c15de0f040d14 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 21:14:59 -0500 Subject: [PATCH 10/12] Zero-touch round 3: fresh app-made unique values in duplicates; transactional restore stops the setup; structured read evidence; plain unique indexes only --- .../openadapt_flow/generate/zero_touch.py | 258 +++++++++++++----- packages/flow/tests/test_zero_touch.py | 144 +++++++++- 2 files changed, 339 insertions(+), 63 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index f522186967..0c76bc3c03 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -88,6 +88,12 @@ class ZeroTouchError(RuntimeError): """The setup can't run as asked (no sandbox, a bad config).""" +class SandboxBroken(ZeroTouchError): + """The sandbox couldn't be put back after a case: it no longer mirrors + the app's store, so nothing validated in it can be admitted, and the + setup stops.""" + + # -- the sandbox -------------------------------------------------------------------- @@ -313,29 +319,39 @@ def unique_indexes(self, table: str) -> dict[str, list[str]]: return cache[table] def _read_unique(self, table: str) -> dict[str, list[str]]: + """Plain unique indexes only: one that is partial, holds an + expression or indexes a prefix of a column is left out, so it can + never be taken as proof that a duplicate is impossible.""" + if self.dialect == "mysql": got = self._exec( - "SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS " - "WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND NON_UNIQUE = 0 " - "AND INDEX_NAME <> 'PRIMARY' ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "SELECT INDEX_NAME, COLUMN_NAME, SUB_PART FROM " + "information_schema.STATISTICS WHERE TABLE_SCHEMA = %s AND " + "TABLE_NAME = %s AND NON_UNIQUE = 0 AND INDEX_NAME <> 'PRIMARY' " + "ORDER BY INDEX_NAME, SEQ_IN_INDEX", (self.name, table), ) else: got = self._exec( - "SELECT i.relname, a.attname FROM pg_index x " + "SELECT i.relname, a.attname, NULL FROM pg_index x " "JOIN pg_class c ON c.oid = x.indrelid " "JOIN pg_namespace n ON n.oid = c.relnamespace " "JOIN pg_class i ON i.oid = x.indexrelid " "JOIN LATERAL unnest(x.indkey) WITH ORDINALITY k(attnum, ord) ON true " - "JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum " + "LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum " "WHERE n.nspname = %s AND c.relname = %s AND x.indisunique " - "AND NOT x.indisprimary ORDER BY i.relname, k.ord", + "AND NOT x.indisprimary AND x.indpred IS NULL " + "AND x.indexprs IS NULL ORDER BY i.relname, k.ord", (self.name, table), ) - out: dict[str, list[str]] = {} - for name, col in got: - out.setdefault(str(name), []).append(str(col)) - return out + plain: dict[str, list[str]] = {} + odd: set[str] = set() + for name, col, sub in got: + if col is None or sub is not None: + odd.add(str(name)) + else: + plain.setdefault(str(name), []).append(str(col)) + return {k: v for k, v in plain.items() if k not in odd} def distinct(self, table: str, column: str, limit: int = 5) -> list[str]: """Up to ``limit`` values the column holds, as text.""" @@ -357,18 +373,63 @@ def max_key(self, table: str) -> Optional[int]: return int(got[0][0] or 0) def put_many(self, ops: Sequence["Op"]) -> None: - """Put each op's row, in two phases: every key is deleted first, then - every row inserted, so rows swapping a unique key don't collide on - the way. The last op for a key wins.""" + """Put each op's row in one transaction: every key is deleted first, + then every row inserted, so rows swapping a unique key don't collide + on the way, and a failure leaves the sandbox as it was. The last op + for a key wins.""" + self._guard() last: dict[tuple[str, str], Op] = {} for op in ops: last[(op.table, store.row_key(self._table(op.table), op.key))] = op - for op in last.values(): - self.put(op.table, None, op.key) - for op in last.values(): - if op.row is not None: - self.put(op.table, op.row, op.key) + if not last: + return + conn = self._connect() + try: + if self.dialect == "mysql": + conn.autocommit(False) # pymysql: a method + else: + conn.autocommit = False + cur = conn.cursor() + self._no_key_checks(cur) + for op in last.values(): + self._delete(cur, op.table, op.key) + for op in last.values(): + if op.row is not None: + self._insert(cur, op.table, op.row) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + def _no_key_checks(self, cur: Any) -> None: + # A planted fault may break a foreign key on purpose (a row the + # write points to is gone): the store must take it. + if self.dialect == "mysql": + cur.execute("SET FOREIGN_KEY_CHECKS=0") + return + cur.execute("SAVEPOINT oa_role") + try: # a superuser only; else a key-breaking fault fails closed + cur.execute("SET LOCAL session_replication_role = replica") + cur.execute("RELEASE SAVEPOINT oa_role") + except Exception: # noqa: BLE001 + cur.execute("ROLLBACK TO SAVEPOINT oa_role") + + def _delete(self, cur: Any, table: str, key: Mapping[str, Optional[str]]) -> None: + where, args = self._where(self._table(table), key) + cur.execute(f"DELETE FROM {self.t(table)} WHERE {where}", tuple(args)) + + def _insert(self, cur: Any, table: str, row: Mapping[str, Optional[str]]) -> None: + t = self._table(table) + cols = [c for c in t.column_names if c in row] + over = "" if self.dialect == "mysql" else " OVERRIDING SYSTEM VALUE" + cur.execute( + f"INSERT INTO {self.t(table)} ({', '.join(self.q(c) for c in cols)})" + f"{over} VALUES ({', '.join(['%s'] * len(cols))})", + tuple(_db_value(t, c, row.get(c)) for c in cols), + ) def put( self, @@ -378,32 +439,7 @@ def put( ) -> None: """Make the row with ``key``'s primary key be ``row`` (None: absent).""" - self._guard() - t = self._table(table) - where, args = self._where(t, key) - conn = self._connect() - try: - cur = conn.cursor() - # A planted fault may break a foreign key on purpose (a row the - # write points to is gone): the store must take it. - if self.dialect == "mysql": - cur.execute("SET FOREIGN_KEY_CHECKS=0") - else: - try: # a superuser only; else a key-breaking fault fails closed - cur.execute("SET session_replication_role = replica") - except Exception: # noqa: BLE001 - pass - cur.execute(f"DELETE FROM {self.t(table)} WHERE {where}", tuple(args)) - if row is not None: - cols = [c for c in t.column_names if c in row] - over = "" if self.dialect == "mysql" else " OVERRIDING SYSTEM VALUE" - cur.execute( - f"INSERT INTO {self.t(table)} ({', '.join(self.q(c) for c in cols)})" - f"{over} VALUES ({', '.join(['%s'] * len(cols))})", - tuple(_db_value(t, c, row.get(c)) for c in cols), - ) - finally: - conn.close() + self.put_many([Op(table, dict(key), dict(row) if row is not None else None)]) def _db_value(table: Table, column: str, text: Optional[str]) -> Any: @@ -515,7 +551,7 @@ class Case: #: For a ``duplicate``: the table, and the row copied with the copy, so a #: refusal by a unique key can be checked against the schema. copy_of: Optional[ - tuple[str, dict[str, Optional[str]], dict[str, Optional[str]]] + tuple[str, dict[str, Optional[str]], dict[str, Optional[str]], set[str]] ] = None @@ -612,6 +648,22 @@ def unique_columns(sb: "Sandbox", table: str) -> set[str]: return {c for cols in get(table).values() for c in cols} +def fresh_value( + sb: "Sandbox", t: Table, col: str, current: Optional[str] +) -> Optional[str]: + """A value for a column of a unique index that no row holds: made by + :func:`mutate` (another row's value would collide), changed again until + the column doesn't hold it, a few times at most.""" + + taken = set(sb.distinct(t.name, col, 1000)) + v = mutate(t, col, current) + for _ in range(5): + if v is None or v not in taken: + return v + v = mutate(t, col, v) + return None + + def edit_value( sb: "Sandbox", t: Table, col: str, current: Optional[str], written: Optional[str] ) -> Optional[str]: @@ -620,7 +672,7 @@ def edit_value( that value would make the store refuse the edit; there, a planted value.""" if col in unique_columns(sb, t.name): - return plant_value(sb, t, col, current) + return fresh_value(sb, t, col, current) return written @@ -816,9 +868,18 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: unplantable="no new key can be made for a copy")) # fmt: skip else: dup = _copy_row(t, op.row, key) + # A unique value the app makes (an invoice or ticket number) + # is made again for a second submit: plant a fresh one, so + # the copy is a duplicate the store accepts. + for col in unique_columns(sb, op.table) - set(key): + if not _is_value(dup.get(col), values): + v = fresh_value(sb, t, col, dup.get(col)) + if v is not None: + dup[col] = v + given = {c for c, v in op.row.items() if _is_value(v, values)} out.append(Case(f"duplicate:{op.table}", "duplicate", write + [Op(op.table, key, dup)], - copy_of=(op.table, dict(op.row), dup))) # fmt: skip + copy_of=(op.table, dict(op.row), dup, given))) # fmt: skip # The values land on an existing look-alike instead. for other in sb.rows(op.table, 20): look = dict(op.row) @@ -858,7 +919,7 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: continue row = _copy_row(t, sample[0], key) for col in unique_columns(sb, name) - set(key): - v = plant_value(sb, t, col, row.get(col)) + v = fresh_value(sb, t, col, row.get(col)) if v is not None: row[col] = v # a copied unique code would be refused out.append(Case(f"other_table:{name}", "other_table", @@ -879,7 +940,33 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: #: A case whose verdict came from a failed read is run this many times in #: all before it counts as not stopped. READ_ATTEMPTS = 3 -_READ_FAILURE = re.compile(r"could(?:n't| not) be read|can't be read|still moving") +_READ_FAILURE = re.compile( + r"could(?:n't| not) be read|can't be read|still (?:moving|changing)" + r"|no readable baseline|could not be judged again" +) + + +def write_judged(r: Any) -> bool: + """``verify_write`` read the state it judged: a receipt was issued, the + frame was read, and the effect was observed (present, absent or + conflicting, not unknown).""" + + frame = dict((getattr(r, "evidence", None) or {}).get("frame") or {}) + return ( + bool(getattr(r, "receipt", None)) + and bool(frame) + and frame.get("status") != "not_checked" + and getattr(r, "observed", "unknown") != "unknown" + ) + + +def recheck_judged(rc: Any) -> bool: + """The recheck read the database and judged its frame.""" + + return bool(getattr(rc, "read_at", "")) and getattr(rc, "frame", "") in ( + "held", + "refuted", + ) def read_failed(reasons: Sequence[str]) -> bool: @@ -898,6 +985,11 @@ class Outcome: outcome: str passed: bool reasons: list[str] = field(default_factory=list) + #: The verifier read the planted state and judged it (structured + #: evidence: a receipt, a frame read, an observed effect; at the + #: recheck, a read and a judged frame). False: an UNCERTAIN from a read + #: that failed, which is never a stopped fault. + judged: bool = True @property def ok(self) -> bool: @@ -906,7 +998,7 @@ def ok(self) -> bool: if self.correct: return self.passed - return not self.passed and self.outcome in STOP_VERDICTS + return not self.passed and self.outcome in STOP_VERDICTS and self.judged def _put_all(sb: Sandbox, ops: Sequence[Op]) -> None: @@ -924,7 +1016,7 @@ def _unique_violation(exc: BaseException) -> Optional[str]: args = getattr(exc, "args", ()) if args and args[0] == 1062: m = re.search(r"for key '([^']+)'", str(args[-1])) - return m.group(1).split(".")[-1] if m else "" + return m.group(1) if m else "" text = str(exc) if text.startswith("UNIQUE constraint failed"): return "" @@ -936,22 +1028,29 @@ def blocked_by_constraint( ) -> Optional[str]: """The unique index that makes ``case``'s duplicate impossible, or None. - Only when the store refused the copy for a unique violation, the - violated index is one the schema declares unique on that table (not the - primary key), and every column of that index holds the same value in the - copy as in the row it copies: then no duplicate of that row can be - stored at all.""" + Only when the store refused the copy for a unique violation; the + violated index is a plain unique index of that table the catalog + declares (not the primary key, not partial, no expression); every + column of it holds a value the task gave; and each holds the same value + in the copy as in the row it copies. Then the same task can't store a + second such row. An index on a value the app makes (a number it + assigns) proves nothing: that value is planted fresh, so the store takes + the copy.""" if case.cls != "duplicate" or case.copy_of is None: return None name = _unique_violation(exc) if not name: return None - table, row, copy = case.copy_of + table, row, copy, given = case.copy_of + if "." in name: + owner, name = name.rsplit(".", 1) + if owner.split(".")[-1].strip('`"') != table: + return None cols = sb.unique_indexes(table).get(name) if not cols: return None - if all(row.get(c) == copy.get(c) for c in cols): + if all(c in given and row.get(c) == copy.get(c) for c in cols): return f"{table}.{name} ({', '.join(cols)})" return None @@ -1001,6 +1100,7 @@ def run_case( outcome = str(getattr(r.outcome, "value", r.outcome)) reasons = list(r.reasons) passed = outcome == "CONFIRMED" + judged = outcome != "UNCERTAIN" or write_judged(r) if passed and getattr(r, "recheck", None) is not None: _put_all(sb, case.late) rc = recheck( @@ -1010,6 +1110,7 @@ def run_case( if rc_out != "CONFIRMED": passed, outcome = False, f"RECHECK_{rc_out}" reasons += list(rc.reasons) + judged = rc_out != "UNCERTAIN" or recheck_judged(rc) elif passed and case.late: # No recheck is scheduled, so the late change is never read: the # fault passes, which fails validation. @@ -1017,7 +1118,9 @@ def run_case( reasons.append( "the contract schedules no recheck, so a late write can't be seen" ) - return Outcome(case.name, case.cls, case.correct, outcome, passed, reasons[:6]) + return Outcome( + case.name, case.cls, case.correct, outcome, passed, reasons[:6], judged + ) except Exception as exc: # noqa: BLE001 - a case that can't run admits nothing blocked = blocked_by_constraint(sb, case, exc) if blocked: @@ -1039,7 +1142,7 @@ def run_case( [Op(table, key, row) for (table, _), (key, row) in saved.items()] ) except Exception as exc: # noqa: BLE001 - the sandbox is no longer known - raise ZeroTouchError(f"the sandbox could not be restored: {exc}") from exc + raise SandboxBroken(f"the sandbox could not be restored: {exc}") from exc # -- answers ---------------------------------------------------------------------- @@ -1144,7 +1247,10 @@ def run_verdict( for _ in range(READ_ATTEMPTS): res = run_case(sb, manifest, demo, case, actor_id, extra, manifest.parent / "state") # fmt: skip - if not (res.outcome in STOP_VERDICTS and read_failed(res.reasons)): + if not ( + res.outcome in STOP_VERDICTS + and (not res.judged or read_failed(res.reasons)) + ): return res res.reasons = [f"read failed {READ_ATTEMPTS} times"] + res.reasons res.outcome = "READ_FAILED" @@ -1184,7 +1290,12 @@ def validate( try: cases += faults(sb.schema, sb, demo, write) finally: - sb.put_many([Op(op.table, op.key, row) for op, row in saved]) + try: + sb.put_many([Op(op.table, op.key, row) for op, row in saved]) + except Exception as exc: # noqa: BLE001 - the sandbox is no longer known + raise SandboxBroken( + f"the sandbox could not be restored: {exc}" + ) from exc for case in cases: res = run_verdict(sb, manifest, demo, case, actor_id, extra) log(f"{demo.name} {case.name}: {res.outcome} {'ok' if res.ok else 'FAIL'}" @@ -1472,6 +1583,7 @@ def onboard( sb.create(schema, reader) sread = sb.read_config() work = out / "sandbox" + broken = "" try: report = gen.propose(read=sread, kinds=kinds, demo_dir=demo_dir, out=work / "q", actor_params=actor_params, answers={}) # fmt: skip @@ -1511,6 +1623,8 @@ def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: modes: dict[str, set[str]] = {} by_kind = validate(sb, work / tag / "manifest.yaml", holdout, actor_id, dict(actor_params), log, cases, modes) # fmt: skip + except SandboxBroken: + raise # the whole setup stops: see onboard except (gen.GenerateError, ZeroTouchError) as exc: # The answers don't fit together, or the sandbox broke: # nothing is admitted. @@ -1537,9 +1651,29 @@ def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: answers, ledger = chosen["answers"], chosen["ledger"] summary, cases = chosen["summary"], chosen["cases"] validated = chosen["dir"] + except SandboxBroken as exc: + broken = str(exc) finally: if not keep_sandbox: sb.drop() + if broken: + # Nothing validated in a sandbox that stopped mirroring the store + # can be trusted: no contract is installed, and every kind says why. + import shutil + + for d in (out / "contracts", out / ".contracts.staged"): + shutil.rmtree(d, ignore_errors=True) + why = f"setup stopped: {broken}" + summary = {k: {"admitted": False, "why": why, "correct_writes": 0, + "false_stops": [], "faults": 0, "faults_stopped": 0, + "faults_passed": [], "classes_not_planted": [], + "by_class": {}} for k in kind_names} # fmt: skip + result = {"minutes": round((time.time() - t0) / 60, 2), "person_decisions": 0, + "answers": {}, "ledger": [], "admitted": [], "kinds": summary, + "exceptions": len(summary), "stopped": broken} # fmt: skip + (out / "ONBOARDING.json").write_text(json.dumps(result, indent=2, default=str)) + (out / "EXCEPTIONS.md").write_text(exceptions_md(summary, {})) + return result # The production contracts: the same answers, against the app's own # store, staged beside the final folder. A kind is kept only when its # contracts are the validated ones, connections and windows aside. diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py index 5a432a902a..61c7c72336 100644 --- a/packages/flow/tests/test_zero_touch.py +++ b/packages/flow/tests/test_zero_touch.py @@ -425,7 +425,8 @@ def unique_indexes(self, table: str) -> dict[str, list[str]]: def _dup_case(copy_changes: dict[str, Any]) -> Any: row = {"id": "5", "fk_event": "46", "kind": "user", "fk_element": "2", "note": "a"} copy = {**row, "id": "1005", **copy_changes} - return zt.Case("duplicate:res", "duplicate", [], copy_of=("res", row, copy)) + given = {"fk_event", "kind", "fk_element"} # the task gave the index's values + return zt.Case("duplicate:res", "duplicate", [], copy_of=("res", row, copy, given)) class _Mysql1062(Exception): @@ -636,3 +637,144 @@ def test_s9_s10_a_new_rows_own_key_and_each_value_on_another_row_are_faulted() - names = {c.name for c in cases} assert "wrong_value:codes.code" in names # S9 assert "unrelated_edit:codes.label" in names # S10 + + +# -- review round 3: R3-B1, R3-B2, S11, S12 ----------------------------------- + + +def test_r3b1_an_index_on_a_value_the_app_makes_never_proves_a_duplicate_impossible() -> ( + None +): + exc = _Mysql1062(1062, "Duplicate entry '46-user-2' for key 'uk_res'") + case = _dup_case({}) + table, row, copy, _ = case.copy_of + case.copy_of = (table, row, copy, {"fk_event", "kind"}) # fk_element: the app's + assert zt.blocked_by_constraint(_UniqueSandbox({}), case, exc) is None + # N5: another table's index of the same name. + other = _Mysql1062(1062, "Duplicate entry 'x' for key 'other_table.uk_res'") + assert zt.blocked_by_constraint(_UniqueSandbox({}), _dup_case({}), other) is None + + +def test_s11_an_uncertain_without_a_judged_read_is_not_stopped() -> None: + class R: + receipt = {"outcome": "reconciliation_required"} + evidence = {"frame": {"status": "not_checked"}} + observed = "present" + + assert not zt.write_judged(R()) + R.evidence = {"frame": {"status": "held"}} + assert zt.write_judged(R()) + R.observed = "unknown" + assert not zt.write_judged(R()) + + class RC: + read_at = "" + frame = "not_checked" + + assert not zt.recheck_judged(RC()) + RC.read_at, RC.frame = "2026-11-03T20:00:00Z", "refuted" + assert zt.recheck_judged(RC()) + out = zt.Outcome( + "x", "bookkeeping", False, "UNCERTAIN", False, ["a judgment"], False + ) + assert not out.ok + for text in ("the database was still changing after the save", + "no readable baseline, so what else changed can't be known", + "the database could not be judged again"): # fmt: skip + assert zt.read_failed([text]) + + +def _uq_sandbox(pg: Any, db: str) -> Any: # noqa: F811 + admin = psycopg2.connect( + host="127.0.0.1", port=pg["port"], user="admin", dbname="postgres" + ) + admin.autocommit = True + admin.cursor().execute(f"CREATE DATABASE {db}") + admin.close() + c = psycopg2.connect(host="127.0.0.1", port=pg["port"], user="admin", dbname=db) + c.autocommit = True + c.cursor().execute( + "CREATE TABLE res (id integer PRIMARY KEY, code text NOT NULL, name text, tag text);" + "CREATE UNIQUE INDEX uk_code ON res (code);" + "CREATE UNIQUE INDEX uk_name ON res (name);" + "CREATE UNIQUE INDEX uk_partial ON res (tag) WHERE name IS NOT NULL;" + "CREATE UNIQUE INDEX uk_expr ON res (lower(tag));" + "INSERT INTO res VALUES (1, 'INV-1', 'Ann', 'a');" + ) + c.close() + read = {"sql_driver": "psycopg2", "sql_connect_args": {"host": "127.0.0.1", + "port": pg["port"], "user": "admin", "dbname": db}} # fmt: skip + sb = zt.Sandbox(read, read) + sb.create(zt.store.read_schema(zt.store.connector(read), "postgresql"), "admin") + return sb + + +def test_r3b1_s12_on_postgresql(pg: Any) -> None: # noqa: F811 + sb = _uq_sandbox(pg, "uq_r3") + try: + # S12: partial and expression indexes are not taken as proof. + # (The sandbox's copies of the indexes get names of their own.) + assert sorted(sb.unique_indexes("res").values()) == [["code"], ["name"]] + # tag NULL: the expression index can't refuse the copy (a refusal by + # it would not count as proof, so this checks uk_name alone). + new = {"id": "2", "code": "INV-2", "name": "Bob", "tag": None} + write = [zt.Op("res", {"id": "2"}, new)] + + def dup_of(values: dict[str, Any]) -> Any: + cases = zt.faults(sb.schema, sb, type("D", (), {"values": values})(), write) + return next(c for c in cases if c.cls == "duplicate") + + # The task gives the name; the app makes the code: the copy gets a + # fresh code, keeps the name, and only uk_name refuses it. + case = dup_of({"name": "Bob"}) + assert case.write[-1].row["code"] != "INV-2" + assert case.write[-1].row["name"] == "Bob" + with pytest.raises(Exception) as err: + sb.put_many(case.write) + blocked = zt.blocked_by_constraint(sb, case, err.value) + assert blocked and blocked.startswith("res.") and blocked.endswith("(name)") + # Neither is the task's: both are made fresh, nothing proves the + # duplicate impossible, and the store takes it. + case = dup_of({}) + assert case.write[-1].row["name"] != "Bob" + sb.put_many(case.write) # the store takes it: a normal duplicate + finally: + sb.drop() + + +def test_r3b2_a_restore_that_fails_midway_changes_nothing(pg: Any) -> None: # noqa: F811 + sb = _uq_sandbox(pg, "uq_tx") + try: + before = sb.rows("res") + bad = [zt.Op("res", {"id": "1"}, {"id": "1", "code": "INV-9", "name": "Ann", "tag": "a"}), + zt.Op("res", {"id": "3"}, {"id": "3", "code": None, "name": "X", "tag": "x"})] # fmt: skip + with pytest.raises(Exception): + sb.put_many(bad) # the second row breaks NOT NULL + assert sb.rows("res") == before + finally: + sb.drop() + + +def test_r3b2_a_broken_sandbox_stops_the_setup( + onboarded: dict[str, Any], + pg: Any, # noqa: F811 + monkeypatch: Any, # noqa: F811 +) -> None: + def broken(*a: Any, **k: Any) -> Any: + raise zt.SandboxBroken("the sandbox could not be restored: injected") + + monkeypatch.setattr(zt, "run_case", broken) + work = onboarded["out"].parent + read = yaml.safe_load((work / "read.yaml").read_text()) + admin_read = {"sql_driver": "psycopg2", "sql_connect_args": {"host": "127.0.0.1", + "port": pg["port"], "user": "admin", "dbname": "tracker_zt"}} # fmt: skip + out = work / "out_broken" + (out / "contracts").mkdir(parents=True) # a stale install must not survive + result = zt.onboard(read=read, admin=admin_read, reader="oa_reader", + kinds_path=work / "kinds.yaml", demo_dir=work / "demos", + holdout_dir=work / "holdout", out=out, + actor_params={"bot_user_id": str(BOT)}, actor_id="ops-bot", + log=lambda s: None) # fmt: skip + assert result["admitted"] == [] and "injected" in result["stopped"] + assert not (out / "contracts").exists() + assert "setup stopped" in (out / "EXCEPTIONS.md").read_text() From 81b5861d08270c9af10b448d34ce489081a1ea08 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 21:58:37 -0500 Subject: [PATCH 11/12] Zero-touch round 4: task-given columns come from the generator's bindings, not value matches; stopped setup keeps old contracts aside and exits non-zero --- .../openadapt_flow/generate/zero_touch.py | 53 ++++++++++++-- packages/flow/tests/test_zero_touch.py | 70 ++++++++++++++++++- 2 files changed, 115 insertions(+), 8 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index 0c76bc3c03..1df82473b0 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -742,7 +742,36 @@ def _copy_row( return out -def faults(schema: Schema, sb: "Sandbox", demo: Demo, write: list[Op]) -> list[Case]: +_BOUND = ( + re.compile(r"^([A-Za-z0-9_$]+)\.([A-Za-z0-9_$]+)\b"), + re.compile(r"^([A-Za-z0-9_$]+) \(whose .*?\)\.([A-Za-z0-9_$]+)\b"), +) + + +def bound_columns(report: Mapping[str, Any], kind: str) -> set[tuple[str, str]]: + """The columns a task's parameters are stored in, as the generator's + report places them (``table.column``, or a row beside the record's + ``table (whose ...).column``). A place in any other form is left out, so + fewer columns count as the task's: never more.""" + + out: set[tuple[str, str]] = set() + entry = (report.get("kinds") or {}).get(kind) or {} + for place in (entry.get("locations") or {}).values(): + for pat in _BOUND: + m = pat.match(str(place)) + if m: + out.add((m.group(1), m.group(2))) + break + return out + + +def faults( + schema: Schema, + sb: "Sandbox", + demo: Demo, + write: list[Op], + bound: Optional[set[tuple[str, str]]] = None, +) -> list[Case]: """Faults planted from one held-out write: its diff, the schema and its task's values. The contracts are never read here. @@ -872,11 +901,13 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: # is made again for a second submit: plant a fresh one, so # the copy is a duplicate the store accepts. for col in unique_columns(sb, op.table) - set(key): - if not _is_value(dup.get(col), values): + if (op.table, col) not in (bound or set()): v = fresh_value(sb, t, col, dup.get(col)) if v is not None: dup[col] = v - given = {c for c, v in op.row.items() if _is_value(v, values)} + # The task's columns: where the generator stores its + # parameters (bindings, never a value that happens to match). + given = {c for c in op.row if (op.table, c) in (bound or set())} out.append(Case(f"duplicate:{op.table}", "duplicate", write + [Op(op.table, key, dup)], copy_of=(op.table, dict(op.row), dup, given))) # fmt: skip @@ -1272,6 +1303,10 @@ def validate( if modes is None: modes = {} + try: + report = json.loads((Path(manifest).parent / "report.json").read_text()) + except (OSError, ValueError): + report = {} # no bindings: no column counts as the task's assert sb.schema is not None by_kind: dict[str, list[Outcome]] = {} @@ -1288,7 +1323,8 @@ def validate( saved = [(op, sb.get(op.table, op.key)) for op in undo] _put_all(sb, undo) try: - cases += faults(sb.schema, sb, demo, write) + cases += faults(sb.schema, sb, demo, write, + bound_columns(report, demo.kind)) # fmt: skip finally: try: sb.put_many([Op(op.table, op.key, row) for op, row in saved]) @@ -1661,8 +1697,11 @@ def trial(picks: Mapping[str, Mapping[str, int]], tag: str) -> dict[str, Any]: # can be trusted: no contract is installed, and every kind says why. import shutil - for d in (out / "contracts", out / ".contracts.staged"): - shutil.rmtree(d, ignore_errors=True) + shutil.rmtree(out / ".contracts.staged", ignore_errors=True) + if (out / "contracts").exists(): + # Kept aside for a look, never served: choose reads contracts/. + aside = out / f"contracts.stopped-{time.strftime('%Y%m%dT%H%M%S')}" + (out / "contracts").rename(aside) why = f"setup stopped: {broken}" summary = {k: {"admitted": False, "why": why, "correct_writes": 0, "false_stops": [], "faults": 0, "faults_stopped": 0, @@ -1770,7 +1809,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: } ) ) - return 0 + return 2 if result.get("stopped") else 0 if __name__ == "__main__": diff --git a/packages/flow/tests/test_zero_touch.py b/packages/flow/tests/test_zero_touch.py index 61c7c72336..1de4dc5ce3 100644 --- a/packages/flow/tests/test_zero_touch.py +++ b/packages/flow/tests/test_zero_touch.py @@ -721,7 +721,9 @@ def test_r3b1_s12_on_postgresql(pg: Any) -> None: # noqa: F811 write = [zt.Op("res", {"id": "2"}, new)] def dup_of(values: dict[str, Any]) -> Any: - cases = zt.faults(sb.schema, sb, type("D", (), {"values": values})(), write) + bound = {("res", k) for k in values} # the generator's bindings + cases = zt.faults(sb.schema, sb, type("D", (), {"values": values})(), + write, bound) # fmt: skip return next(c for c in cases if c.cls == "duplicate") # The task gives the name; the app makes the code: the copy gets a @@ -777,4 +779,70 @@ def broken(*a: Any, **k: Any) -> Any: log=lambda s: None) # fmt: skip assert result["admitted"] == [] and "injected" in result["stopped"] assert not (out / "contracts").exists() + assert list(out.glob("contracts.stopped-*")) # kept aside, never served assert "setup stopped" in (out / "EXCEPTIONS.md").read_text() + + +# -- review round 4: R4-B1 ------------------------------------------------------ + + +def test_r4b1_an_app_value_equal_to_a_task_value_is_still_made_fresh(pg: Any) -> None: # noqa: F811 + """The app's number 7 equals the task's quantity 7, and the app's + reference starts with the task's due date: neither is the task's, by the + generator's bindings, so both are made fresh and the store takes the + duplicate (it is never waived as impossible).""" + + admin = psycopg2.connect( + host="127.0.0.1", port=pg["port"], user="admin", dbname="postgres" + ) + admin.autocommit = True + admin.cursor().execute("CREATE DATABASE uq_r4") + admin.close() + c = psycopg2.connect( + host="127.0.0.1", port=pg["port"], user="admin", dbname="uq_r4" + ) + c.autocommit = True + c.cursor().execute( + "CREATE TABLE r4 (id integer PRIMARY KEY, num integer NOT NULL, qty integer," + " ref text NOT NULL, due date);" + "CREATE UNIQUE INDEX uk_num ON r4 (num); CREATE UNIQUE INDEX uk_ref ON r4 (ref);" + "INSERT INTO r4 VALUES (1, 5, 3, 'X-1', '2026-01-01');" + ) + c.close() + read = {"sql_driver": "psycopg2", "sql_connect_args": {"host": "127.0.0.1", + "port": pg["port"], "user": "admin", "dbname": "uq_r4"}} # fmt: skip + sb = zt.Sandbox(read, read) + sb.create(zt.store.read_schema(zt.store.connector(read), "postgresql"), "admin") + try: + new = { + "id": "2", + "num": "7", + "qty": "7", + "ref": "2026-12-01-0001", + "due": "2026-12-01", + } + demo = type("D", (), {"values": {"qty": 7, "due": "2026-12-01"}})() + bound = {("r4", "qty"), ("r4", "due")} + cases = zt.faults(sb.schema, sb, demo, [zt.Op("r4", {"id": "2"}, new)], bound) + dup = next(c for c in cases if c.cls == "duplicate") + copy = dup.write[-1].row + assert copy["num"] != "7" and copy["ref"] != "2026-12-01-0001" + assert copy["qty"] == "7" and copy["due"] == "2026-12-01" + sb.put_many(dup.write) # accepted: a duplicate the contract must stop + finally: + sb.drop() + + +def test_r4b1_bindings_come_from_the_generators_report() -> None: + report = {"kinds": {"k": {"locations": { + "qty": "r4.qty (number)", + "project": "issues.project_id -> projects.name (exact)", + "ref": "custom_values (whose customized_id is the issues key).value (exact)", + "odd": "something else", + }}}} # fmt: skip + assert zt.bound_columns(report, "k") == { + ("r4", "qty"), + ("issues", "project_id"), + ("custom_values", "value"), + } + assert zt.bound_columns({}, "k") == set() From b1950b95c01e3ad6fa24d047c1a26170d1038714 Mon Sep 17 00:00:00 2001 From: abrichr Date: Wed, 4 Nov 2026 00:23:21 -0500 Subject: [PATCH 12/12] Zero-touch: typing-only fixes for mypy (no Optional reassignment, one type per variable) --- .../generate/verify_contract.py | 20 ++++---- .../openadapt_flow/generate/zero_touch.py | 50 ++++++++++--------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/packages/flow/openadapt_flow/generate/verify_contract.py b/packages/flow/openadapt_flow/generate/verify_contract.py index 29de6a9a5b..e1c547639f 100644 --- a/packages/flow/openadapt_flow/generate/verify_contract.py +++ b/packages/flow/openadapt_flow/generate/verify_contract.py @@ -2025,11 +2025,10 @@ def build_contract( _anchor_roles(schema, plan, variant, settings.actor_params), settings.strict_pins, ) - for col, role in sorted(roles.items()): - if flavor == "found": - role = _found_role(role) - if role is None: - continue + for col, full in sorted(roles.items()): + role = _found_role(full) if flavor == "found" else full + if role is None: + continue _pin( b, main, @@ -2158,13 +2157,12 @@ def build_contract( m[key] = {"literal": "1"} else: m[key] = _value_expr(pname, loc.rule) - for col, role in sorted(_strict(g.roles, settings.strict_pins).items()): - if role.role == "link": + for col, full in sorted(_strict(g.roles, settings.strict_pins).items()): + if full.role == "link": + continue + role = _found_role(full) if found else full + if role is None: continue - if found: - role = _found_role(role) - if role is None: - continue _pin( b, surf, diff --git a/packages/flow/openadapt_flow/generate/zero_touch.py b/packages/flow/openadapt_flow/generate/zero_touch.py index 1df82473b0..8cc28c0711 100644 --- a/packages/flow/openadapt_flow/generate/zero_touch.py +++ b/packages/flow/openadapt_flow/generate/zero_touch.py @@ -792,8 +792,10 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: new = plant_value(sb, t, col, op.row.get(col)) name = f"{cls}:{op.table}.{col}" if new is None: + c = t.column(col) + kind = c.data_type if c is not None else "?" out.append(Case(name, cls, [], unplantable=f"no value of type " - f"{t.column(col).data_type if t.column(col) else '?'} can be planted")) # fmt: skip + f"{kind} can be planted")) # fmt: skip return row = dict(op.row) row[col] = new @@ -823,8 +825,8 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: # A new row's own key, when the store doesn't fill it (a code, # a user name): a wrong one is planted like any other value. for col in t.primary_key: - c = t.column(col) - if c is None or c.auto: + pk = t.column(col) + if pk is None or pk.auto: continue new = plant_value(sb, t, col, op.row.get(col)) cls = ( @@ -864,23 +866,23 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: other = next((r for r in sb.rows(op.table, 20) if store.row_key(t, r) != store.row_key(t, op.key)), None) # fmt: skip if other is None: - key = _new_key(sb, t, before) - other = _copy_row(t, before, key) if key is not None else None + copy_key = _new_key(sb, t, before) + other = _copy_row(t, before, copy_key) if copy_key is not None else None if other is not None: moved = dict(other) - for c in changed: - moved[c] = op.row.get(c) + for col in changed: + moved[col] = op.row.get(col) out.append(Case(f"wrong_record:{op.table}", "wrong_record", write[:i] + write[i + 1 :] + [Op(op.table, other, moved)])) # fmt: skip # The same change on another row: an excuse must hold only # on the rows the write declares. - for c in changed: - v = edit_value(sb, t, c, other.get(c), op.row.get(c)) - if v is None or other.get(c) == v: + for col in changed: + v = edit_value(sb, t, col, other.get(col), op.row.get(col)) + if v is None or other.get(col) == v: continue edit = dict(other) - edit[c] = v - out.append(Case(f"unrelated_edit:{op.table}.{c}", "unrelated_edit", + edit[col] = v + out.append(Case(f"unrelated_edit:{op.table}.{col}", "unrelated_edit", write + [Op(op.table, other, edit)])) # fmt: skip # A late change to the target itself: its first changed column. if changed: @@ -891,16 +893,16 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: out.append(Case(f"delayed:{op.table}.{changed[0]}", "delayed", list(write), late=[Op(op.table, op.key, late)])) # fmt: skip if is_new: - key = _new_key(sb, t, op.row) - if key is None: + dup_key = _new_key(sb, t, op.row) + if dup_key is None: out.append(Case(f"duplicate:{op.table}", "duplicate", [], unplantable="no new key can be made for a copy")) # fmt: skip else: - dup = _copy_row(t, op.row, key) + dup = _copy_row(t, op.row, dup_key) # A unique value the app makes (an invoice or ticket number) # is made again for a second submit: plant a fresh one, so # the copy is a duplicate the store accepts. - for col in unique_columns(sb, op.table) - set(key): + for col in unique_columns(sb, op.table) - set(dup_key): if (op.table, col) not in (bound or set()): v = fresh_value(sb, t, col, dup.get(col)) if v is not None: @@ -909,13 +911,13 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: # parameters (bindings, never a value that happens to match). given = {c for c in op.row if (op.table, c) in (bound or set())} out.append(Case(f"duplicate:{op.table}", "duplicate", - write + [Op(op.table, key, dup)], + write + [Op(op.table, dup_key, dup)], copy_of=(op.table, dict(op.row), dup, given))) # fmt: skip # The values land on an existing look-alike instead. for other in sb.rows(op.table, 20): look = dict(op.row) - for c in t.primary_key: - look[c] = other.get(c) + for col in t.primary_key: + look[col] = other.get(col) out.append(Case(f"look_alike:{op.table}", "look_alike", write[:i] + write[i + 1 :] + [Op(op.table, other, look)])) # fmt: skip break @@ -945,16 +947,16 @@ def column_fault(cls: str, i: int, op: Op, col: str) -> None: for name in sorted(set(schema.tables) - touched): t = schema.tables[name] sample = sb.rows(name, 1) - key = _new_key(sb, t, sample[0]) if sample else None - if key is None: + new_key = _new_key(sb, t, sample[0]) if sample else None + if new_key is None: continue - row = _copy_row(t, sample[0], key) - for col in unique_columns(sb, name) - set(key): + row = _copy_row(t, sample[0], new_key) + for col in unique_columns(sb, name) - set(new_key): v = fresh_value(sb, t, col, row.get(col)) if v is not None: row[col] = v # a copied unique code would be refused out.append(Case(f"other_table:{name}", "other_table", - write + [Op(name, key, row)])) # fmt: skip + write + [Op(name, new_key, row)])) # fmt: skip planted += 1 if planted >= 2: break