diff --git a/.gitleaks.toml b/.gitleaks.toml index 4feb898a6e..fd25e61328 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -23,3 +23,14 @@ regexTarget = "secret" regexes = [ '''^397b3becedcd5a98769967ff1ff8501ddc89f8368b8f766e4701377d7dbaabe5$''', ] + +# benchmark/rdp_avc_real records each trial's made-up note value (job id, +# trial number, condition) under a "token" key. These seven trip the generic +# entropy rule; they are lab labels typed into a test app, not credentials. +[[allowlists]] +description = "Made-up per-trial note values in benchmark/rdp_avc_real results" +regexTarget = "secret" +regexes = [ + '''^Oaun30(1[5-9])deco$''', + '''^Oaun30(21|24)sibl$''', +] diff --git a/CAPABILITIES.md b/CAPABILITIES.md index d1a437518f..3f93444acc 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -194,7 +194,7 @@ improvising or retrying when something is off. | Web (Playwright, Chromium) | Released | Real app, every CI build | [SURFACES.md](packages/flow/docs/SURFACES.md) | | Native Windows (UI Automation) | Released | Synthetic (WinForms, 3 of 3 trials) | [SURFACES.md](packages/flow/docs/SURFACES.md) | | Native macOS (Accessibility) | Released | Synthetic (TextEdit, 3 of 3 trials) | [SURFACES.md](packages/flow/docs/SURFACES.md) | -| RDP (pixels; Aardwolf or headless FreeRDP with a session pool) | Released | Real Windows 11 host, 3 of 3, plus a FreeRDP batch | [SURFACES.md](packages/flow/docs/SURFACES.md), `backends/xfreerdp.py` | +| RDP (pixels; Aardwolf or headless FreeRDP with a session pool) | Released | Real Windows 11 host, 3 of 3, plus a FreeRDP batch; through the stock Remote Desktop client on a real H.264/AVC444 session, 105 runs on two apps: 55 saves confirmed in the database, 0 wrong-record saves, 40 false stops ranked by cause | [SURFACES.md](packages/flow/docs/SURFACES.md), `backends/xfreerdp.py`, [benchmark/rdp_avc_real](packages/flow/benchmark/rdp_avc_real/README.md) | | Citrix and other virtual desktops, with OpenAdapt installed inside the session (`in_session`): the same structure-first path as a local desktop | Released | Same evidence as the Windows path | [SURFACES.md](packages/flow/docs/SURFACES.md) | | Citrix from outside the session, pixels only (`remote_display`), the fallback when nothing can be installed inside | Exploratory | A Citrix stand-in with no DOM: 3 healthy runs, 3 stops on drift | [LIMITS.md](packages/flow/docs/LIMITS.md) | | Chain programs across surfaces, passing confirmed values | Released | Synthetic | `openadapt-flow compose`, `process`; [PROCESS_CONTRACT.md](packages/flow/docs/PROCESS_CONTRACT.md) | @@ -239,7 +239,7 @@ improvising or retrying when something is off. | A scroll that never finds the next click's target halts on that click, so self-repair can take a relabelled control behind a scroll; the refusal for unreviewed-risk steps names them | On main ([#1337](https://github.com/OpenAdaptAI/OpenAdapt/pull/1337)) | Synthetic; openIMIS drift row checked offline on its recorded frames | `runtime/replayer.py`, `qualify set-risk` | | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](packages/flow/benchmark/pixel_scale/README.md) | -| Find icons again on Citrix, RDP and VNC after a display-scale, zoom, font, theme or codec change, with a local model that counts look-alikes and stops on a tie (`--local-grounding`, off by default) | On main ([#1596](https://github.com/OpenAdaptAI/OpenAdapt/pull/1596)) | Synthetic (Chromium-drawn clinic screen, pixels only: icons found under 21 of 26 conditions, template matching 8; 0 wrong targets in 65; whole runs unchanged, 4 of 39 in both arms, because the layout, identity and census checks stop first) | [benchmark/pixel_grounding](packages/flow/benchmark/pixel_grounding/README.md), `runtime/local_grounder.py` | +| Find icons again on Citrix, RDP and VNC after a display-scale, zoom, font, theme or codec change, with a local model that counts look-alikes and stops on a tie (`--local-grounding`, off by default) | On main ([#1596](https://github.com/OpenAdaptAI/OpenAdapt/pull/1596)) | Synthetic (Chromium-drawn clinic screen, pixels only: icons found under 21 of 26 conditions, template matching 8; 0 wrong targets in 65; whole runs unchanged, 4 of 39 in both arms, because the layout, identity and census checks stop first). Held out on real AVC444 RDP frames: 0 wrong picks, 6 right, 75 abstentions in 81 | [benchmark/pixel_grounding](packages/flow/benchmark/pixel_grounding/README.md), `runtime/local_grounder.py` | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | | Repair a pinned bundle: the fix runs the pinned bundle's own qualification campaign on a test copy, keeps every write, effect, identity rule and record check, and is re-admitted with signed lineage (at most three repairs, no revoked key) that the pin verifies | On main | Synthetic | [SELF_REPAIR.md](packages/flow/docs/SELF_REPAIR.md#a-pinned-bundle-re-qualify-and-re-admit) | | Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | On main ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair and stopped with no wrong save; silent-saves 1.1.0 UI-change blocks: 18 of 28 items that always stopped now correct, 0 wrong | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](packages/flow/benchmark/auto_repair/README.md) | diff --git a/llms-full.txt b/llms-full.txt index 07aba0ea88..33f547d4f3 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -591,7 +591,7 @@ improvising or retrying when something is off. | Web (Playwright, Chromium) | Released | Real app, every CI build | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | | Native Windows (UI Automation) | Released | Synthetic (WinForms, 3 of 3 trials) | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | | Native macOS (Accessibility) | Released | Synthetic (TextEdit, 3 of 3 trials) | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | -| RDP (pixels; Aardwolf or headless FreeRDP with a session pool) | Released | Real Windows 11 host, 3 of 3, plus a FreeRDP batch | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md), `backends/xfreerdp.py` | +| RDP (pixels; Aardwolf or headless FreeRDP with a session pool) | Released | Real Windows 11 host, 3 of 3, plus a FreeRDP batch; through the stock Remote Desktop client on a real H.264/AVC444 session, 105 runs on two apps: 55 saves confirmed in the database, 0 wrong-record saves, 40 false stops ranked by cause | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md), `backends/xfreerdp.py`, [benchmark/rdp_avc_real](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/rdp_avc_real/README.md) | | Citrix and other virtual desktops, with OpenAdapt installed inside the session (`in_session`): the same structure-first path as a local desktop | Released | Same evidence as the Windows path | [SURFACES.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SURFACES.md) | | Citrix from outside the session, pixels only (`remote_display`), the fallback when nothing can be installed inside | Exploratory | A Citrix stand-in with no DOM: 3 healthy runs, 3 stops on drift | [LIMITS.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/LIMITS.md) | | Chain programs across surfaces, passing confirmed values | Released | Synthetic | `openadapt-flow compose`, `process`; [PROCESS_CONTRACT.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/PROCESS_CONTRACT.md) | @@ -636,7 +636,7 @@ improvising or retrying when something is off. | A scroll that never finds the next click's target halts on that click, so self-repair can take a relabelled control behind a scroll; the refusal for unreviewed-risk steps names them | On main ([#1337](https://github.com/OpenAdaptAI/OpenAdapt/pull/1337)) | Synthetic; openIMIS drift row checked offline on its recorded frames | `runtime/replayer.py`, `qualify set-risk` | | Self-repair across several new steps: a bridge of up to three actions for a new required dialog or a control moved into a menu, each new step armed with a record check and probed up to the save, one person approving | On main ([#1325](https://github.com/OpenAdaptAI/OpenAdapt/pull/1325)) | Synthetic (MockMed menu and provider drifts, scripted agent; [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md)) | `run --self-repair`, [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md) | | Keep pixel-path runs working across display scaling and font smoothing changes | On main ([#1286](https://github.com/OpenAdaptAI/OpenAdapt/pull/1286)) | Synthetic (in-process RDP and canvas renderings: 17 of 24 healthy runs across 100–175%, up from 3; 0 wrong saves) | [benchmark/pixel_scale](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_scale/README.md) | -| Find icons again on Citrix, RDP and VNC after a display-scale, zoom, font, theme or codec change, with a local model that counts look-alikes and stops on a tie (`--local-grounding`, off by default) | On main ([#1596](https://github.com/OpenAdaptAI/OpenAdapt/pull/1596)) | Synthetic (Chromium-drawn clinic screen, pixels only: icons found under 21 of 26 conditions, template matching 8; 0 wrong targets in 65; whole runs unchanged, 4 of 39 in both arms, because the layout, identity and census checks stop first) | [benchmark/pixel_grounding](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_grounding/README.md), `runtime/local_grounder.py` | +| Find icons again on Citrix, RDP and VNC after a display-scale, zoom, font, theme or codec change, with a local model that counts look-alikes and stops on a tie (`--local-grounding`, off by default) | On main ([#1596](https://github.com/OpenAdaptAI/OpenAdapt/pull/1596)) | Synthetic (Chromium-drawn clinic screen, pixels only: icons found under 21 of 26 conditions, template matching 8; 0 wrong targets in 65; whole runs unchanged, 4 of 39 in both arms, because the layout, identity and census checks stop first). Held out on real AVC444 RDP frames: 0 wrong picks, 6 right, 75 abstentions in 81 | [benchmark/pixel_grounding](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/pixel_grounding/README.md), `runtime/local_grounder.py` | | Pre-approved repairs: a signed policy lists low-risk changes that apply without a person, a test-record run before each queue, and one approval for every item halted the same way | On main ([#1294](https://github.com/OpenAdaptAI/OpenAdapt/pull/1294)) | Synthetic | standard changes | | Repair a pinned bundle: the fix runs the pinned bundle's own qualification campaign on a test copy, keeps every write, effect, identity rule and record check, and is re-admitted with signed lineage (at most three repairs, no revoked key) that the pin verifies | On main | Synthetic | [SELF_REPAIR.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/SELF_REPAIR.md#a-pinned-bundle-re-qualify-and-re-admit) | | Repair by default: once a repair agent is configured, every stop that wrote nothing goes to it without anyone starting it; a stop after a possible write goes to reconciliation; every stop is a row in a ledger with the funnel from stop to resumed item | On main ([#1360](https://github.com/OpenAdaptAI/OpenAdapt/pull/1360)) | Synthetic (MockMed, 22 drift modes, real Claude agent: 6 of 21 repairable stops resumed and verified, 0 wrong saves in 66 runs); routing over recorded OpenEMR stops; live openIMIS relabel: 3 of 3 reached repair and stopped with no wrong save; silent-saves 1.1.0 UI-change blocks: 18 of 28 items that always stopped now correct, 0 wrong | `run` (on by default, `--no-auto-repair`), `repair:` in `deploy.yaml`, [benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/benchmark/auto_repair/README.md) | @@ -1184,6 +1184,7 @@ Every published result used synthetic patients, claims and records. | OpenEMR 8.0.0.3, local test copies (open-source EHR; its 7.0 release is ONC-certified) | Referral entry by agent, screen program and API step; injected database faults; repair of a relabeled save button; duplicate check | | openIMIS, local test copy (open-source insurance and claims system) | Claim entry over a worklist; a relabeled Save sent to repair with no wrong save | | Windows 11 over RDP, real network | 3 of 3 effect-confirmed trials, plus a FreeRDP batch | +| Windows Server 2022 over RDP with H.264/AVC444, stock Remote Desktop client | 105 runs on two apps: 55 database-confirmed saves, 0 wrong-record saves, 40 false stops ([benchmark/rdp_avc_real](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/../benchmark/rdp_avc_real/README.md)) | | Citrix stand-in, no DOM | 3 healthy runs and 3 drift stops through the Citrix backend | | WinForms, TextEdit, a GTK3 app | 3 of 3 effects on each native backend | | MockMed, MockLoan and fixtures (OpenAdapt's own test apps) | Fault suites, approvals, repair and the agent comparisons | @@ -1204,7 +1205,7 @@ signed; the Python packages are published from the reviewed release workflow. | Windows UI Automation | **Counted task acceptance** | The native UIA backend completed 3/3 exact WinForms effects and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. | | Native macOS automation | **Counted task acceptance** | The AX/AppKit backend completed 3/3 exact TextEdit file effects and refused an ambiguous two-window target; structural record/locate/refuse behavior is covered separately. Known AX-only ` - Google Chrome` and ` - Google Chrome for Testing` window-title suffixes are accepted only for the corresponding exact CoreGraphics application owner; PID, unique-candidate, focused/main, and topmost-window checks remain mandatory. | Acceptance is bound to the tested host and application. Accessibility exposure and supported native actions vary by application. | | Native Linux automation | **Counted task acceptance plus required CI** | Required current-main CI exercises AT-SPI against GTK3 with 3 exact effects, 3 ambiguity refusals, and 3 stale-target refusals. | The built-in path is X11. Wayland needs an operator-approved portal session, and each application retains its own qualification. | -| RDP automation | **Counted task acceptance plus required CI** | A real-network Aardwolf/Windows batch passed 3/3 with an independent guest-tools oracle; a separate real-FreeRDP batch covers record, compile, governed replay, and drift refusal. | The two records do not by themselves cover arbitrary applications, WAN conditions, display policy, or Citrix ICA/HDX. | +| RDP automation | **Counted task acceptance plus required CI** | A real-network Aardwolf/Windows batch passed 3/3 with an independent guest-tools oracle; a separate real-FreeRDP batch covers record, compile, governed replay, and drift refusal. | The records do not cover arbitrary applications, WAN conditions, or Citrix ICA/HDX. A real AVC444 session (same virtual network, software encoding) stops on 12 of 20 healthy OpenEMR runs; the causes are ranked in benchmark/rdp_avc_real. | | Citrix / VDI pixel-window automation | **Required CI plus counted no-DOM stand-in** | The dedicated exact-Workspace-window backend, readiness gate, governed run, and durable resume passed 3 healthy effect-confirmed no-DOM trials plus 3 drift safe-halts. | The retained record explicitly has `ica_hdx_accepted=false`; live ICA/HDX acceptance stays bound to the exact deployment instead of being inferred from a stand-in or RDP. | | Managed browser execution | **Separate product target and workflow admissions** | The hosted lane admits attested browser bundles; production mode requires a configured real runner and refuses silent mock fallback. | It does not extend the supported claim to Windows, RDP, Citrix, PHI-bearing shared-cloud execution, an SLA, or a regulated certification. | | On-premises / customer-managed deployment | **Release and clean-machine evidence; site acceptance required** | Deployment configuration, local run gates, egress checks, and audit primitives are supplied. | OpenAdapt does not configure the customer's firewall, KMS, storage, identity provider, backups, retention, incident response, or legal compliance program. | diff --git a/packages/flow/benchmark/rdp_avc_real/README.md b/packages/flow/benchmark/rdp_avc_real/README.md new file mode 100644 index 0000000000..1e2e98d369 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/README.md @@ -0,0 +1,224 @@ +# Screen-only runs over a real H.264/AVC444 remote desktop session + +OpenAdapt's pixel path (`RemoteDisplayBackend` with the Windows window +client) drove two apps through the stock Windows Remote Desktop Connection +client (`mstsc`), connected to a Windows Server 2022 host whose sessions are +encoded with H.264/AVC444. Every save was judged from the app's own database, +outside the session. + +It replaces a stand-in: the earlier Citrix and RDP evidence used a synthetic +window or an RDP session without the AVC444 codec. It is not Citrix ICA/HDX +and not Azure Virtual Desktop (see [What this does not prove](#what-this-does-not-prove)). + +## Results + +105 counted runs, from October 28 to 29, 2026. Every run made zero model calls. + +| App and condition | Runs | Saves confirmed in the database | Wrong record | Saved but not reported, or reported but not saved | False stops | Stops on a look-alike | +|---|---:|---:|---:|---:|---:|---:| +| Patient Notes, healthy | 20 | 20 | 0 | 0 | 0 | 0 | +| Patient Notes, rows reordered | 10 | 10 | 0 | 0 | 0 | 0 | +| Patient Notes, 16-bit colour | 10 | 10 | 0 | 0 | 0 | 0 | +| Patient Notes, decoy patient above the target | 10 | 0 | 0 | 0 | 10 | 0 | +| Patient Notes, near-identical sibling patient | 10 | 0 | 0 | 0 | 0 | 10 | +| Patient Notes, 1440×900 session | 10 | 0 | 0 | 0 | 10 | 0 | +| OpenEMR new patient, healthy | 20 | 8 | 0 | 0 | 12 | 0 | +| OpenEMR new patient, 16-bit colour | 10 | 7 | 0 | 0 | 3 | 0 | +| OpenEMR new patient, 1440×900 session | 5 | 0 | 0 | 0 | 5 | 0 | +| **All** | **105** | **55** | **0** | **0** | **40** | **10** | + +Zero wrong-record saves in 105 runs bounds the rate at 3.5 per 100 (95% +Wilson upper bound). Per-condition rates with 95% intervals are in +[`results/results.json`](results/results.json). + +A healthy Patient Notes run took a median of 18 s; an OpenEMR run, which +signs in and fills a 12-field form, took a median of 82 s (90 s at 16-bit). + +The stops are the finding. The product stopped instead of guessing every +time the screen did not match the demonstration. On the plain recordings +that is still too often: 12 of 20 healthy OpenEMR runs stopped. + +### What stops runs on real AVC444 frames, ranked + +Each stop has one cause, taken from the failing step's own message +([`analyze.py`](analyze.py)). One frame per cause is in +[`results/frames/`](results/frames). + +| Rank | Cause | False stops | Where | What it needs | +|---:|---|---:|---|---| +| 1 | Screen still changing where the identity check looked | 13 | OpenEMR, Sex list (8) and First Name (5) | A bounded wait for the region to settle before failing. The frame shows OpenEMR's date picker still open after Escape. A blinking caret or the codec refining a region can do the same. | +| 2 | A screen check compiled from an auto-selected row | 10 | Patient Notes, decoy, after Search | Search selects the first row, so the recording expects "Patient: Neil Sorenson" in the header. With a decoy above him, the header names the decoy. The patient is picked by the next click, so this text should not be a check on the Search step. | +| 3 | Re-resolve after the pointer moves, exact-pixel guard | 10 | Patient Notes, 1440×900, Save Note | The target resolved at (561, 490) after the hover and (560, 489) before. The guard requires equal points. Accept a point inside the target's own box, or within the codec's measured 1–2 px jitter. | +| 4 | Layout reflow, absolute postcondition region | 5 | OpenEMR, 1440×900, first click | The click found the Username box (the geometry rung was within about 12 px). The postcondition then checked the recorded frame's absolute region, which on the wider page covers the focused box. Bind postcondition regions to the resolved target, or stop at once with a "layout changed" reason. | +| 5 | Identity check before a click | 2 | OpenEMR DOB | The template missed DOB, and the geometry rung proposed a point whose surrounding text read "Preferred Name". The identity check refused the click, so a wrong target was caught before the click. What it needs: find out why the template missed (the field looks different while the date picker or the duplicate highlight is up). (On the older build the sibling look-alike also stopped here; those 5 runs count as look-alike stops.) | + +Not counted above: on the merged build, the sibling look-alike is refused one +step earlier ("OCR landmark evidence did not uniquely establish a target"). +That is the intended stop and needs no fix. + +Three more causes appeared while building the OpenEMR workflow, before the +counted runs. Each is fixed in the lab setup or the recording, and each would +recur at a site: + +- **A product-registration dialog after sign-in.** It covered the menu the + recording clicks. The lab records an opt-out in OpenEMR's + `product_registration` table, with the version string OpenEMR compares + (`7.0.3 (4)`); nothing is sent. +- **Edge autofill.** Typing into First Name opened a "Manage…" list over the + DOB field, so the recording captured the list as DOB's template. Turned off + by Edge policy on the host. +- **A run parameter inside an identity band.** Recording First Name before + Last Name put the first name's value into Last Name's identity band. Each + run types a new value, so 2 of 2 pilot runs stopped. The counted recording + types Last Name first. + +### The record check (verify-run, merged in #1570 and #1582) + +For the last Patient Notes campaign (35 runs) and the last OpenEMR campaign +(15 runs), each run was also judged by `openadapt_flow.verify_run`: + +1. a baseline read of the whole table before the replay; +2. the effect contract (the note on Neil Sorenson's row; the new patient's + last name, DOB and sex); +3. a frame check that nothing else in the table changed. + +It read the database through a read-only service beside each app +([`oracle_service.py`](oracle_service.py)), never through the screen. + +| Campaign | Runs | CONFIRMED and saved | REFUTED and not saved | Disagreed with the database | +|---|---:|---:|---:|---:| +| Patient Notes | 35 | 20 | 15 | 0 | +| OpenEMR | 15 | 10 | 5 | 0 | + +### Local grounding rung (#1596), held out + +The rung's thresholds were set on a synthetic Chromium fixture. Here it was +applied unchanged to the replays' real frames: 86 target-frames, 81 with a +known true place ([`grounding_heldout.py`](grounding_heldout.py), +[`results/results_grounding_heldout.json`](results/results_grounding_heldout.json)). + +| Outcome | Count | +|---|---:| +| Picked the true target | 6 (all the OpenEMR password field, whose crop holds the eye icon) | +| Picked a wrong target | 0 | +| Abstained: text label, left to OCR by design | 56 | +| Abstained: look-alikes overlap or several score alike | 19 | +| No truth (1440×900 Patient Notes frames) | 5, not scored | + +On these screens the true place scored 0.81 to 0.97 and the best other place +0.78 to 0.95. The gap is usually smaller than `LOOKALIKE_MARGIN` (0.15), so +the rung counts several look-alikes and abstains. It never picked a wrong +target, but on text-heavy forms over AVC444 it rarely helps. Queries took a +median of 119 ms (159 ms at most) on the operator's laptop CPU, not on the +client VM. + +The template matcher found the true place on 74 of 81. Its 3 "wrong" rows +are the Patient Notes note box, a 590×120 field: the point lies outside this +script's ±24×12 px tolerance but inside the box, and all three runs saved +correctly. + +### Separate-session read-back (`desktop_readback`) + +The read-back verifier looked Neil Sorenson up in its own RDP session, under +a separate `pnreader` logon that may read the record but not write it. Each +case first wrote a known note straight into the database. Results are in +[`results/campaigns/`](results/campaigns). + +- **First run:** 1 correct verdict and 9 INDETERMINATE, with no wrong + verdicts. The verifier refused because the record view showed the same + patient before and after its search. The lab's driver had left Neil + Sorenson open between look-ups, so nothing on screen could prove a + refresh. That is a lab fault, not a product fault. +- **Second run:** restarts the reader's app before each look-up. It + confirmed all 5 cases whose stored note matched. All 5 mismatches came + back INDETERMINATE and halted the run; none was falsely confirmed. The + verifier never reports an OCR read that disagrees as REFUTED ("OCR can + misread, so this is not proof the save is wrong"). So on the pixel path, + read-back can confirm a save or stop a run, but it cannot refute one. A + look-up took a median of 45 s. The tier is persisted-state reacquisition + (tier 3): a separate logon, without the application's own sign-in. + +## Licence and publishing + +- **Microsoft.** The Product Terms ("Universal License Terms for Online + Services", "Competitive Benchmarking") do not bar publishing results. They + ask a customer who offers a competing product to share the method on + request; OpenAdapt does not offer one. Windows Server and Remote Desktop + Connection are used as shipped. +- **Citrix.** This lab was meant to be Citrix. Two things stopped it: + - Citrix no longer offers self-serve accounts, so the CVAD ISO could not be + downloaded. The provisioning scripts stay in + [`scripts/citrix_lab/`](../../scripts/citrix_lab) for a design partner's + environment. + - The Cloud Software Group End User Agreement (v2.5) makes "performance + test results derived by Customer" Citrix Confidential Information + (section 1.8). It bars competitive benchmarking (2.7(x)) and limits + trials to internal evaluation (14.1). +- **Other VDI.** The Parallels RAS terms forbid performing or disclosing + benchmark tests (Product Licensing Terms 3.3). Omnissa Horizon has no + self-serve trial and needs Omnissa's approval to publish (General Terms + 1.4). + +## What this proves + +- The shipped pixel path records, compiles and replays through a real + AVC444-encoded RDP session in the stock Windows client. Every run made zero + model calls. +- In 105 runs on two apps it never saved to the wrong record. It never + reported a save the database did not hold, and never saved without + reporting it. +- The merged record check agreed with the database on all 50 runs it judged. + +## What this does not prove + +- **Not Citrix ICA/HDX, and not Azure Virtual Desktop.** It is the same + Windows RDP graphics stack AVD uses, but no AVD broker, gateway or client + was involved: the founder's account could not create the Entra user AVD + needs. +- **Not a WAN.** Client and host share an Azure virtual network. Bandwidth, + latency and loss were not shaped. +- **Not GPU encoding.** The host has no GPU, so H.264 was encoded in + software. +- **Not a real clinic.** Both apps are on made-up data. Patient Notes is a + benchmark app; OpenEMR is a stock 7.0.3 container. +- **Not reliable enough to run unattended.** Healthy OpenEMR runs stopped 12 + times in 20. The ranked causes above are the work list. +- **Not a model-based agent.** Small samples: 10 to 20 runs per condition. + +## Reproduce + +Lab scripts are in [`scripts/rdp_avc_lab/`](../../scripts/rdp_avc_lab). One +Azure resource group (`openadapt-avd-lab`) holds the VMs, a Key Vault for +every password, and a private blob container for results. RDP is open only +to the operator's IP. + +```bash +bash scripts/rdp_avc_lab/provision.sh +SECRETS="EmrPassword=rdlab-emruser PnPassword=rdlab-pnuser" \ + bash scripts/rdp_avc_lab/run_ps.sh vm-rdhost host_setup.ps1 PnDbB64=... PnAppB64=... +SECRETS="HarnessPassword=rdlab-harness EmrPassword=rdlab-emruser PnPassword=rdlab-pnuser" \ + bash scripts/rdp_avc_lab/run_ps.sh vm-client client_setup.ps1 Ref= +bash scripts/rdp_avc_lab/submit_job.sh job.json out/ # {"kind": "pn_campaign" | "emr_campaign" | "pn_readback", ...} +RG=openadapt-avd-lab bash scripts/citrix_lab/lifecycle.sh stop # deallocate when idle +python benchmark/rdp_avc_real/analyze.py out/pn-run-3 out/emr-run-3 ... --out analysis/ +python benchmark/rdp_avc_real/grounding_heldout.py out/pn-run-3 out/emr-run-3 --output ... +``` + +Code versions: + +- The first Patient Notes campaign and the first two OpenEMR campaigns ran + `4376a5d` with the UTF-8 bundle fix applied. +- The record-check campaigns ran `d62ea71` (main with #1570, #1582 and #1596 + merged). + +Raw outputs (run reports, frames, bundles; 97 MB) are listed by SHA-256 in +[`results/raw_files.sha256`](results/raw_files.sha256). They are kept off the +repository. + +## Product fix found here + +On Windows, `Workflow.save` wrote `workflow.json` in the platform codec +(cp1252) with CRLF newlines. The compile crashed as soon as OCR recorded a +character outside cp1252 (an "ε" read off the OpenEMR screen). It now writes +and reads UTF-8 bytes +([`tests/test_bundle_encoding.py`](../../tests/test_bundle_encoding.py)). diff --git a/packages/flow/benchmark/rdp_avc_real/analyze.py b/packages/flow/benchmark/rdp_avc_real/analyze.py new file mode 100644 index 0000000000..c19a071047 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/analyze.py @@ -0,0 +1,227 @@ +"""Summarise the AVC444 campaigns and rank what stops runs, by cause. + +Reads each campaign's ``trials.json`` (written by harness.py on vm-client and +fetched by submit_job.sh), and writes ``results.json``: per app and +condition, trials, verified saves, wrong targets, silent wrong saves, false +stops and stops on a look-alike, each with a rate per 100 and a 95% Wilson +interval; then every stop with its cause, and the causes ranked by how many +trials they stopped. + +Causes (a stop gets exactly one, from the failing step's own message): + +- ``harness``: the lab, not the product (a crash in the harness, or state an + earlier trial left behind); +- ``locator``: no rung found the target, or a heal was refused; +- ``hover_reresolve``: the target re-resolved a pixel or two away after the + pointer moved onto it, and the exact-match guard refused the click; +- ``ambiguity_refusal``: OCR landmarks matched more than one target, so no + click was admitted (the intended stop on a look-alike); +- ``identity_precheck``: the identity check before a click disagreed; +- ``state_check``: a postcondition's recorded screen text did not appear; +- ``layout_reflow``: the session's desktop size changed the page layout, and + a postcondition still checked the recorded frame's absolute region; +- ``timing_codec``: the screen was still changing (or not stable) where the + check looked: animation, a caret, or the codec refining a region. + +It also copies, for each cause, the frame of one stopped step out of the +kept run archives into ``frames/``. + + python analyze.py [...] --out +""" + +from __future__ import annotations + +import argparse +import collections +import json +import math +import re +import zipfile +from pathlib import Path + +CAUSES = [ + ( + "hover_reresolve", + re.compile(r"after remote pointer hover, not the prepared point"), + ), + ("timing_codec", re.compile(r"screen kept changing|region_stable")), + ("ambiguity_refusal", re.compile(r"did not uniquely establish a target")), + ("identity_precheck", re.compile(r"Identity check failed")), + ( + "locator", + re.compile(r"all resolution rungs failed|heal quarantined|Could not resolve"), + ), + ("state_check", re.compile(r"text_present|expected screen state not reached")), +] +FIXES = { + "timing_codec": "Wait for the region to settle (bounded) before the identity and " + "stability checks instead of failing on the first changing frame; " + "mask a blinking caret; record the codec's refinement interval.", + "hover_reresolve": "Accept a re-resolve inside the target's own box (or within the " + "codec's measured jitter, 1-2 px) instead of exact pixel equality.", + "identity_precheck": "Exclude run parameters and transient dialog content (duplicate " + "lists) from the recorded identity band.", + "state_check": "Do not compile an auto-selected row's header text into the next " + "step's postcondition when the row is chosen by a later click.", + "layout_reflow": "Bind postcondition regions to the resolved target (offsets from " + "it), not the recorded frame's absolute pixels; or halt at the " + "first step with an explicit 'layout changed' reason.", + "locator": "Record after the screen has painted; on a reflowed layout use OCR " + "landmarks rather than the recorded crop position.", + "ambiguity_refusal": "None: the intended stop on a look-alike row.", + "harness": "Lab fix, not product: reset every record a job created before each trial.", +} + + +def wilson(k: int, n: int) -> list[float]: + if n == 0: + return [0.0, 0.0] + z = 1.96 + p = k / n + centre = (p + z * z / (2 * n)) / (1 + z * z / n) + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n) + return [ + round(100 * max(0.0, centre - half), 1), + round(100 * min(1.0, centre + half), 1), + ] + + +def rate(k: int, n: int) -> dict: + return { + "count": k, + "per_100": round(100 * k / n, 1) if n else None, + "ci95": wilson(k, n), + } + + +def cause_of(trial: dict) -> tuple[str, dict]: + bad = next((s for s in trial["steps"] if not s["ok"]), None) + if trial.get("error") or bad is None: + return "harness", {"step_id": None, "detail": trial.get("error")} + detail = bad.get("detail") or "" + if trial["condition"].startswith("desktop") and "region_stable" in detail: + return "layout_reflow", {"step_id": bad["step_id"], "detail": detail} + for name, pattern in CAUSES: + if pattern.search(detail): + return name, {"step_id": bad["step_id"], "detail": detail} + return "other", {"step_id": bad["step_id"], "detail": detail} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("campaigns", nargs="+", type=Path) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--only", + nargs="*", + default=[], + help="campaign:condition pairs; a campaign named here contributes only these " + "conditions (the rest were superseded by a later campaign)", + ) + args = parser.parse_args() + frames_dir = args.out / "frames" + frames_dir.mkdir(parents=True, exist_ok=True) + + conditions: dict = {} + stops: list[dict] = [] + for campaign in args.campaigns: + trials = json.loads((campaign / "trials.json").read_text()) + app = campaign.name.split("-", 1)[0] # campaign ids start with the app + keep = { + pair.split(":", 1)[1] + for pair in args.only + if pair.split(":", 1)[0] == campaign.name + } + for t in trials: + if keep and t["condition"] not in keep: + continue + key = f"{app}/{t['condition']}" + conditions.setdefault(key, []).append(t) + if t["class"] in ("verified_save",): + continue + cause, where = cause_of(t) + stops.append( + { + "campaign": campaign.name, + "app": app, + "trial": t["trial"], + "condition": t["condition"], + "class": t["class"], + "cause": cause, + **where, + } + ) + + summary = {} + for key, ts in sorted(conditions.items()): + n = len(ts) + count = collections.Counter(t["class"] for t in ts) + summary[key] = { + "trials": n, + "verified_saves": rate(count["verified_save"], n), + "wrong_target": rate(count["silent_wrong"] + count["flagged_wrong"], n), + "silent_wrong_saves": rate( + count["silent_wrong"] + count["false_completion"], n + ), + "false_stops": rate(count["false_stop"], n), + "stops_on_lookalike": rate(count["stop_on_lookalike"], n), + "model_calls": sum(t["model_calls"] for t in ts), + "median_elapsed_s": sorted(t["elapsed_s"] for t in ts)[n // 2], + } + + by_cause = collections.Counter( + s["cause"] for s in stops if s["class"] == "false_stop" + ) + ranked = [ + { + "cause": c, + "false_stops": k, + "fix": FIXES.get(c, ""), + "where": sorted( + { + f"{s['app']}/{s['condition']}:{s['step_id']}" + for s in stops + if s["cause"] == c + } + ), + } + for c, k in by_cause.most_common() + ] + + # One frame per cause: the failing step's 'before' frame from a kept archive. + kept: dict[str, str] = {} + for s in stops: + if s["cause"] in kept or not s["step_id"]: + continue + campaign = next(c for c in args.campaigns if c.name == s["campaign"]) + for archive in sorted(campaign.glob(f"{s['app']}_run_{s['condition']}*.zip")): + with zipfile.ZipFile(archive) as zf: + name = f"steps/{s['step_id']}_before.png" + if name in zf.namelist(): + dest = ( + frames_dir + / f"{s['cause']}_{s['app']}_{s['condition']}_{s['step_id']}.png" + ) + dest.write_bytes(zf.read(name)) + kept[s["cause"]] = dest.name + break + + result = { + "schema": "openadapt.rdp-avc444-campaigns.v1", + "conditions": summary, + "stops": stops, + "ranked_causes": ranked, + "cause_frames": kept, + } + (args.out / "results.json").write_text(json.dumps(result, indent=2)) + print( + json.dumps( + {"conditions": summary, "ranked_causes": ranked, "cause_frames": kept}, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/rdp_avc_real/grounding_heldout.py b/packages/flow/benchmark/rdp_avc_real/grounding_heldout.py new file mode 100644 index 0000000000..116258f988 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/grounding_heldout.py @@ -0,0 +1,254 @@ +"""Held-out test of the local grounding rung (#1596) on real AVC444 frames. + +The rung's thresholds (``ACCEPT_SCORE``, ``LOOKALIKE_MARGIN``) were set on a +synthetic Chromium fixture (``benchmark/pixel_grounding``). This script +applies it, unchanged, to frames captured from a real H.264/AVC444 RDP +session: for every click step of each campaign's compiled bundle, and every +condition whose run frames were kept, it shows the model the replay's +``before`` frame and the recorded crop, and records what the resolver's rule +makes of the look-alikes (``_model_grounded``), the look-alike count, the +true and best-other scores, the query time, and the template matcher on the +same crop for comparison. + +Ground truth is the place the target really was on that frame: + +- for a frame from a trial whose save the system of record confirmed, the + point that trial's replay clicked (its report's resolution); +- otherwise the recorded click point, when the condition leaves the layout + unchanged (healthy, 16-bit colour, Patient Notes reorder), shifted to the + right row for the Patient Notes look-alike rows (decoy, siblings); +- otherwise no truth: the row is reported as ``truth: null`` and not scored. + +A point is on target within ``TOL_X`` x ``TOL_Y`` pixels of the truth. + + python grounding_heldout.py [...] --output results_grounding_heldout.json +""" + +from __future__ import annotations + +import argparse +import json +import platform +import statistics +import sys +import tempfile +import time +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +TOL_X, TOL_Y = 24, 12 +PN_ROW_PITCH = 44 +# Rows above Neil Sorenson after searching "Neil", per pn_db.py's ordering. +PN_ROWS_ABOVE = {"decoy": 1, "siblings": 1} +LAYOUT_SAME = {"healthy", "bpp16", "reorder", "decoy", "siblings"} +REPEATS = 3 + + +def _load_bundle(campaign: Path, app: str, tmp: Path): + from openadapt_flow.ir import Workflow + + dest = tmp / f"{campaign.name}_bundle" + with zipfile.ZipFile(campaign / f"{app}_bundle.zip") as zf: + zf.extractall(dest) + return Workflow.load(dest), dest + + +def _verified_points(campaign: Path, trials: list[dict], condition: str) -> dict: + """step_id -> clicked point, from the condition's first trial if verified.""" + first = next((i for i, t in enumerate(trials) if t["condition"] == condition), None) + if first is None or trials[first]["class"] != "verified_save": + return {} + return { + s["step_id"]: tuple(s["point"]) for s in trials[first]["steps"] if s["point"] + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("campaigns", nargs="+", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + from openadapt_flow import vision + from openadapt_flow.runtime.local_grounder import ( + ACCEPT_SCORE, + LOOKALIKE_MARGIN, + LocalGroundingModel, + ) + from openadapt_flow.runtime.resolver import ( + TEMPLATE_THRESHOLD, + _model_grounded, + _scaled_click_point, + ) + from openadapt_flow.vision.ocr import OcrResolutionRefused + + model = LocalGroundingModel.load() + rows: list[dict] = [] + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + for campaign in args.campaigns: + trials = json.loads((campaign / "trials.json").read_text()) + app = "pn" if (campaign / "pn_bundle.zip").exists() else "emr" + workflow, bundle_dir = _load_bundle(campaign, app, tmp) + clicks = {s.id: s for s in workflow.steps if s.anchor is not None} + for run_zip in sorted(campaign.glob(f"{app}_run_*.zip")): + # _run_[_].zip (a stopped trial's frames) + condition = run_zip.stem.split("_run_", 1)[1] + for suffix in ( + "_false_stop", + "_stop_on_lookalike", + "_stopped_after_save", + ): + condition = condition.removesuffix(suffix) + verified = _verified_points(campaign, trials, condition) + with zipfile.ZipFile(run_zip) as zf: + names = set(zf.namelist()) + for step_id, step in clicks.items(): + name = f"steps/{step_id}_before.png" + if name not in names: + continue + png = zf.read(name) + anchor = step.anchor + if step_id in verified: + truth, source = verified[step_id], "verified_trial" + elif condition in LAYOUT_SAME: + x, y = anchor.click_point + if app == "pn" and abs(y - 145) <= 6: + y += PN_ROW_PITCH * PN_ROWS_ABOVE.get(condition, 0) + truth, source = (x, y), "recorded_layout" + else: + truth, source = None, None + template = (bundle_dir / anchor.template).read_bytes() + + def on_target(pt, truth=truth): + return ( + truth is not None + and abs(pt[0] - truth[0]) <= TOL_X + and abs(pt[1] - truth[1]) <= TOL_Y + ) + + timings, found = [], [] + for _ in range(REPEATS): + model._template_cache.clear() + started = time.monotonic() + found = model.locate_anchor(png, template) + timings.append((time.monotonic() - started) * 1000.0) + try: + grounded = _model_grounded( + anchor, png, vision, model.locate_anchor, template + ) + if grounded is None: + rung = "abstain" + elif truth is None: + rung = "picked_unscored" + else: + rung = ( + "target" + if on_target(grounded[0]) + else "wrong_target" + ) + except OcrResolutionRefused as exc: + rung = "halt:" + type(exc).__name__ + scored = model.candidates(png, template) + + def point_of(c, a=anchor): + return _scaled_click_point(a, tuple(c.region)) + + true_scores = [ + c.score for c in scored if on_target(point_of(c)) + ] + other_scores = [ + c.score for c in scored if not on_target(point_of(c)) + ] + started = time.monotonic() + tmatch = vision.find_template( + png, template, threshold=TEMPLATE_THRESHOLD + ) + template_ms = (time.monotonic() - started) * 1000.0 + tpoint = ( + _scaled_click_point(anchor, tuple(tmatch.region)) + if tmatch + else None + ) + rows.append( + { + "app": app, + "campaign": campaign.name, + "condition": condition, + "archive": run_zip.name, + "step_id": step_id, + "text_label": anchor.ocr_text, + "truth": list(truth) if truth else None, + "truth_source": source, + "rung_outcome": rung, + "lookalikes": len(found), + "true_found": any( + on_target(point_of(c)) for c in found + ), + "true_score": round(max(true_scores), 3) + if true_scores + else None, + "best_other_score": round(max(other_scores), 3) + if other_scores + else None, + "ms": round(statistics.median(timings), 1), + "template_found": bool(tpoint and on_target(tpoint)), + "template_wrong": bool( + tpoint and truth and not on_target(tpoint) + ), + "template_ms": round(template_ms, 1), + } + ) + print(json.dumps(rows[-1]), flush=True) + + scored = [r for r in rows if r["truth"] is not None] + summary = { + "frames_steps": len(rows), + "scored": len(scored), + "rung_outcomes": { + k: sum(1 for r in scored if r["rung_outcome"] == k) + for k in sorted({r["rung_outcome"] for r in scored}) + }, + "wrong_target": sum(1 for r in scored if r["rung_outcome"] == "wrong_target"), + "abstain": sum(1 for r in scored if r["rung_outcome"] == "abstain"), + "abstain_text_label": sum( + 1 for r in scored if r["rung_outcome"] == "abstain" and r["text_label"] + ), + "template_found": sum(1 for r in scored if r["template_found"]), + "template_wrong": sum(1 for r in scored if r["template_wrong"]), + "median_ms": round(statistics.median(r["ms"] for r in rows), 1) + if rows + else None, + "max_ms": max((r["ms"] for r in rows), default=None), + } + args.output.write_text( + json.dumps( + { + "schema": "openadapt.rdp-avc444-grounding-heldout.v1", + "generated_at": datetime.now(timezone.utc).isoformat( + timespec="seconds" + ), + "host": { + "machine": platform.machine(), + "system": platform.system(), + "python": sys.version.split()[0], + }, + "thresholds": { + "accept_score": ACCEPT_SCORE, + "lookalike_margin": LOOKALIKE_MARGIN, + "template_threshold": TEMPLATE_THRESHOLD, + "tolerance_px": [TOL_X, TOL_Y], + }, + "summary": summary, + "rows": rows, + }, + indent=2, + ) + ) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/rdp_avc_real/harness.py b/packages/flow/benchmark/rdp_avc_real/harness.py new file mode 100644 index 0000000000..ad8dd9b41c --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/harness.py @@ -0,0 +1,839 @@ +"""Harness for the real H.264/AVC444 remote-desktop lab. Runs on vm-client. + +``python harness.py ``, started by runner.py inside the +interactive ``harness`` session. Job kinds: + +- ``probe``: open the Remote Desktop Connection (mstsc) window for one app, + capture frames through the shipped ``RemoteDisplayBackend`` + + ``Win32WindowClient``, and save them with the window geometry. + +The backend is the unmodified product code; only the window selection +(process ``mstsc``, exact window title) is configured here. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import subprocess +import sys +import time +from pathlib import Path + +from PIL import Image + +from openadapt_flow.backends.remote_display import RemoteDisplayBackend + +APPS = { + "emr": {"target": "10.30.1.7"}, + "pn": {"target": "vm-rdhost"}, + # The read-back session: its own RDP logon (pnreader), own mstsc window. + "pnread": { + "target": "vm-rdhost.x3hkkcl3scte1kfhx4qhkg0tta.bx.internal.cloudapp.net" + }, +} +DESKTOP = (1280, 800) + + +def window_title(app: str) -> str: + return f"{APPS[app]['target']} - Remote Desktop Connection" + + +def open_session( + app: str, *, bpp: int = 32, desktop: tuple[int, int] = DESKTOP, wait_s: float = 60.0 +) -> None: + """Start mstsc for ``app`` unless its window already exists.""" + backend = RemoteDisplayBackend(owner_substr="mstsc", title_substr=window_title(app)) + if backend._client.find_windows("mstsc", window_title(app)): # noqa: SLF001 + return + # Settings go in Default.rdp, which mstsc reads for /v: connections; + # opening an unsigned .rdp file would raise a publisher warning instead. + rdp = Path.home() / "Documents" / "Default.rdp" + rdp.parent.mkdir(parents=True, exist_ok=True) + rdp.write_text( + "\n".join( + [ + f"desktopwidth:i:{desktop[0]}", + f"desktopheight:i:{desktop[1]}", + "screen mode id:i:1", + "smart sizing:i:0", + f"session bpp:i:{bpp}", + "authentication level:i:0", + "prompt for credentials:i:0", + "redirectclipboard:i:0", + "audiomode:i:2", + "connection type:i:7", + "networkautodetect:i:1", + "bandwidthautodetect:i:1", + "displayconnectionbar:i:0", + "autoreconnection enabled:i:1", + ] + ), + encoding="utf-16", + ) + subprocess.Popen( + ["mstsc", f"/v:{APPS[app]['target']}", f"/w:{desktop[0]}", f"/h:{desktop[1]}"] + ) + deadline = time.monotonic() + wait_s + while time.monotonic() < deadline: + if backend._client.find_windows("mstsc", window_title(app)): # noqa: SLF001 + return + time.sleep(1) + raise TimeoutError(f"mstsc window for {app} did not appear") + + +def close_sessions() -> None: + subprocess.run(["taskkill", "/im", "mstsc.exe", "/f"], capture_output=True) + + +def backend_for(app: str, **kwargs) -> RemoteDisplayBackend: + return RemoteDisplayBackend( + owner_substr="mstsc", + title_substr=window_title(app), + max_frame_age_s=kwargs.pop("max_frame_age_s", 10.0), + **kwargs, + ) + + +def save_frame(png: bytes, out: Path, name: str) -> dict: + path = out / f"{name}.png" + path.write_bytes(png) + w, h = Image.open(io.BytesIO(png)).size + return { + "file": path.name, + "sha256": hashlib.sha256(png).hexdigest(), + "size": [w, h], + } + + +def probe(job: dict, out: Path) -> dict: + app = job["app"] + open_session(app, bpp=job.get("bpp", 32)) + time.sleep(job.get("settle_s", 20)) + backend = backend_for(app) + frames = [] + for i in range(job.get("frames", 3)): + frames.append(save_frame(backend.screenshot(), out, f"{app}_probe_{i}")) + time.sleep(2) + windows = [ + {"title": w.title, "bounds": list(w.bounds), "pid": w.pid} + for w in backend._client.find_windows("mstsc", window_title(app)) # noqa: SLF001 + ] + return { + "app": app, + "viewport": list(backend.viewport), + "windows": windows, + "frames": frames, + } + + +def diagnose(job: dict, out: Path) -> dict: + """Full-screen grab plus every visible top-level window and its process.""" + import ctypes + from ctypes import wintypes + + from PIL import ImageGrab + + if job.get("app"): + try: + open_session(job["app"], wait_s=job.get("wait_s", 30)) + except TimeoutError: + pass + buf = io.BytesIO() + ImageGrab.grab(all_screens=True).save(buf, "PNG") + frame = save_frame(buf.getvalue(), out, "screen") + user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 + rows: list[dict] = [] + + @ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM) + def visit(hwnd, _lparam): + if not user32.IsWindowVisible(hwnd): + return True + title = ctypes.create_unicode_buffer(512) + user32.GetWindowTextW(hwnd, title, 512) + cls = ctypes.create_unicode_buffer(256) + user32.GetClassNameW(hwnd, cls, 256) + pid = wintypes.DWORD() + user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) + exe = ctypes.create_unicode_buffer(1024) + size = wintypes.DWORD(1024) + h = kernel32.OpenProcess(0x1000, False, pid.value) + if h: + kernel32.QueryFullProcessImageNameW(h, 0, exe, ctypes.byref(size)) + kernel32.CloseHandle(h) + rect = wintypes.RECT() + user32.GetWindowRect(hwnd, ctypes.byref(rect)) + texts: list[str] = [] + if cls.value == "#32770": + + @ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM) + def child(ch, _lp): + t = ctypes.create_unicode_buffer(1024) + user32.GetWindowTextW(ch, t, 1024) + if t.value: + texts.append(t.value) + return True + + user32.EnumChildWindows(hwnd, child, 0) + rows.append( + { + "title": title.value, + "class": cls.value, + "exe": exe.value, + "rect": [rect.left, rect.top, rect.right, rect.bottom], + "texts": texts, + } + ) + return True + + user32.EnumWindows(visit, 0) + return {"screen": frame, "windows": rows} + + +# --- Patient Notes campaign --------------------------------------------------- + +PN_ORACLE = "http://10.30.1.7:8770" +PN_TARGET_ID = 1 # Neil Sorenson +# Window-pixel points on the 1280x800 session, read off a probe frame. +PN_STEPS = { + "search_box": (260, 55), + "search_button": (559, 55), + "first_row": (115, 145), + "note_box": (315, 400), + "save_button": (560, 489), +} +# Conditions under which the target stays unambiguous: a stop is a false stop. +PN_MUST_SAVE = {"healthy", "reorder", "decoy", "bpp16", "desktop1440"} + + +def oracle(base: str, path: str, timeout: float = 60.0): + import urllib.request + + with urllib.request.urlopen(f"{base}{path}", timeout=timeout) as resp: + return json.loads(resp.read()) + + +def wait_steady(app: str, timeout_s: float = 45.0) -> None: + """Wait until the remote window is painted and two frames agree.""" + deadline = time.monotonic() + timeout_s + last = None + while time.monotonic() < deadline: + frame = backend_for(app).screenshot() + digest = hashlib.sha256(frame).hexdigest() + if digest == last: + return + last = digest + time.sleep(3) + + +def pn_record(app: str, rec_dir: Path) -> None: + from openadapt_flow.recorder import Recorder + + rec = Recorder( + backend_for(app), + rec_dir, + settle_interval_s=0.3, + settle_stable_frames=2, + settle_timeout_s=6.0, + ) + rec.click(*PN_STEPS["search_box"]) + rec.type_text("Neil") + rec.click(*PN_STEPS["search_button"]) + rec.click(*PN_STEPS["first_row"]) + rec.click(*PN_STEPS["note_box"]) + rec.type_text("recorded demonstration note", param="note") + rec.click(*PN_STEPS["save_button"]) + rec.finish() + + +def step_summary(r) -> dict: + res = r.resolution + return { + "step_id": r.step_id, + "intent": r.intent, + "ok": r.ok, + "skipped": r.skipped, + "rung": getattr(res.rung, "value", str(res.rung)) if res else None, + "confidence": round(res.confidence, 3) if res else None, + "point": list(res.point) if res else None, + "safety_halt": r.safety_halt, + "failure_category": getattr(r.failure_category, "value", r.failure_category), + "detail": (getattr(r, "error", None) or getattr(r, "message", None)), + } + + +def classify(db_outcome: str, success: bool, must_save: bool) -> str: + if db_outcome == "correct": + return "verified_save" if success else "stopped_after_save" + if db_outcome == "wrong_target": + return "silent_wrong" if success else "flagged_wrong" + if success: + return "false_completion" + return "false_stop" if must_save else "stop_on_lookalike" + + +def zip_dir(src: Path, dest: Path) -> None: + import zipfile + + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as zf: + for f in sorted(src.rglob("*")): + if f.is_file(): + zf.write(f, f.relative_to(src)) + + +def record_check_config(spec: dict): + """The merged record check (verify-run): effect contract plus whole-read frame.""" + from openadapt_flow.verify_run import parse_verify_config + + return parse_verify_config(spec["verify"], source=f"{spec['name']} verify") + + +def record_check( + cfg, params: dict, run_dir: Path, report_path: Path, baseline: Path +) -> dict: + """Judge the trial's write from the system of record, as verify-run does.""" + from openadapt_flow.verify_run import verify_run + + run_record = {"executor_kind": "flow_replay"} + if report_path.exists(): + run_record["report_sha256"] = hashlib.sha256( + report_path.read_bytes() + ).hexdigest() + try: + result = verify_run( + cfg, + params=params, + run_record=run_record, + authorization_ref=f"lab-{run_dir.name}", + baseline_path=baseline if baseline.exists() else None, + private_dir=run_dir / "record_check", + ) + except Exception as exc: # noqa: BLE001 -- a refused check is reported, not hidden + return {"effect_verdict": None, "outcome": "refused", "error": repr(exc)} + receipt = result["receipt"] + return { + "effect_verdict": receipt.get("effect_verdict"), + "outcome": receipt.get("outcome"), + "verification_tier": receipt.get("verification_tier"), + "checked_how": receipt.get("checked_how"), + } + + +def run_campaign(job: dict, out: Path, app: str, spec: dict) -> dict: + """Record once, compile, then replay every condition; judge from the record. + + ``spec``: record(rec_dir), reset(drift), params(token), judge(token) -> + (db_outcome, detail), must_save (condition names), name. + """ + from openadapt_flow.compiler import compile_recording + from openadapt_flow.runtime.replayer import Replayer + + work = Path(r"C:\oa\work") / job["id"] + work.mkdir(parents=True, exist_ok=True) + spec["reset"]("none") + open_session(app) + wait_steady(app) + save_frame(backend_for(app).screenshot(), out, f"{app}_record_start") + t0 = time.monotonic() + spec["record"](work / "rec") + workflow = compile_recording(work / "rec", work / "bundle", name=spec["name"]) + record_s = time.monotonic() - t0 + zip_dir(work / "bundle", out / f"{app}_bundle.zip") + + check_cfg = record_check_config(spec) if job.get("record_check") else None + trials = [] + session = (32, DESKTOP) + for cond in job["conditions"]: + name, drift = cond["name"], cond.get("drift", "none") + want = (cond.get("bpp", 32), tuple(cond.get("desktop", DESKTOP))) + for i in range(cond["n"]): + if want != session: + close_sessions() + time.sleep(3) + open_session(app, bpp=want[0], desktop=want[1]) + session = want + spec["reset"](drift) + wait_steady(app) + token = f"Oa{job['id'][-4:]}{len(trials):03d}{name[:4]}".replace("-", "") + run_dir = work / f"run_{len(trials):03d}_{name}" + first = backend_for(app).screenshot() + params = spec["params"](token) + baseline = work / f"baseline_{len(trials):03d}.json" + if check_cfg is not None: + from openadapt_flow.verify_run import capture_baseline + + capture_baseline(check_cfg, params, baseline) + t = time.monotonic() + error, extra = None, {} + try: + report = Replayer(backend_for(app), poll_interval_s=0.3).run( + workflow, + params=params, + bundle_dir=work / "bundle", + run_dir=run_dir, + ) + success, model_calls = bool(report.success), report.model_calls + rungs, terminal = dict(report.rung_counts), report.terminal_outcome + steps = [step_summary(r) for r in report.results] + extra = { + "execution_outcome": getattr( + report.execution_outcome, "value", report.execution_outcome + ), + "heal_count": report.heal_count, + "halt": report.halt.model_dump(mode="json") + if report.halt + else None, + "render_scale": report.render_scale, + } + except Exception as exc: # noqa: BLE001 -- a crash is a stop, recorded + success, model_calls, rungs, terminal, steps = False, 0, {}, None, [] + error = repr(exc) + elapsed = time.monotonic() - t + db_outcome, detail = spec["judge"](token) + if check_cfg is not None: + detail["record_check"] = record_check( + check_cfg, params, run_dir, run_dir / "report.json", baseline + ) + must_save = name in spec["must_save"] + frames = sorted(run_dir.rglob("*.png")) + trial = { + "trial": len(trials) + 1, + "condition": name, + "drift": drift, + "bpp": want[0], + "desktop": list(want[1]), + "token": token, + "success": success, + "terminal_outcome": terminal, + "error": error, + "model_calls": model_calls, + "rung_counts": rungs, + "steps": steps, + **extra, + "db_outcome": db_outcome, + "record_detail": detail, + "class": classify(db_outcome, success, must_save), + "must_save": must_save, + "elapsed_s": round(elapsed, 2), + "first_frame_sha256": hashlib.sha256(first).hexdigest(), + "frame_sha256": [ + hashlib.sha256(f.read_bytes()).hexdigest() for f in frames + ], + } + report_path = run_dir / "report.json" + if report_path.exists(): + (out / f"report_{len(trials):03d}.json").write_bytes( + report_path.read_bytes() + ) + if i == 0: + trial["first_frame"] = save_frame(first, out, f"{app}_{name}_first") + zip_dir(run_dir, out / f"{app}_run_{name}.zip") + elif trial["class"] != "verified_save" and not any( + t["condition"] == name and t["class"] == trial["class"] for t in trials + ): + zip_dir(run_dir, out / f"{app}_run_{name}_{trial['class']}.zip") + trials.append(trial) + (out / "trials.json").write_text( + json.dumps(trials, indent=2), encoding="utf-8" + ) + return {"app": app, "record_s": round(record_s, 2), "trials": trials} + + +def pn_campaign(job: dict, out: Path) -> dict: + def judge(token: str): + holders = [r["id"] for r in oracle(PN_ORACLE, "/all") if r["note"] == token] + outcome = ( + "correct" + if holders == [PN_TARGET_ID] + else "wrong_target" + if holders + else "no_write" + ) + return outcome, {"note_holders": holders} + + return run_campaign( + job, + out, + "pn", + { + "name": "avc444-patient-notes", + "record": lambda rec_dir: pn_record("pn", rec_dir), + "reset": lambda drift: oracle(PN_ORACLE, f"/reset?drift={drift}"), + "params": lambda token: {"note": token}, + "judge": judge, + "must_save": PN_MUST_SAVE, + "verify": { + "environment_id": "rdp-avc444-lab-patient-notes", + "verifier": { + "kind": "rest", + "base_url": PN_ORACLE, + "records_path": "/all", + "records_key": None, + }, + "effects": [ + { + "kind": "field_equals", + "match": { + "first": {"literal": "Neil"}, + "last": {"literal": "Sorenson"}, + }, + "field": "note", + "value": {"param": "note"}, + } + ], + }, + }, + ) + + +# --- OpenEMR campaign ---------------------------------------------------------- + +EMR_ORACLE = "http://10.30.1.6:8771" +EMR_LAST = "Avclab" +EMR_DOB = "2001-01-01" +# Window-pixel points on the 1280x800 session, read off drive frames. +EMR_STEPS = { + "username": (705, 396), + "password": (684, 450), + "login": (640, 558), + "patient_menu": (361, 43), + "new_search": (378, 71), + "first_name": (445, 311), + "last_name": (706, 311), + "dob": (429, 423), + "sex": (965, 423), + "billing_label": (146, 599), + "create": (272, 733), + "confirm": (600, 261), +} +EMR_MUST_SAVE = {"healthy", "bpp16", "desktop1440"} + + +def emr_record(rec_dir: Path, order: str = "last_first") -> None: + from openadapt_flow.recorder import Recorder + + rec = Recorder( + backend_for("emr"), + rec_dir, + settle_interval_s=0.4, + settle_stable_frames=2, + settle_timeout_s=10.0, + ) + st = EMR_STEPS + # Lab fixture account from benchmark/openemr_live/docker-compose.yml; + # passed as run parameters so no credential is compiled into the bundle. + rec.click(*st["username"]) + rec.type_text("admin", param="username") + rec.click(*st["password"]) + rec.type_text("pass", param="password") + rec.click(*st["login"]) + # The main screen loads for several seconds after login; a click before it + # paints would record a blank crop as the menu's template. + time.sleep(12) + rec.click(*st["patient_menu"]) + rec.click(*st["new_search"]) + time.sleep(6) + if order == "first_last": + rec.click(*st["first_name"]) + rec.type_text("Recorded", param="first_name") + rec.click(*st["last_name"]) + rec.type_text(EMR_LAST) + else: + # Fixed last name first: no later click's identity band then holds a + # value that changes from run to run (see README, "identity band"). + rec.click(*st["last_name"]) + rec.type_text(EMR_LAST) + rec.click(*st["first_name"]) + rec.type_text("Recorded", param="first_name") + rec.click(*st["dob"]) + rec.type_text(EMR_DOB) + rec.press("escape") + rec.click(*st["sex"]) + rec.type_text("F") + rec.press("enter") + rec.click(*st["billing_label"]) + rec.press("end") + rec.click(*st["create"]) + time.sleep(3) + rec.click(*st["confirm"]) + rec.finish() + + +def emr_judge(token: str): + rows = oracle(EMR_ORACLE, f"/patients?token={token}") + exact = [ + r + for r in rows + if r["fname"] == token + and r["lname"] == EMR_LAST + and r["DOB"] == EMR_DOB + and r["sex"] == "Female" + ] + if len(exact) == 1 and len(rows) == 1: + outcome = "correct" + elif rows: + outcome = "wrong_target" # a field landed elsewhere, or a duplicate row + else: + outcome = "no_write" + return outcome, {"rows": rows} + + +def emr_campaign(job: dict, out: Path) -> dict: + prefix = "Oa" + job["id"][-4:].replace("-", "") + + def reset(_drift: str) -> None: + # Each trial starts from a record with none of this job's patients: + # an earlier trial's patient would raise OpenEMR's duplicate warning. + oracle(EMR_ORACLE, f"/cleanup?prefix={prefix}") + oracle(EMR_ORACLE, "/cleanup?prefix=Recorded") + oracle(PN_ORACLE, "/edge_restart") + time.sleep(4) + + try: + return run_campaign( + job, + out, + "emr", + { + "name": "avc444-openemr-new-patient", + "record": lambda rec_dir: emr_record( + rec_dir, job.get("order", "last_first") + ), + "reset": reset, + "params": lambda token: { + "username": "admin", + "password": "pass", + "first_name": token, + }, + "judge": emr_judge, + "must_save": EMR_MUST_SAVE, + "verify": { + "environment_id": "rdp-avc444-lab-openemr", + "verifier": { + "kind": "rest", + "base_url": EMR_ORACLE, + "records_path": "/all", + "records_key": None, + }, + "effects": [ + { + "kind": "record_written", + "match": {"fname": {"param": "first_name"}}, + }, + { + "kind": "field_equals", + "match": {"fname": {"param": "first_name"}}, + "field": "lname", + "value": {"literal": EMR_LAST}, + }, + { + "kind": "field_equals", + "match": {"fname": {"param": "first_name"}}, + "field": "DOB", + "value": {"literal": EMR_DOB}, + }, + { + "kind": "field_equals", + "match": {"fname": {"param": "first_name"}}, + "field": "sex", + "value": {"literal": "Female"}, + }, + ], + "frame": {"identity_field": "pid"}, + }, + }, + ) + finally: + oracle(EMR_ORACLE, "/cleanup?prefix=Oa" + job["id"][-4:].replace("-", "")) + oracle(EMR_ORACLE, "/cleanup?prefix=Recorded") + + +# --- Separate-session read-back (desktop_readback) -------------------------------- + +PN_READBACK_SPEC = { + "session": { + "kind": "rdp", + "rdp_window": "mstsc", + "rdp_window_title": window_title("pnread"), + }, + "search": { + "box": {"region": [20, 43, 480, 24]}, + "query": {"literal": "Neil"}, + "submit": {"text": "Search"}, + }, + "open": {"results": {"region": [20, 125, 590, 175]}}, + "ready": {"text": "Clinical note:"}, + "identity": { + "reader": {"text": "Patient:", "at": "right"}, + "value": {"literal": "Neil Sorenson"}, + "field": "patient", + }, + "fields": {"note": {"control": {"text": "Clinical note:", "at": "below"}}}, + "navigation_timeout_s": 20.0, + "settle_s": 1.0, +} + + +def pn_readback(job: dict, out: Path) -> dict: + """Score the read-back verifier against notes written straight to the record. + + Each case writes ``stored`` to Neil Sorenson's note, then asks the + verifier whether the note equals ``expected``. Truth: CONFIRMED when they + match, REFUTED when they do not; INDETERMINATE is a stop, never a pass. + """ + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from runner import store_credentials + + from openadapt_flow.runtime.effects.desktop_readback import ( + DesktopReadbackSpec, + DesktopSessionFacts, + ) + from openadapt_flow.runtime.effects.desktop_readback_sessions import ( + BackendSessionDriver, + build_desktop_readback_verifier, + ) + from openadapt_flow.runtime.effects.effect import Effect, EffectKind, ValueExpr + + store_credentials() + spec = DesktopReadbackSpec.model_validate(PN_READBACK_SPEC) + + def open_backend(_user: str, _password: str): + # Each look-up starts the reader's app afresh, so no look-up can read + # a record view an earlier one left on screen. + open_session("pnread") + oracle(PN_ORACLE, "/reader_restart") + wait_steady("pnread") + facts = DesktopSessionFacts( + own_session=True, fresh_session=True, signed_in=False, path="ocr" + ) + return backend_for("pnread"), facts + + driver = BackendSessionDriver(open_backend, settle_s=spec.settle_s) + verifier = build_desktop_readback_verifier(spec, {}, driver=driver, timeout_s=0.0) + oracle(PN_ORACLE, "/reset?drift=none") + cases = [] + for i in range(job["n"]): + stored = f"Rb{i:03d}note" + match = i % 2 == 0 + expected = stored if match else f"Rb{i:03d}other" + oracle(PN_ORACLE, f"/set_note?id={PN_TARGET_ID}¬e={stored}") + effect = Effect( + kind=EffectKind.FIELD_EQUALS, + match={"patient": ValueExpr(literal="Neil Sorenson")}, + field="note", + value=ValueExpr(literal=expected), + timeout_s=0.0, + ) + t = time.monotonic() + before = verifier.capture_pre_state() + verdict = verifier.verify(effect, before) + elapsed = time.monotonic() - t + got = getattr(verdict.verdict, "value", str(verdict.verdict)) + truth = "confirmed" if match else "refuted" + cases.append( + { + "case": i + 1, + "stored": stored, + "expected": expected, + "truth": truth, + "verdict": got, + "correct": got.lower() == truth, + "false_confirm": (not match) and got.lower() == "confirmed", + "reason": verdict.reason, + "assurance": verdict.assurance, + "elapsed_s": round(elapsed, 2), + } + ) + if i < 2: + save_frame(backend_for("pnread").screenshot(), out, f"pnread_case{i + 1}") + (out / "readback.json").write_text( + json.dumps(cases, indent=2), encoding="utf-8" + ) + from openadapt_flow.verification import verifier_effect_tier + + return {"tier": str(verifier_effect_tier(verifier)), "cases": cases} + + +def drive(job: dict, out: Path) -> dict: + """Exploration: run a list of actions through the backend, frame after each. + + Actions: ["click", x, y] | ["type", text] | ["key", name] | ["wait", s] + | ["oracle", path] (a call to the host reset service). + """ + app = job["app"] + open_session(app) + time.sleep(job.get("settle_s", 3)) + backend = backend_for(app) + frames = [save_frame(backend.screenshot(), out, f"{app}_drive_00")] + for i, act in enumerate(job["actions"], start=1): + verb = act[0] + if verb == "click": + backend.click(int(act[1]), int(act[2])) + elif verb == "type": + backend.type_text(act[1]) + elif verb == "key": + backend.press(act[1]) + elif verb == "wait": + time.sleep(float(act[1])) + elif verb == "scroll": + backend.scroll(int(act[1]), int(act[2])) + elif verb == "oracle": + oracle(PN_ORACLE, act[1]) + time.sleep(job.get("step_wait_s", 2.0)) + if verb != "wait" or job.get("frame_after_wait"): + frames.append(save_frame(backend.screenshot(), out, f"{app}_drive_{i:02d}")) + return {"app": app, "frames": frames} + + +def main(job_path: str, out_dir: str) -> int: + job = json.loads(Path(job_path).read_text(encoding="utf-8-sig")) + out = Path(out_dir) + if job.get("fresh"): + close_sessions() + time.sleep(3) + kinds = { + "probe": probe, + "diagnose": diagnose, + "pn_campaign": pn_campaign, + "emr_campaign": emr_campaign, + "pn_readback": pn_readback, + "drive": drive, + } + try: + result = kinds[job["kind"]](job, out) + (out / "result.json").write_text(json.dumps(result, indent=2), encoding="utf-8") + return 0 + except Exception: + import traceback + + (out / "error.json").write_text( + json.dumps({"error": traceback.format_exc()}, indent=2), encoding="utf-8" + ) + return 1 + finally: + upload(job, out) + + +def upload(job: dict, out: Path) -> None: + """PUT every output file to the job's write-only blob SAS, if given.""" + sas = job.get("upload_sas") + if not sas: + return + import urllib.request + + base, query = sas.split("?", 1) + for path in sorted(out.iterdir()): + if not path.is_file(): + continue + req = urllib.request.Request( + f"{base}/{job['id']}/{path.name}?{query}", + data=path.read_bytes(), + method="PUT", + headers={"x-ms-blob-type": "BlockBlob"}, + ) + urllib.request.urlopen(req, timeout=120).close() + + +if __name__ == "__main__": + raise SystemExit(main(*sys.argv[1:])) diff --git a/packages/flow/benchmark/rdp_avc_real/oracle_service.py b/packages/flow/benchmark/rdp_avc_real/oracle_service.py new file mode 100644 index 0000000000..b200f96f96 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/oracle_service.py @@ -0,0 +1,223 @@ +"""Reset and system-of-record service for the AVC444 lab (stdlib only). + +Runs beside each app, never in the session the harness drives: + +- ``pn`` mode on vm-rdhost: reseeds the Patient Notes SQLite record with a + drift condition, restarts the app in pnuser's session, and dumps every row. +- ``emr`` mode on vm-emr: reads ``patient_data`` from OpenEMR's MariaDB + through ``docker exec`` and deletes rows a trial created. + +The harness asks it for the record after each trial; it never reads the +screen to decide whether a save happened. The listener is bound to the +private address and the host firewall admits only the client VM. + + python oracle_service.py pn 10.30.1.7 8770 + python oracle_service.py emr 10.30.1.6 8771 +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse + +PY = r"C:\Program Files\Python312\python.exe" +PN_DB = r"C:\oa\pn_db.py" +CLIENT_IP = "10.30.1.5" +TOKEN_RE = re.compile(r"^[A-Za-z0-9 _-]{1,64}$") +EMR_SQL = [ + "docker", + "exec", + "-i", + "openemr-live-mysql-1", + "mariadb", + "-uroot", + "-proot", + "openemr", + "--batch", + "--raw", + "-e", +] + + +def pn_reset(drift: str) -> dict: + if drift not in ("none", "siblings", "decoy", "reorder"): + raise ValueError(f"unknown drift {drift!r}") + subprocess.run( + [ + "powershell", + "-NoProfile", + "-Command", + "Get-CimInstance Win32_Process -Filter \"Name='powershell.exe'\" | " + "Where-Object { $_.ProcessId -ne $PID -and " + "$_.CommandLine -like '*patient_notes.ps1*' } | " + "ForEach-Object { Stop-Process -Id $_.ProcessId -Force }", + ], + check=True, + capture_output=True, + ) + seeded = subprocess.run( + [PY, PN_DB, "seed", "--drift", drift], + check=True, + capture_output=True, + text=True, + ).stdout + subprocess.run( + ["schtasks", "/run", "/tn", "oa-app-pnuser"], check=True, capture_output=True + ) + time.sleep(6) + return {"seeded": json.loads(seeded), "restarted": True} + + +def edge_restart() -> dict: + """Restart OpenEMR's Edge window in emruser's session (cookies kept).""" + subprocess.run( + [ + "powershell", + "-NoProfile", + "-Command", + "Get-Process msedge -IncludeUserName -ErrorAction SilentlyContinue | " + "Where-Object { $_.UserName -like '*emruser' } | Stop-Process -Force", + ], + check=True, + capture_output=True, + ) + time.sleep(2) + subprocess.run( + ["schtasks", "/run", "/tn", "oa-app-emruser"], check=True, capture_output=True + ) + time.sleep(8) + return {"restarted": True} + + +def pn_set_note(pid: str, note: str) -> dict: + """Write a note straight into the record: read-back ground truth.""" + import base64 + + if not pid.isdigit() or not TOKEN_RE.match(note): + raise ValueError("bad note") + b64 = base64.b64encode(note.encode()).decode() + out = subprocess.run( + [PY, PN_DB, "save", pid, b64], check=True, capture_output=True, text=True + ).stdout + return json.loads(out) + + +def reader_restart() -> dict: + """Restart Patient Notes in the read-back session, so each look-up starts fresh.""" + subprocess.run( + [ + "powershell", + "-NoProfile", + "-Command", + "Get-CimInstance Win32_Process -Filter \"Name='powershell.exe'\" | " + "Where-Object { $_.ProcessId -ne $PID -and " + "$_.CommandLine -like '*patient_notes.ps1*' -and " + "(Invoke-CimMethod -InputObject $_ -MethodName GetOwner).User -eq 'pnreader' } | " + "ForEach-Object { Stop-Process -Id $_.ProcessId -Force }", + ], + check=True, + capture_output=True, + ) + subprocess.run( + ["schtasks", "/run", "/tn", "oa-app-pnreader"], check=True, capture_output=True + ) + time.sleep(6) + return {"restarted": True} + + +def pn_all() -> list: + out = subprocess.run( + [PY, PN_DB, "all"], check=True, capture_output=True, text=True + ).stdout + return json.loads(out) + + +def emr_query(sql: str) -> list[dict]: + out = subprocess.run( + EMR_SQL + [sql], check=True, capture_output=True, text=True + ).stdout + lines = out.splitlines() + if not lines: + return [] + cols = lines[0].split("\t") + return [dict(zip(cols, line.split("\t"))) for line in lines[1:]] + + +def emr_patients(token: str) -> list[dict]: + if not TOKEN_RE.match(token): + raise ValueError("bad token") + return emr_query( + "SELECT pid, fname, lname, DOB, sex FROM patient_data " + f"WHERE fname LIKE '%{token}%' OR lname LIKE '%{token}%' ORDER BY pid" + ) + + +def emr_all() -> list[dict]: + """Every patient row: the record check's whole read set.""" + return emr_query( + "SELECT pid, fname, lname, DOB, sex FROM patient_data ORDER BY pid" + ) + + +def emr_cleanup(prefix: str) -> dict: + if not TOKEN_RE.match(prefix) or len(prefix) < 4: + raise ValueError("bad prefix") + rows = emr_patients(prefix) + emr_query( + f"DELETE FROM patient_data WHERE fname LIKE '{prefix}%' OR lname LIKE '{prefix}%'" + ) + return {"deleted": len(rows)} + + +class Handler(BaseHTTPRequestHandler): + mode = "pn" + + def _reply(self, code: int, body: object) -> None: + data = json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _route(self) -> object: + if self.client_address[0] not in (CLIENT_IP, "127.0.0.1"): + raise PermissionError(self.client_address[0]) + url = urlparse(self.path) + q = {k: v[0] for k, v in parse_qs(url.query).items()} + routes = { + ("pn", "/reset"): lambda: pn_reset(q.get("drift", "none")), + ("pn", "/all"): pn_all, + ("pn", "/edge_restart"): edge_restart, + ("pn", "/reader_restart"): reader_restart, + ("pn", "/set_note"): lambda: pn_set_note(q["id"], q["note"]), + ("emr", "/patients"): lambda: emr_patients(q["token"]), + ("emr", "/all"): emr_all, + ("emr", "/cleanup"): lambda: emr_cleanup(q["prefix"]), + } + return routes[(self.mode, url.path)]() + + def do_GET(self) -> None: # noqa: N802 + try: + self._reply(200, self._route()) + except PermissionError: + self._reply(403, {"error": "forbidden"}) + except Exception as exc: # noqa: BLE001 -- report, keep serving + self._reply(500, {"error": repr(exc)}) + + def log_message(self, fmt: str, *args) -> None: + sys.stderr.write("%s %s\n" % (self.client_address[0], fmt % args)) + + +def main(mode: str, host: str, port: str) -> None: + Handler.mode = mode + ThreadingHTTPServer((host, int(port)), Handler).serve_forever() + + +if __name__ == "__main__": + main(*sys.argv[1:]) diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-1/reports.tar.gz b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-1/reports.tar.gz new file mode 100644 index 0000000000..5a5ac499cd Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-1/reports.tar.gz differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-1/trials.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-1/trials.json new file mode 100644 index 0000000000..979b711364 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-1/trials.json @@ -0,0 +1,6374 @@ +[ + { + "trial": 1, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1000heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 602, + 254 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "2", + "fname": "Oaun1000heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ] + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 128.42, + "first_frame_sha256": "36c576e719f09090473b3e08e566d00b4015cffa7b4aca67ce144c2ae36d09e0", + "frame_sha256": [ + "13b25797886d5c9327209c3497439655086a6d8420c75f5e00ff9d2948986224", + "f4b7e3d4bcf0f1a4c2e0e1e8b38d833e33b3d71edc51b8a8a373238027bfa763", + "5a0c4af8b7a91830a8543ba0538ceb533ec29127e7934e271fd9c3abf2ab8d98", + "36c576e719f09090473b3e08e566d00b4015cffa7b4aca67ce144c2ae36d09e0", + "5a3fae47b15fcc105f0de140221c8180c4ef2e06a8b57d722c3b2ded71792f7e", + "5a0c4af8b7a91830a8543ba0538ceb533ec29127e7934e271fd9c3abf2ab8d98", + "51d537f2e0ca1e28988b20715a20668978d9ced8f2ec68be1bf262270db390e3", + "5a3fae47b15fcc105f0de140221c8180c4ef2e06a8b57d722c3b2ded71792f7e", + "de6555a96afd1ea3a90ae2b253bcb92c1c84a4d7efb72b62ed51d98441a13518", + "51d537f2e0ca1e28988b20715a20668978d9ced8f2ec68be1bf262270db390e3", + "8004dd03f3c14635c5de588a20103f5dbf4ffe096ccc0c93d0c8ff1c1ba4347e", + "de6555a96afd1ea3a90ae2b253bcb92c1c84a4d7efb72b62ed51d98441a13518", + "b3b5a9eaadf54eca19c60206fd35f3c28478eb800913ad5c48d6a41956146959", + "75d2b08fefa5caeed9dc4a93fd73ddc58ca80173a42c2a2ceb83a9be15f660cf", + "0e20b447d902e8f0411d0bedcb39a9d6c6f8820d9a13aef3f0804de36a49914a", + "b3b5a9eaadf54eca19c60206fd35f3c28478eb800913ad5c48d6a41956146959", + "3c083f7c0ff8be7fd7b8c9e9b6edc27f302ea9a41bb225f88739fb8cbfe91bc4", + "de782feb3f167e10cf3795398ca23165558ca0b7716ec868c9c2d64226d7cb78", + "43bfe1c965f8a65e3631259aadbbced407577ddbfa22d667e7bc27a4bfd35269", + "1e11a49c8248cefc6a970128b7bc23a061b3d609fd6256295c638a826b711cdf", + "dbfe62c0f349492a5d87be873b58354c428377f046ad94ac5cd0413d7eb2baa8", + "b51196f9b0232028ec3ea2141abb8e6989663a75ddf6f64b93e472ed61e2099a", + "e530e0c286ad1f8b958871afa84582c59745cfa16c31b69bf3c3de0f4b5f036e", + "716c2fd4414153234a31a8518551937a649980edf42d53d493ed5e30b62948ae", + "7c685ff4472280fec3e27595c5b4e43eb573339eaadb0b3e17f0e8ebb5a614c2", + "e530e0c286ad1f8b958871afa84582c59745cfa16c31b69bf3c3de0f4b5f036e", + "924efbadd22abd648e2ddb64498f06f33e32c12aaea50fcf97df82271d91d363", + "206275a325fa22deafde92489474d297cb8229429d5442602e9aad0bf79c1b9e", + "e3def7f08c046e49cdc6fc92e9b9a27405e06b3848d648cafc40f4dd8e3dd0fc", + "e3def7f08c046e49cdc6fc92e9b9a27405e06b3848d648cafc40f4dd8e3dd0fc", + "16df660cf7b409c45589222a1c00ebe7aa137f9e99cbc32106b7863d54d61f37", + "16ca93467c87b4fc15bab6dcae53b2a37f1f12f891df0cfc5c2f3ab014ed40e8", + "afa765dc5ba0a6b9638b99f9de1642cb5f65b00e9b917b357097966a0e397f11", + "16df660cf7b409c45589222a1c00ebe7aa137f9e99cbc32106b7863d54d61f37", + "6fa9d0696be8c55703eb9044f3ba4849d2f62c3c56f09e1c3f4c731eed6921f4", + "afa765dc5ba0a6b9638b99f9de1642cb5f65b00e9b917b357097966a0e397f11", + "5ecaac4655233188c0df248d216660ef84744bfcefed59ee260b61e418c4754b", + "6fa9d0696be8c55703eb9044f3ba4849d2f62c3c56f09e1c3f4c731eed6921f4", + "e5c1fd30ec21d386d56d163e2095211e87bcef0fc6eb0dd5c5047441d5d3f7e6", + "5ecaac4655233188c0df248d216660ef84744bfcefed59ee260b61e418c4754b", + "58dc53e415030777db51e32da05109bb79fdff72690a332abd67ad0f9a9e19f5", + "b1ed20436864bebc5251483d09f33ccc7d0829b79b3c687cee4b3612551487f9", + "156e2c1b985b6d8349c0bb9dcc1116b4cd700c2ecc2ca876a6945569dc9a0c62", + "13b25797886d5c9327209c3497439655086a6d8420c75f5e00ff9d2948986224" + ], + "first_frame": { + "file": "emr_healthy_first.png", + "sha256": "36c576e719f09090473b3e08e566d00b4015cffa7b4aca67ce144c2ae36d09e0", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 2, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1001heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 0.917, + "point": [ + 602, + 369 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "reason": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "\u00d7", + "Calendar Finder Flow", + "emograpr", + "1 - 2 of 2", + "Calendar", + "36", + "Mess", + "Preferred", + "Birth", + "External", + "Previous", + "External ID:", + "Hits", + "Name", + "DOB", + "Sex", + "ID", + "Names", + "License/ID:", + "2", + "Avclab,", + "2001-01-", + "Female", + "User Defined:", + "Oaun1000heal", + "01", + "Billing Note:", + "Avclab,Recorded", + "Previous Nam", + "Confirm Create New Patient", + "Cancel", + "Contact", + "Choices", + "Employer", + "Stats", + "Misc", + "Guardian", + "Insurance", + "Search", + "Create New Patient", + "8", + "O Type here to search", + "\u8425", + "2:26 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape", + "click 'Unassigned'", + "type 'F'", + "press enter", + "click 'Billing Note:'", + "press end", + "click 'Create New Patient'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 87.3, + "first_frame_sha256": "d5df3e681033553be683bf636142374d52ffa6a7a7c061cb8b860bce56817477", + "frame_sha256": [ + "9267968af7f960e147ba66733125b7f2bc9d01959e51b6be186aea57aec11eaa", + "d5df3e681033553be683bf636142374d52ffa6a7a7c061cb8b860bce56817477", + "0769c8884694c2f7f1f721bd8d6ed994c7771b2e02c1342b0402c0aead351f8f", + "9267968af7f960e147ba66733125b7f2bc9d01959e51b6be186aea57aec11eaa", + "25ac0f1cc04579beffc4e18409532777b28ab4fdb4952a2dd9300037e89569c0", + "0769c8884694c2f7f1f721bd8d6ed994c7771b2e02c1342b0402c0aead351f8f", + "77a13f0722dbb6bfa5c1b4451f8ad772bcd67112935b6a14d3e65986a98830e0", + "25ac0f1cc04579beffc4e18409532777b28ab4fdb4952a2dd9300037e89569c0", + "d085ba1c0c9fdf94cedfccf5c157feef61ac79d5bd5976c34b8ef25731078f15", + "f7fe77aeacc38d6f9a03f61a7451e865ef7198b70203d8611156576e972f3e8f", + "bd4589a876d5d63a2adc14220188377baa08d743cbbf2b80076d16c81b4e23f6", + "a3fc367be5271bf24cb56269839a71dfabff7ae7d4a5890ff538d652a6afb603", + "32cb7927eb7876978b65b1009f768d0666e663955e9b2a80feab70b1b2cb6b62", + "bd4589a876d5d63a2adc14220188377baa08d743cbbf2b80076d16c81b4e23f6", + "e73f1cffde353b3ebd5eddcb0bb2ae8cb48c02d401b1dd7032edf50aa2921cb4", + "9589695767f4bda74fc76ad456ea5f020d793234ba96a09067f458664427992c", + "5a534b2c9c1f6d1cebe1c57bea753e6c7d8a428a0855fa1b740df13be08b1f62", + "5361f5e048e7c9d7523b4b2ab18ddb76cb9c5929ddab2a7ebd5d38111413ad5f", + "922aceff36f289106a236419762e217364baf4a4a844519fc0e54665b6e80405", + "722808f795ba04c843d946aa4fce7fccbf80b41396bbf0ed1a1bd52c70e17774", + "09a5f2568b5e8bd1c8547bf65b88af2b966ef9c014af1195be8cc0cdf35a27a8", + "6e6dc04fa275ef4671313937b62c4b4432f67547dd62a8269dc5cf2952cfc0e6", + "04393291c22215411ff56463044f3ba51b48958f212416cdd696ade02da8e514", + "09a5f2568b5e8bd1c8547bf65b88af2b966ef9c014af1195be8cc0cdf35a27a8", + "c661fac009d1441c02dbd893f0f5d7ca486760d6032d26c95f77714a856ea841", + "1a3ae7a957327d41d750485a4a73beb4de6b8abf6b7602913c4375ea77b396b6", + "3e79e5be8f25d8d0f5278004da91414567127cc22b78516d882469a46429a5d9", + "3e79e5be8f25d8d0f5278004da91414567127cc22b78516d882469a46429a5d9", + "8fd4f3ff33eff99c9585c004b32b17efcb28c983111e886539372cb38ba8c7bd", + "ef838157214662d34301429e829d82de775a875f5bd718117d11018350d7a8eb", + "55f07f4d80bfa2090796e91b763ee7ddd837b008c2121946e776adee054726d5", + "8fd4f3ff33eff99c9585c004b32b17efcb28c983111e886539372cb38ba8c7bd", + "cb3e1d66928448d2c259958955c03b10697e825aec04d53d00ad0dde9fd7505a", + "55f07f4d80bfa2090796e91b763ee7ddd837b008c2121946e776adee054726d5", + "9c490fc21b0efe29293eec8970f07738bdac67b85f8a14034aaddcfe73383c94", + "cb3e1d66928448d2c259958955c03b10697e825aec04d53d00ad0dde9fd7505a", + "d6ea371945b359ee99d53ce77c7e6aba0ea684c5b0207f096679819a83f26537", + "9c490fc21b0efe29293eec8970f07738bdac67b85f8a14034aaddcfe73383c94", + "ce2b8b6770ee8ca572ed329e1b08ede0bcaf2f9cef4e3f88af11460e9afdc10b", + "21932e20ee3654eccd72cb089f282c3c89e244b51062e03b79899a311be091da", + "ce2b8b6770ee8ca572ed329e1b08ede0bcaf2f9cef4e3f88af11460e9afdc10b", + "ce2b8b6770ee8ca572ed329e1b08ede0bcaf2f9cef4e3f88af11460e9afdc10b" + ] + }, + { + "trial": 3, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1002heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1002heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:28 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 85.19, + "first_frame_sha256": "dec95fe1c35d9453da73aa426151c11fa704795254a8483c98114f179e10ae2a", + "frame_sha256": [ + "bbb6900508f88fcbc72d8274f04275fd843a83e84a74dbf2bcb07107381851ec", + "dec95fe1c35d9453da73aa426151c11fa704795254a8483c98114f179e10ae2a", + "73917f4607c12796497f64c321cf1a62db1ee5967ad6a1b0b55c4a9523723a22", + "bbb6900508f88fcbc72d8274f04275fd843a83e84a74dbf2bcb07107381851ec", + "0a4ecc251868a23a089a5fa49bf467f3e7c4db47270f5329a42fe60b0aea732a", + "73917f4607c12796497f64c321cf1a62db1ee5967ad6a1b0b55c4a9523723a22", + "115b1c81118d067f8e4b7ffdcb5f99e0c009d9b4728314e5ba74ce3d856ea32d", + "0a4ecc251868a23a089a5fa49bf467f3e7c4db47270f5329a42fe60b0aea732a", + "fff9789fc9ab79d5b325c89d27a9ae76b13328b2cf4b9ea59bf6790a152e31e7", + "115b1c81118d067f8e4b7ffdcb5f99e0c009d9b4728314e5ba74ce3d856ea32d", + "7937c5a55e88c6e06d0f29baec0d117bd0b0c8d6c0903bff4239a80fcd2d1d1a", + "9965e6ce61acd24f161869e2729ec6adce046c4024e6e044fd452bb801fbd9f6", + "09b364bcd551149e9fae3538b5e36342929a37398fe0190381f11349cccab75b", + "7937c5a55e88c6e06d0f29baec0d117bd0b0c8d6c0903bff4239a80fcd2d1d1a", + "d8c10e5d333492349aa2ed9b6d497cf107e0a014348152d89672ec671045ddbf", + "4fdaf9d4e3dec69190b08b95bb0f3efbd0cdfc5acc5c755d563e76f3a15593b7", + "c36a332d1cbcfbe5fa3d2df6775b22fdf283f1e5d8a083e9ff6443bcd2d8caf7", + "e70171a2cbecc65d787cac4b3333ee2eb7846ab9769d2d8f3590e5a17a1eacea", + "885d350966bc8f2e0c627032c59351d97cee8d976f06a8434b0135aef760eb0d", + "0c2b36aa5c4657ace027888703497fae2d4b4146e00b7e363a4d54aff75f6679", + "c787319c3866a831af0b570ffe484a16fa7fd07a232b05ddacbc6e10b937e450", + "885d350966bc8f2e0c627032c59351d97cee8d976f06a8434b0135aef760eb0d", + "3ec829b649543e53013cbcd1323de4cc2925898ade6dab8816aca274c6372f00", + "c787319c3866a831af0b570ffe484a16fa7fd07a232b05ddacbc6e10b937e450", + "5abe20f2498bfdecaf262a284b80726b231112ad248fd91c9783ee801292516f", + "dce2b372ea073dfd1d1cd96e39691744f2f33aaebf23b6a466c096544ea606dc", + "5abe20f2498bfdecaf262a284b80726b231112ad248fd91c9783ee801292516f", + "5abe20f2498bfdecaf262a284b80726b231112ad248fd91c9783ee801292516f", + "8f65407415dd577a3d888f6a47921ea1a20fcc0b27a1117edb1e992abbe30785", + "8f65407415dd577a3d888f6a47921ea1a20fcc0b27a1117edb1e992abbe30785" + ] + }, + { + "trial": 4, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1003heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1003heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:29 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 78.61, + "first_frame_sha256": "c27f926224a6b58ec620f4cd2089d13ae8ba7363d150bedfd51bb2323ecd127c", + "frame_sha256": [ + "a32a7e5941c9ab97d5541f918d26b91dd8a2a0f0ba66c3360881e316e27f3c37", + "c27f926224a6b58ec620f4cd2089d13ae8ba7363d150bedfd51bb2323ecd127c", + "6862a6583c89b7d33151d1c8e20b44906e4a191938e85a94f2f249f1380995c2", + "a32a7e5941c9ab97d5541f918d26b91dd8a2a0f0ba66c3360881e316e27f3c37", + "0216e6c7abb2d6080d08d09e955a2b82f6fc8914ddaa3ea8c0bbb52233004407", + "6862a6583c89b7d33151d1c8e20b44906e4a191938e85a94f2f249f1380995c2", + "25b8ece0a34d8c5d7865f7cf596ce0f1ca38da25f1256296a69aafa83153607e", + "0216e6c7abb2d6080d08d09e955a2b82f6fc8914ddaa3ea8c0bbb52233004407", + "be373e51adc7f9a8a60b82761103f5fe3a351f32b7adac34787187d759052cf4", + "65f7bcfb2d24185a31b310ac396fb0f45623a2bad5bda5fcfff8ab0a8924cd33", + "74ea81c855386f3d5be82fb51844b21da1358d3f0b7149f34267fc7ab215ea5a", + "11ea0fe2027d8a5cd9e1b334362919612c85063dff7864ddb797360c56f0d8e5", + "621303c0b70b7608ba1cabe0360e2a2721aaea2da062b0e8f7e306e944d13c97", + "74ea81c855386f3d5be82fb51844b21da1358d3f0b7149f34267fc7ab215ea5a", + "89fcd3e535c01602b89d8725eb12512867469ab4acb2bc88d70845b97e0eafec", + "4cd47de8090d565b947035a2540b124a644c9bfc2afa1868ead78d8887166086", + "2be8d1118010d854dd6389305a49ca59dc5ecfb9258db4783d3e16df0604a95b", + "03f7c992ff26ead5ffdc5ab77d3118cd91cf354a692185ef63152534aa54806f", + "2c326db015caffc9ba781f935477f676e819889afbfea0a1ed84f32a265159fc", + "3f14d10fa6ee80bc32344ab06ee67ab77786d1a3f0d7576113fa4858e6b69eeb", + "3276cb4a88e0364a5ffd7e512a1478a952b100afa963e4cbc3db7907292995aa", + "2c326db015caffc9ba781f935477f676e819889afbfea0a1ed84f32a265159fc", + "98484dce5d1222c8a27bc3b70cbbd8ad9046c7240e274025b60096149b6098bd", + "3276cb4a88e0364a5ffd7e512a1478a952b100afa963e4cbc3db7907292995aa", + "ce49f0f874b542287ee5126839f42de5bf8c865e64f198fe6c4978cce40062cf", + "b414c8367d38109173250f6a2e819314b64a3a350d31dbae34653d2e5567ef80", + "9899241ed1ba090cc59b1a028fa3d263acfa47159269b7ce8471e203059b447e", + "ce49f0f874b542287ee5126839f42de5bf8c865e64f198fe6c4978cce40062cf", + "9899241ed1ba090cc59b1a028fa3d263acfa47159269b7ce8471e203059b447e", + "9899241ed1ba090cc59b1a028fa3d263acfa47159269b7ce8471e203059b447e" + ] + }, + { + "trial": 5, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1004heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1004heal", + "\u2460", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Middle Ne", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:31 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 82.58, + "first_frame_sha256": "5bccb3aef74202a2bf735b5ce315a0179f9993298fce85f6d2f119104511d366", + "frame_sha256": [ + "0126acb06e602d22266a4bc7e2a330ebbf4c2e780b0504088628d522e866b99a", + "5bccb3aef74202a2bf735b5ce315a0179f9993298fce85f6d2f119104511d366", + "4b8b4b6aa93d42272e6f60cd1cc6015046e42898785b83818e4389fcda80fc19", + "0126acb06e602d22266a4bc7e2a330ebbf4c2e780b0504088628d522e866b99a", + "dbb46e1e67aa7c646187fe10dba6a21cbeafe34914591a832f8cbd795203fc89", + "4b8b4b6aa93d42272e6f60cd1cc6015046e42898785b83818e4389fcda80fc19", + "450b88453ee934f00635e98f7595dfeac2c79fd1a6266ab3ba48a8559081b62c", + "dbb46e1e67aa7c646187fe10dba6a21cbeafe34914591a832f8cbd795203fc89", + "88cbaa1993f83c3b7afbbb6bfe75bc4dc31ff872bc2fdf7f71ebeece5c405cb0", + "450b88453ee934f00635e98f7595dfeac2c79fd1a6266ab3ba48a8559081b62c", + "2e0096c1e7a39a2263d0830e7eb39fe90c27638d332b7d2b30f1b01f23fc979c", + "9fc777dfa4777cc6033d6e0a2665221f4ea82e0ba7270630f1a1805badaa43fc", + "5f56e321d39df5e2ed32c8a98d2b99dd806bedc2676c494b097f398dd049f723", + "2e0096c1e7a39a2263d0830e7eb39fe90c27638d332b7d2b30f1b01f23fc979c", + "6f758e260697b3f8d37e41495c91890d30394ced0a5cc6716b5cf2fc90d6e391", + "7b8cbffa75ec665c65839ece79ad08688de9f411378cd94c3249e2dd1cb9891c", + "5de47728b7bfe564bb53589a0984625c8d694eb5dbb5fc2bd70091f1848c565f", + "f81693ec9df6bd7efd8898adf16f3482326d2798b2d2aaef83707559135466a7", + "e16a1cb750b5e2b04ee386a698c152b8ba593a4a1f891661545b560baca76719", + "6f5e9c3e42e5cded36740ced89d52f74d619aee46475dc9f7724bff1638601b9", + "a47c5a4241b946e95ec65d11c5cdd54d4f0d7d90873a8c96b8cdc27dc66b1a1d", + "e16a1cb750b5e2b04ee386a698c152b8ba593a4a1f891661545b560baca76719", + "e58c446e80bdff5ff89c6c891e26a7d95fd0189757c74a9fa7c737d4dbeabeff", + "a47c5a4241b946e95ec65d11c5cdd54d4f0d7d90873a8c96b8cdc27dc66b1a1d", + "807886edf30729c68b75573ef072569c40e73a6bfefca17e578aeb51c2440cd9", + "6f704b312032a9b3a5ce06dad5fdee8f95443b550b70a3c59d77189e7f430dd6", + "c57211e4c0e17477504b4e0f58fbb52114a4b14c012d2989f2c22f87ea8a4162", + "807886edf30729c68b75573ef072569c40e73a6bfefca17e578aeb51c2440cd9", + "c57211e4c0e17477504b4e0f58fbb52114a4b14c012d2989f2c22f87ea8a4162", + "c57211e4c0e17477504b4e0f58fbb52114a4b14c012d2989f2c22f87ea8a4162" + ] + }, + { + "trial": 6, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1005heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1005heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:33 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 86.31, + "first_frame_sha256": "3352a3913acd2c4bcb00097ab7007031e794eab73877da07a736890d29c331ba", + "frame_sha256": [ + "f7d396584006ee825d66ca91fd113576efbf586cb48760e2694fab6e4376470a", + "3352a3913acd2c4bcb00097ab7007031e794eab73877da07a736890d29c331ba", + "e8016683e528066725e8083975dc118edd15f630575e6f244fbdedcdf61591a2", + "f7d396584006ee825d66ca91fd113576efbf586cb48760e2694fab6e4376470a", + "25bec4789a0520c66dbbc7384b11a66d19fef2a6e93d06435e43a2f03b98cf0b", + "e8016683e528066725e8083975dc118edd15f630575e6f244fbdedcdf61591a2", + "6049e72a1947852aea1205fd74a08b58847be1b85e7a47df9910c59852eafd17", + "25bec4789a0520c66dbbc7384b11a66d19fef2a6e93d06435e43a2f03b98cf0b", + "892048c683e35320cff8a2b087c2b65fcd3410d4a5d18848b03d71107980d102", + "6049e72a1947852aea1205fd74a08b58847be1b85e7a47df9910c59852eafd17", + "9d847b4a43e1465f75f1f72cd856c9e5fcd73d21f866963671ac63d7865aa4c4", + "db41029b66367097c9baaae231cf8659f96bfbdccb2c2954df509afc51eb8646", + "b46185b3dbe1437d1599127ff107e9ff51bf6d0407f3947948487b946cb4037f", + "9d847b4a43e1465f75f1f72cd856c9e5fcd73d21f866963671ac63d7865aa4c4", + "00af9f41b1d92e2a79af0999a531a22c91a4ed4d8a9075db29cf940aaa1a8148", + "4a7ed804419033d727fed16b166aecbc26166694f6411311577f90b2005f3450", + "eabeeccc110540559c0c81d0d5ece76e9f0de450238b6a2a5f5e6f3620e9d4c7", + "41c6b2a6ff3483c24b7d6de6ada7960957ef5bad527c9e2d97b0adcd643a0ebe", + "05cd4bd29a49fc49a17afcc8b01804cf5830a5b851cf3c6036a74b48582cde02", + "5bca60b40744f14a883bee6b1f4289ac4f8a75160aa7c8a9b6b6a62ef54d2439", + "c92c64b2eedb19b763db6a33ec935025734274ff65dbcf93b923c5b0d2bad05c", + "05cd4bd29a49fc49a17afcc8b01804cf5830a5b851cf3c6036a74b48582cde02", + "95e0dfe15b99d8bd49447ff751bbfa3e9d7966ee49e798e15297d96cbeca642a", + "c92c64b2eedb19b763db6a33ec935025734274ff65dbcf93b923c5b0d2bad05c", + "de5c6888668384c809da39aaddba3d5658beb4f4f7c19ac636c09789ba44d710", + "fe3883e9368d9616d3072a99d623fca264fca16ce60b4880e8ffad34819c2f50", + "e8ba55c32650371cbd74ab5c82543adce2f1fca0fc02e23b5bb2b06a3228f724", + "de5c6888668384c809da39aaddba3d5658beb4f4f7c19ac636c09789ba44d710", + "e8ba55c32650371cbd74ab5c82543adce2f1fca0fc02e23b5bb2b06a3228f724", + "e8ba55c32650371cbd74ab5c82543adce2f1fca0fc02e23b5bb2b06a3228f724" + ] + }, + { + "trial": 7, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1006heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1006heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:34 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 79.76, + "first_frame_sha256": "9b2c6430cfaa3c5bf9eadc2a4554fb20f7ccc8e4958f979d975a6a180ea038ae", + "frame_sha256": [ + "bec68373b4e57e0d38682e76105a663d7fba67ca1fbb8bceeb9aa69c58417f71", + "9b2c6430cfaa3c5bf9eadc2a4554fb20f7ccc8e4958f979d975a6a180ea038ae", + "76a7b7b5ffa8c4337aaaa8b9fc6c62a18bf8dec674af8a27e5289dcabac19687", + "bec68373b4e57e0d38682e76105a663d7fba67ca1fbb8bceeb9aa69c58417f71", + "53780cf45b967b021d417030fb49a0b98ecb79c31b9048235a70c7f93cef9548", + "76a7b7b5ffa8c4337aaaa8b9fc6c62a18bf8dec674af8a27e5289dcabac19687", + "a4245d9f6404191b017582838b06b1ba5c3d1b3a3cec5541b074f8a4af0cdff3", + "53780cf45b967b021d417030fb49a0b98ecb79c31b9048235a70c7f93cef9548", + "15698de51b63193c4442221e26d39b681056205a3dce290cb203c49a4ce18283", + "661c5d394660baab1edbd3db377584360701769513a4fd9e0ce5f960485a2846", + "79975b863446fba9d5ef35aa6238032b00698987b927a459c75dee2f6b8ebe8f", + "a6bdfb96d55b962866405d51ed0e22e063b7246a734886ad85126e023a061f2e", + "c17cc7391468594f9334ab6a15d9b3d2494d40d4f45573c51c66498e96c40d52", + "79975b863446fba9d5ef35aa6238032b00698987b927a459c75dee2f6b8ebe8f", + "04ac5fa012a5096fed8c65c0b63b22836c3f73fd763dc8f5003b0774b69f45e0", + "da845958e15cfb70e1d648d7d1e18566f6c4a5eddf2c26ce84aee76012d213a4", + "177109d6a8641746954df7d4b22929028a4c2d1cb72dd266500b5a20d239d5f7", + "15bb0ad5ebab44dc8c46515c6f423ef0584af6c911724f636976050d642e16c1", + "fd34e0dc8e3149d96da51935a6f4a797c31b661a3a6792305083e1c3df33d8ed", + "5a7f5b225178f102741d461371f98d2c6a4e097a0d48227a21b5f8cc5bb8ff32", + "26914fd6e21e632043d891fbe3265b1d1e766dc87a7b083256292ec9b1f5b56f", + "fd34e0dc8e3149d96da51935a6f4a797c31b661a3a6792305083e1c3df33d8ed", + "4a82909ad8e87fef94ef2f257e05b809ae83385b3822df32d13600821ae1500c", + "26914fd6e21e632043d891fbe3265b1d1e766dc87a7b083256292ec9b1f5b56f", + "75e97e0645deb63da7856c6e21e68eca95118b03d39b09912ecadb66b2986026", + "4f6f0dc8378fb14d4bea0862de78aa91bcfdb6b53006da82a235fa826da103fe", + "4bcb8efa1231c7390eaabd97040e8824cfbf215fc65a5def1d40ff12bf0af545", + "75e97e0645deb63da7856c6e21e68eca95118b03d39b09912ecadb66b2986026", + "4bcb8efa1231c7390eaabd97040e8824cfbf215fc65a5def1d40ff12bf0af545", + "4bcb8efa1231c7390eaabd97040e8824cfbf215fc65a5def1d40ff12bf0af545" + ] + }, + { + "trial": 8, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1007heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1007heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:36 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 80.47, + "first_frame_sha256": "b653da06ccf51952e0a177b14d97a8f8141c502fd0c923d08df443e867c4cef2", + "frame_sha256": [ + "604d93e9c0a59237498615fd6ee6deb8bbe10254c4b5423037b41731b802d7c4", + "b653da06ccf51952e0a177b14d97a8f8141c502fd0c923d08df443e867c4cef2", + "1978760f00cdea693bf418863d10f0db52c7096139ba047f4d953988a410cff1", + "604d93e9c0a59237498615fd6ee6deb8bbe10254c4b5423037b41731b802d7c4", + "867f341eca077c46e71f6d13fb177be1b94ea32a3079a7e1e5aa4ac5b5610b53", + "1978760f00cdea693bf418863d10f0db52c7096139ba047f4d953988a410cff1", + "68e36b535add363e17ccaf72fb22e039f1f29b77bd4e42ff05405026d06a616f", + "867f341eca077c46e71f6d13fb177be1b94ea32a3079a7e1e5aa4ac5b5610b53", + "a17fdfef01c23d3955b05f17286f22baf8f7767a37111796da4486d6710d69af", + "13369430ba0ebb5cb698624330d66f94e6d4cadfbcc9e98da231f1729916f6c3", + "1a366dad2e369ac66519041b74cd96a5d5ecf400cc143faef7e641953e6feee7", + "335b244d7e26cc7bd543e6cd9a0768c0b7eb522a9d1e5460f30f5d85534991be", + "73906517d65fd1585caf68ac8dc998b3fc83260f88d163e1fe8b0a10be89a132", + "1a366dad2e369ac66519041b74cd96a5d5ecf400cc143faef7e641953e6feee7", + "405519c03c16ac0d938f2bb45590ce0d667c01be9e83ad94e710357946df9b43", + "a3c0358f4bc3e918af7b110ce51c0917824095dc2269958481fc79013d47de99", + "f1e1bc3fea84512e8c4f9eb9ac80a1b8ed2e6d37bca72d5384850f805208dbd7", + "608512e90c2b035d4ffeb62ef950667b3c44b2b2236a7f1f1a88d016743679b4", + "8ae14d9110c05f85e82a77054f14520c05df623bcd88ed3bfd7286b5bce67879", + "ca1a44df658750bb3812030d45f5c12a863ecb496ec52bf9fa34442c06742398", + "c541717d1e418b2e83fbb4d0c75593a6c9c28609e06fd3b60cf258c33e14b373", + "9af2fdaaaa459d8e9c609542ad1f6107aa4add30883e6d75d8f0827a2c4f51ca", + "8d5a7bec53fb32af99a24227c6e2ff475f46efc7f235e5856a8217b1f276cab8", + "ccd1b0baa5f8ff1756748e3e5003f45ce2aa6f8fc048ee1fb1a546f0180362d1", + "254e3c0d095cca3b469df38ae1585dfe8838c74154fdd8e8dd4326697c4ec06c", + "e16f655d532109014acf8aca744eb43292870322cdc69a4f6aa03c7e6fee4926", + "73ec6e6aee665f829460b20cc1375f6f04d150d26b51c0263c45b42362122560", + "73ec6e6aee665f829460b20cc1375f6f04d150d26b51c0263c45b42362122560", + "0660ba0dd2b7aac7e6e8963a2348a9a117fe15e27c69ae31d6a6a9985b2108c1", + "0660ba0dd2b7aac7e6e8963a2348a9a117fe15e27c69ae31d6a6a9985b2108c1" + ] + }, + { + "trial": 9, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1008heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1008heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:38 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 83.67, + "first_frame_sha256": "e0aaf85c5b212450b5709c65f8b1a31b6a2eb5fd8ee4c1cbb6910b4e9d6db21e", + "frame_sha256": [ + "c6806290852c2fcdf39fe347a934788f56c12ebe07906883eb8b9da9a737d88b", + "e0aaf85c5b212450b5709c65f8b1a31b6a2eb5fd8ee4c1cbb6910b4e9d6db21e", + "881e28ca6c8c6c0cc573d927730d276a8578381a73b9dbd47389b73fb520ab85", + "c6806290852c2fcdf39fe347a934788f56c12ebe07906883eb8b9da9a737d88b", + "39ab5cd97c27b67f76be94e307c1d235814f89e080551e3c80cffcd6fdecf255", + "881e28ca6c8c6c0cc573d927730d276a8578381a73b9dbd47389b73fb520ab85", + "49ebbb73a15d6de2c705a0978cbc5cf69f3ea180594456f485e74a54e6f42e21", + "39ab5cd97c27b67f76be94e307c1d235814f89e080551e3c80cffcd6fdecf255", + "b36c4ebbaebaf3a4cbbcbd178750faa154994323f2a34f8cdb79883fe3945144", + "980a013be73e61b05148b85d771b6a2dc02ca8be376904cbf3661035a492b9a9", + "7e02495083761a5ce374c16f93aa6345b87afdecc5d430960a8b56c7f8d5e6cb", + "961258a60906ccd73ce7a5110db894dd0c2058e8d39a42aa3492309273a06e25", + "bd8f3a596e857dc4a1e5453c7293d204bcd767d9b5692299278f0a23531fd7d3", + "7e02495083761a5ce374c16f93aa6345b87afdecc5d430960a8b56c7f8d5e6cb", + "3c7c0a6799a89031698417d561695894eaf98ff6a9405e59095984305004011b", + "91f81db4eb3e48b183c66b95869db0a9cb6fa257bc7e82856b15c3ea3acf3878", + "0ee830dae7d5e5b668b282ce551026b7c66ce8fd617d0bcadbb34f6629547f30", + "f32c7cf849ade1194dec56ad30db0df98816c129e147568f9f0dc73f0e85fdc3", + "982f0f6e5382a7aef6a25ff469b6036afa88ac85671d91c86745f2437402599c", + "67332d82aeb597b3fdcc8ffeef9fbbf33bdfa569b52bda655cfdad56624db170", + "b8f3bbbbb85e5e37e82198bc9a2b87cd58ad82605c12175b7828d81abe5c03d3", + "df546416b639c7f7e91b39411d9ec8ebf6808a4868730de79561c2f524979440", + "9dd0b455d28f91d89d6bd5157b84c10cd8e43d0ceeaa40b3084ca701a93b9334", + "b8f3bbbbb85e5e37e82198bc9a2b87cd58ad82605c12175b7828d81abe5c03d3", + "3fbaae8ebe67286e9497c12038527577e98fe93166be725792d23bc6c7518f2b", + "d656edc7604cda71e73ee5fdfd3f9e601cc8488ec81034dff56fc5e608f1e517", + "677d0d4d9ebd65c0f0dcc54dc3168a47c28ae9068f4dfb980a2d70b25dc1c7c1", + "3fbaae8ebe67286e9497c12038527577e98fe93166be725792d23bc6c7518f2b", + "677d0d4d9ebd65c0f0dcc54dc3168a47c28ae9068f4dfb980a2d70b25dc1c7c1", + "677d0d4d9ebd65c0f0dcc54dc3168a47c28ae9068f4dfb980a2d70b25dc1c7c1" + ] + }, + { + "trial": 10, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1009heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun1009heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:39 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 79.41, + "first_frame_sha256": "a71371ffb5afa0b72b6193c4a1e7b98b36c9fb523f3f4f57ed8a05cc27fddb9a", + "frame_sha256": [ + "4257a9a1ed1f0cc02d3a042a13fbf89172e7cdf25ad3b88de9e989069f747726", + "a71371ffb5afa0b72b6193c4a1e7b98b36c9fb523f3f4f57ed8a05cc27fddb9a", + "a0f85d4de508190e2295fff3a45155d28b68c8a29b4335128743bb226948d32d", + "4257a9a1ed1f0cc02d3a042a13fbf89172e7cdf25ad3b88de9e989069f747726", + "fe76d9269f8d352865eab0c837c2d73af03f848c26d8c2b338b11e1616e8d53b", + "a0f85d4de508190e2295fff3a45155d28b68c8a29b4335128743bb226948d32d", + "fb6e661de9ade0f945434a3398fb12c6313a4cb31d2bfa1d908a69f070bfc98d", + "fe76d9269f8d352865eab0c837c2d73af03f848c26d8c2b338b11e1616e8d53b", + "9b2c460997cec16d8ee28f45ae30e54c7a9f5d61afb436ed9322c38749b0d01e", + "b67f2c4f2e1f7da8c2e66d3e754948caf38abf18f54f7b7d5c232a240f4c7c64", + "030f3bb03ebd81c141fb45d8758bb7a5813da848a36e696fe9786f33ee6a4107", + "5f5e48123ce4b08bcf058d1a4da2aadfd657ca5dd89243b602f1faa5a0dbd51c", + "84e570b840aca2a4bf3b6634066968eb82c719e0db5f8f876cd7c61dc72101db", + "030f3bb03ebd81c141fb45d8758bb7a5813da848a36e696fe9786f33ee6a4107", + "c597a2cae983abadf8ebe68bb8806e63aaabb2a908393df3eed186e51c4fb6b4", + "b9d2328b70fffee968b16593d9c6da924ea9526e19315684fd65b76e2ea54d56", + "6855c76e54844646c8779c5cd0c8cdbfe4c71916420dbee3e961490f4e89f0cd", + "b51e508e11ff4fb7256ad1e8b1b3c037783b32cac2e178b6a75a0786f7726a65", + "550f49fa9c848ef531a7a33c15fbeaefbf1032ba4dd566b9fcdb8983043d6d46", + "567e8617f03727b15f73061e66419f4bafcf6709a54535d35104b6832bfa49f5", + "7bc3e91d66fba5ae5f677e59d7bf64dc7bd73d7acc49529cefeed4f6e01ae95e", + "550f49fa9c848ef531a7a33c15fbeaefbf1032ba4dd566b9fcdb8983043d6d46", + "8ed108b4145abb1a7421ce843b72bc43f41d1f6b036c8c54081c192099e7a507", + "7bc3e91d66fba5ae5f677e59d7bf64dc7bd73d7acc49529cefeed4f6e01ae95e", + "5c95353b44d6c8fbcd39daac332ed06b90735bbfd661dd1227304eb29a6fdb16", + "0eb777e483b18987a0da28b6df46299d025de8f7098fb5db5358c8d541565888", + "d544f9d5acf27e2913689d903c6ff9fd7090b036188b786d774c118ec6bac6aa", + "5c95353b44d6c8fbcd39daac332ed06b90735bbfd661dd1227304eb29a6fdb16", + "d544f9d5acf27e2913689d903c6ff9fd7090b036188b786d774c118ec6bac6aa", + "d544f9d5acf27e2913689d903c6ff9fd7090b036188b786d774c118ec6bac6aa" + ] + }, + { + "trial": 11, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1010bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 0.917, + "point": [ + 602, + 369 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "reason": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "\u00d7", + "Calendar Finder Flow", + "emograpl", + "1 - 2 of 2", + "Calendar", + "36", + "Mess", + "Preferred", + "Birth", + "External", + "Previous", + "External ID:", + "Hits", + "Name", + "DOB", + "Sex", + "ID", + "Names", + "License/ID:", + "2", + "Avclab,", + "2001-01-", + "Female", + "User Defined:", + "Oaun1000heal", + "01", + "Billing Note:", + "Avclab,Recorded", + "Previous Nam", + "Confirm Create New Patient", + "Cancel", + "Contact", + "Choices", + "Employer", + "Stats", + "Misc", + "Guardian", + "Insurance", + "Search", + "Create New Patient", + "8", + "O Type here to search", + "\u8425", + "2:41 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape", + "click 'Unassigned'", + "type 'F'", + "press enter", + "click 'Billing Note:'", + "press end", + "click 'Create New Patient'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 102.02, + "first_frame_sha256": "083bf6dca3d0a2f3579c92ffb83a36c174e643e006d5eb1820c4596ea54133d0", + "frame_sha256": [ + "c8c4ff171d5f338ce3e49cdeb9828d82fbb41c344b82534a5e24c654cf204b94", + "083bf6dca3d0a2f3579c92ffb83a36c174e643e006d5eb1820c4596ea54133d0", + "97f8b0d690be2ebffbc3ce6d831e273b703920e962f3f78fa5c2b3b572ec6631", + "c8c4ff171d5f338ce3e49cdeb9828d82fbb41c344b82534a5e24c654cf204b94", + "e81f5baee95c0103f839439e1988e599290d4b4a3bf31d42e79a1b37fdb982ad", + "97f8b0d690be2ebffbc3ce6d831e273b703920e962f3f78fa5c2b3b572ec6631", + "588f1220eb6f3790fbb7e0dc340fade029d35a6cba72730d8964e420ff56a9e7", + "e81f5baee95c0103f839439e1988e599290d4b4a3bf31d42e79a1b37fdb982ad", + "c3b80c504ef348fcaacace68566683c26b923783f2fbff158c9acc254f439335", + "588f1220eb6f3790fbb7e0dc340fade029d35a6cba72730d8964e420ff56a9e7", + "abb43324d5f50e0997cb021c3b9dab398f75005752744acd7e7b93ffec1b1313", + "c3be9d3136d83ff7c8a2053623ac0b0053f0b21fc79962a9d6ee981c9c19807d", + "b7346f370b3fc3a8ce8757167800934c9d668c6f21c4f99b763e0c4b69a690c6", + "abb43324d5f50e0997cb021c3b9dab398f75005752744acd7e7b93ffec1b1313", + "2aa80cfba55f5d62fe0c6fa1018cb82804ecd8c5713e78586bd48b218bfd7de8", + "18ebbb894fa3fab8a720bb84e65abe447f2306b00600cbd3056b34f1ee6b9c25", + "699ae5b0b7382565e3763a20b5dbc8e5eca8e5de266e7f987e6806ddb9a04ceb", + "d645227fccb2dae2eddb7a644c559db294b5290d805f3b024ae992d4e9c0d3f7", + "5b9fde489f035706425ac9a328c83a7626eedf08f9bdba393f88e57bf6d66678", + "6e2db2ac455f9f5dac066e9ca8985c12a2ef8913617669e5963a854a385a968c", + "66228e108914f97c6710f92f3873b750475c90d3fc2bfabc7b6e7ec56b0e4f2b", + "221fab1f5a5df0ce29e41dbbb00402fc00fed0c4158b294692fe6a7fb9e7e12a", + "344c25df98451c128f968d225334d763e9334c4434241c4634bc2d07ea8129fb", + "66228e108914f97c6710f92f3873b750475c90d3fc2bfabc7b6e7ec56b0e4f2b", + "fc7688d7f625416db192d734ba94fa2e29dd945140fb13b0ae9d64799c365b7c", + "1ae43308425b091e993653c2371c1379252a988955188895ee5a8041281be2ce", + "7cdf8b2f6613660d923725e12ffad01604be69feb839ca6bb77753c70a3173eb", + "fc7688d7f625416db192d734ba94fa2e29dd945140fb13b0ae9d64799c365b7c", + "d7ccb8b91115151b82a93573e9e744e55f73810388b486d0144b1448f2c78012", + "7cdf8b2f6613660d923725e12ffad01604be69feb839ca6bb77753c70a3173eb", + "6d5d1670af6544073f0d80033da05822a270b33b4188abe495a686b14faa8571", + "d7ccb8b91115151b82a93573e9e744e55f73810388b486d0144b1448f2c78012", + "4d5ab550d0fe4314d99bb2125ee27eb297f2e716b5d9e335aafe196a37540fed", + "6d5d1670af6544073f0d80033da05822a270b33b4188abe495a686b14faa8571", + "d84ce7f343862c469dd6ec6f2660948ba6eecb5a6858520b6b4ff627be38cb52", + "4d5ab550d0fe4314d99bb2125ee27eb297f2e716b5d9e335aafe196a37540fed", + "9e016fd24e292fdb395bfbf4831ebe1ba3a09dbe58f061311dd4fc060994c0c4", + "d84ce7f343862c469dd6ec6f2660948ba6eecb5a6858520b6b4ff627be38cb52", + "263f52b6fe3e5a9ac24e4871d152f00b3192fbdc5925aa00b1c40d29e1984c65", + "b3577e2fdfc31b099dba2eb40e6321bfe627317c30cc78b97defc929f0642e30", + "263f52b6fe3e5a9ac24e4871d152f00b3192fbdc5925aa00b1c40d29e1984c65", + "263f52b6fe3e5a9ac24e4871d152f00b3192fbdc5925aa00b1c40d29e1984c65" + ], + "first_frame": { + "file": "emr_bpp16_first.png", + "sha256": "083bf6dca3d0a2f3579c92ffb83a36c174e643e006d5eb1820c4596ea54133d0", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 12, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1011bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 0.917, + "point": [ + 602, + 369 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "reason": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "\u00d7", + "Calendar Finder Flow", + "emograpl", + "1 - 2 of 2", + "Calendar", + "36", + "Mess", + "Preferred", + "Birth", + "External", + "Previous", + "External ID:", + "Hits", + "Name", + "DOB", + "Sex", + "ID", + "Names", + "License/ID:", + "2", + "Avclab,", + "2001-01-", + "Female", + "User Defined:", + "Oaun1000heal", + "01", + "Billing Note:", + "Avclab,Recorded", + "Previous Nam", + "Confirm Create New Patient", + "Cancel", + "Contact", + "Choices", + "Employer", + "Stats", + "Misc", + "Guardian", + "Insurance", + "Search", + "Create New Patient", + "8", + "O Type here to search", + "\u8425", + "2:43 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape", + "click 'Unassigned'", + "type 'F'", + "press enter", + "click 'Billing Note:'", + "press end", + "click 'Create New Patient'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 91.26, + "first_frame_sha256": "d16b63ec21f12d6793dd788a1571237cc40697dac061ee9fab8f7cb8dca4529c", + "frame_sha256": [ + "44d7d795736006e676a3493fbe5e97f4c89369b3576bfd9b32029c390e78c052", + "d16b63ec21f12d6793dd788a1571237cc40697dac061ee9fab8f7cb8dca4529c", + "90f69d98127a965f3ea4c7acae1e617e2db5edf5538934f474aa4cb269166267", + "44d7d795736006e676a3493fbe5e97f4c89369b3576bfd9b32029c390e78c052", + "1ec500aaaa121512c13a094c415c11a23eb1aae30a8cbd98a74d0a384bbbdb5b", + "90f69d98127a965f3ea4c7acae1e617e2db5edf5538934f474aa4cb269166267", + "5182fdfa823fc21a2fdf9053ed07d396bbf8718d6f23955ce3268fde6e3dee9e", + "1ec500aaaa121512c13a094c415c11a23eb1aae30a8cbd98a74d0a384bbbdb5b", + "e4f02a9000629ea2a26f1295f408bc50ce91b9fa8095859bb200afe915040aca", + "452aa841816d705804e882e60f0270f265616eeb6a05a250c551951033dbb12d", + "355e354129ac8e82d9914dde83bc59f617174eaba293f326f96ea052d4349a8b", + "1fad695ec28dd850085b521ea7c3d76d83a5ece2991d10e8812160ecf424bb99", + "848461634740cc1fdc2707f12817cfbbb0675cefff51e105c59bb88a2b48021c", + "355e354129ac8e82d9914dde83bc59f617174eaba293f326f96ea052d4349a8b", + "eedf58b3827419eda761de7b891f84152df0612e19708afd3b14668acf80eb5c", + "9be53b2df6dbd111d9266f046d8598fbac60c401aa88e4bb60fcc7affe3312c3", + "e78c786cbacab07d07d071bcccdd4e00c9f7209cf22976a02e79ed6dbf51cef2", + "ece2638539ff9540b6ab4d309ed2f4f4586b72d8bc2e8681783b667cf614b9ea", + "cd01fc5252898eb16cf99663a1b989da0da1153164200443294a66a3cd40829e", + "48562c1d268ffb5190099fe1960fcb8675406d6363777f48aa0fcb49772343f1", + "e51107e42e10b8cfc3fffea5c5c08684bbb910298e558483ac43eef1f9fa3381", + "c9cdcf8c278bb3dacd3c44aa7510d45f6efcd05be4e0111e0edbd62b62dd237d", + "45a5aca5a74d33e10eff9938b0613e7f5f1ab39b494e5e310b21c49d77f13632", + "e51107e42e10b8cfc3fffea5c5c08684bbb910298e558483ac43eef1f9fa3381", + "3c1e32caf7f4b173667257c8fa5744f09fb3bf8c1731eb5052e1cd1f9fdfd2fd", + "0948183d7383ad9f106721788e45a8a7b24b2fc62aeafdae593b8566771851d3", + "00c2f89c509b52db77abd4b815d23d7c2fda4e2ceef0573ed3b52a8e075c9b9e", + "00c2f89c509b52db77abd4b815d23d7c2fda4e2ceef0573ed3b52a8e075c9b9e", + "7effc76947102221496fed2e6bf944cb9709a804b92fea3ebc6ade7f7352f35c", + "00c2f89c509b52db77abd4b815d23d7c2fda4e2ceef0573ed3b52a8e075c9b9e", + "601476c83c001fcde9b6a22bc06a59548eeb4b0727dd0756b30607245adcdcf3", + "7effc76947102221496fed2e6bf944cb9709a804b92fea3ebc6ade7f7352f35c", + "77f89c07862c74d0a9c94741361c6e5d9510b94e244653b9ee71f3a142c04766", + "601476c83c001fcde9b6a22bc06a59548eeb4b0727dd0756b30607245adcdcf3", + "ad91280856c18dc875f5969c32a89202f12d1e089b0715c31d62559d5f06a4bb", + "77f89c07862c74d0a9c94741361c6e5d9510b94e244653b9ee71f3a142c04766", + "6d2bfa0fb78fcf932b16ca3e0caa1b19079983b81d17b01f11049ad6867ab6be", + "ad91280856c18dc875f5969c32a89202f12d1e089b0715c31d62559d5f06a4bb", + "67c321674993031f8d91acdda9a1904276a7a00121a174bba33a2f6ef3390ad6", + "bc3f31f01174534796c6a126f1352b8634c191f392f8196becaef523adbe582a", + "67c321674993031f8d91acdda9a1904276a7a00121a174bba33a2f6ef3390ad6", + "67c321674993031f8d91acdda9a1904276a7a00121a174bba33a2f6ef3390ad6" + ] + }, + { + "trial": 13, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1012bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 0.917, + "point": [ + 602, + 369 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "reason": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "\u00d7", + "Calendar Finder Flow", + "emograpl", + "1 - 2 of 2", + "Calendar", + "36", + "Mess", + "Preferred", + "Birth", + "External", + "Previous", + "External ID:", + "Hits", + "Name", + "DOB", + "Sex", + "ID", + "Names", + "License/ID:", + "2", + "Avclab,", + "2001-01-", + "Female", + "User Defined:", + "Oaun1000heal", + "01", + "Billing Note:", + "Avclab,Recorded", + "Previous Nam", + "Confirm Create New Patient", + "Cancel", + "Contact", + "Choices", + "Employer", + "Stats", + "Misc", + "Guardian", + "Insurance", + "Search", + "Create New Patient", + "8", + "O Type here to search", + "\u8425", + "2:45 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape", + "click 'Unassigned'", + "type 'F'", + "press enter", + "click 'Billing Note:'", + "press end", + "click 'Create New Patient'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 93.14, + "first_frame_sha256": "6e520ed6ac80f7419d0145599b1218d72d64e44980d9b41055dd82d418ff7c65", + "frame_sha256": [ + "7045c2d5236e20621b962b0870753d109f042991e96b963e0f3ca1a266ea7065", + "6e520ed6ac80f7419d0145599b1218d72d64e44980d9b41055dd82d418ff7c65", + "1c24a411e7e78eddfdab229550ad26819a0a941398be55557cbfc1af122093b1", + "7045c2d5236e20621b962b0870753d109f042991e96b963e0f3ca1a266ea7065", + "98f784959bc150529d1e53fe3c7850c982c9afd2dfd121ee34921759fa4b5e12", + "1c24a411e7e78eddfdab229550ad26819a0a941398be55557cbfc1af122093b1", + "1f7c2dbe97d4ba35b3bfbabc3c38b36a75c1a4c2fd104e465b28f879ddf9a08e", + "98f784959bc150529d1e53fe3c7850c982c9afd2dfd121ee34921759fa4b5e12", + "8ac6be12c6ca4213b735f3ef73ab5c54d8109c2a9bb2daef52f4ff28dae63d71", + "1da04ce7421b3bdd2dbd3cd942ae94801e65a24edc386925051d1a6b3ee30afd", + "af1e364fecc720038a1aa5a7f26f5c93de186515ab819241b8f9a6b8f69f752b", + "7b47cde39b96c8c8a4120575f80bddf14ff7dca2ee84c5c3035265bf998414a3", + "c51e5d23c45cff660a8db26ade096aef6e299ee8cacd6c3a75a9b412a58c822e", + "af1e364fecc720038a1aa5a7f26f5c93de186515ab819241b8f9a6b8f69f752b", + "46f517f214fe6eeacdaffe220f3545acb37a5d448e3203e4af54d4990e2fec46", + "97061ee53905c369c9647fa0239d00a19277f3eb7a4714ec2334d93d41c0a449", + "e5b4cb650e2a56f13b2ca29d14e4db785b9d12ee6326d622d7833bdf9f60d77e", + "2242d415f9cdfb8bdf05dc9424529790b2e54ed21ed94936f2e4e5550c7bb073", + "32e4e6fd0c34814fbe8154956f17a5191b9cef59888ae002b67936a6e1255043", + "911f848587f787e23de4a321a87b7b1789fe64733136bde7229382dfce1ba1ae", + "9996ed41a450dd4c3d503a52c4be233c66db1bcbb0f03159759812e44e9ea2d9", + "9ca62201e89a4e276c6244188f2d94ccb364c4b2e43e4c4d6d568883220bb4ed", + "c68488c3d790fe7c0741612fde48316fd148b153c107522672ce4ae9cf6e7a22", + "9996ed41a450dd4c3d503a52c4be233c66db1bcbb0f03159759812e44e9ea2d9", + "adf8d0adc49bc82db4258b8b903ead74d2aa25119e4ede3874ccef1c4e0df46c", + "a6b749359b3a83d5e46c9ff6e29b021436a4c72cd2f2834888919d8dfa93e02f", + "a5f37cbfd702ed8404006a9938c877c3aa078132d702924d3f7f759f3a207862", + "adf8d0adc49bc82db4258b8b903ead74d2aa25119e4ede3874ccef1c4e0df46c", + "9bd6d248831e932fa861e90436e64a8697e4139038d107a9fe1e63704629c9fa", + "a5f37cbfd702ed8404006a9938c877c3aa078132d702924d3f7f759f3a207862", + "0141a7550df4b0d6139252b195caed563b7c8e4926dfaa52e7e9ac7da410a9fe", + "9bd6d248831e932fa861e90436e64a8697e4139038d107a9fe1e63704629c9fa", + "82c6394771c50354383e51c8ab76a9fd943dbe1a87677de5ae14842fc0c64f8d", + "0141a7550df4b0d6139252b195caed563b7c8e4926dfaa52e7e9ac7da410a9fe", + "fdd28458be169fd22b274ec12f4a7e99d2bf1b579a195585d1947e4aa79d6b8e", + "82c6394771c50354383e51c8ab76a9fd943dbe1a87677de5ae14842fc0c64f8d", + "5b29a0b3a28ac8f23001e308a91b5815a3b219ed94489b2b36608c73fc010a83", + "fdd28458be169fd22b274ec12f4a7e99d2bf1b579a195585d1947e4aa79d6b8e", + "f775b810d02fe6ccf0f74cfd7a3982dd3aac382c3b6e638d41646ded422bb318", + "aed3141ffa173d6237b5e3959f347a612b8d98eb6a1dd5907012966617c2e27e", + "f775b810d02fe6ccf0f74cfd7a3982dd3aac382c3b6e638d41646ded422bb318", + "f775b810d02fe6ccf0f74cfd7a3982dd3aac382c3b6e638d41646ded422bb318" + ] + }, + { + "trial": 14, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1013bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 0.917, + "point": [ + 602, + 369 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "reason": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "\u00d7", + "Calendar Finder Flow", + "emograpl", + "1 - 2 of 2", + "Calendar", + "36", + "Mess", + "Preferred", + "Birth", + "External", + "Previous", + "External ID:", + "Hits", + "Name", + "DOB", + "Sex", + "ID", + "Names", + "License/ID:", + "2", + "Avclab,", + "2001-01-", + "Female", + "User Defined:", + "Oaun1000heal", + "01", + "Billing Note:", + "Avclab,Recorded", + "Previous Nam", + "Confirm Create New Patient", + "Cancel", + "Contact", + "Choices", + "Employer", + "Stats", + "Misc", + "Guardian", + "Insurance", + "Search", + "Create New Patient", + "8", + "O Type here to search", + "\u8425", + "2:47 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape", + "click 'Unassigned'", + "type 'F'", + "press enter", + "click 'Billing Note:'", + "press end", + "click 'Create New Patient'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 87.62, + "first_frame_sha256": "6e3d2edde8f61659de8b7c81a831360576616dc9f8d26917088cf1d70a6516db", + "frame_sha256": [ + "7f524a976b62c5a41598c1eacb326001f5eea8e6792213f3b8cc54a7f5b426ed", + "6e3d2edde8f61659de8b7c81a831360576616dc9f8d26917088cf1d70a6516db", + "550b94b1a224976b74516625ab80f04c678836513a61c6fc143da1e2996b2c81", + "7f524a976b62c5a41598c1eacb326001f5eea8e6792213f3b8cc54a7f5b426ed", + "7034dceb83eb4e67272949de247af778d46d6f4e7ea76f04d40b722b610ec474", + "14341e984551e4df8d2a7bdf9cf79e866672f6c1e52d46b0fc329972be59da29", + "5a20079dd718c789c025f47841c2a4e58e8e1a3c91991cbaa0e5401b5d36c34b", + "7034dceb83eb4e67272949de247af778d46d6f4e7ea76f04d40b722b610ec474", + "e30b1f957aa8f779ee15aae4385220e24cae3b8e4e357e8a4c136fd7fdb03aa8", + "5a20079dd718c789c025f47841c2a4e58e8e1a3c91991cbaa0e5401b5d36c34b", + "111390fdb1a5b564e3feaa4ab0d68864da91f9132d9fbcff4887dce62876d3d3", + "db6e5e4f424761412d59e0058025e98fe5a4b0238b20612bc9ed4b9e7e081033", + "cf077d78772961ace57f749b8a92f6c11df803fcd4d4aaea6f5b2527c226cab4", + "111390fdb1a5b564e3feaa4ab0d68864da91f9132d9fbcff4887dce62876d3d3", + "5cc215a28ad58fe8e79c4518f5d5a7ca09ad773fa08381c89e4c8022a059357f", + "cf077d78772961ace57f749b8a92f6c11df803fcd4d4aaea6f5b2527c226cab4", + "22b6224e0ba511bfe326481fced932fed462e85c655763d1fb143762fd9fd30e", + "c72246f09acb2e24d893190dec91d6c1e6cc4f93e1c244fea91839ef8d806862", + "fc974905e694458ded71041fb87d3f44a5e63d5d8af74939263582deccb9eae6", + "fa2138a8dba9de344e60633022adaabc566351e86c067fe72946a9aff98d8f35", + "6355dd8940bb783c3cdb209609092142ec7db7f5a2f2a2148c9fc08a7f643261", + "fc974905e694458ded71041fb87d3f44a5e63d5d8af74939263582deccb9eae6", + "b120b5c808671153a5bc6c7e900c7e33b40affd51255ddfffab43926758c2a4d", + "6355dd8940bb783c3cdb209609092142ec7db7f5a2f2a2148c9fc08a7f643261", + "dfb01ee935247111375208584562fbb4df49402a56d95fc2aee1f87ade436614", + "f5d0f39650d0f23b1ae8021fd31cb2ea58d2228afbb90ae317ccec2bc714d232", + "8cbb65ad3baf4bdadac9e3183fd8296ec05e52d44d9078d6328a131b2ffd0a7a", + "dfb01ee935247111375208584562fbb4df49402a56d95fc2aee1f87ade436614", + "b357620dcc4085a942cd8afe21b4b6151c1c699829c6afbcfbf87a1c360d115a", + "aae623a6fd76dab2cf96c8af4d0a77e92b13987980ed7a026e010b16d336cfc5", + "8b57763f79309ba41755d96bcf921e45d818f6fe660055acf96c25926d1c5d38", + "b357620dcc4085a942cd8afe21b4b6151c1c699829c6afbcfbf87a1c360d115a", + "7cb2e14a4e2fb5be27756cd1d6f0697b8ddbdaf4d9906144fd1a50a050940d70", + "8b57763f79309ba41755d96bcf921e45d818f6fe660055acf96c25926d1c5d38", + "01567643085b475aa3a3fc981a69666fe8ac096c9a1bc56fb6714094863721b3", + "7cb2e14a4e2fb5be27756cd1d6f0697b8ddbdaf4d9906144fd1a50a050940d70", + "54e473f2313620ab664097b1f9f1ec678d890ffa6c4cb0ab76391ab852475fd4", + "01567643085b475aa3a3fc981a69666fe8ac096c9a1bc56fb6714094863721b3", + "6661928c16e44459361ed6ce417365d45c7b1a80f91cc1cd8af67a437c7a6422", + "093828e1925e6faf273cb08f8b5f43add5b61a61ea2ee01b41af873e5342c609", + "6661928c16e44459361ed6ce417365d45c7b1a80f91cc1cd8af67a437c7a6422", + "6661928c16e44459361ed6ce417365d45c7b1a80f91cc1cd8af67a437c7a6422" + ] + }, + { + "trial": 15, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun1014bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 11 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 0.917, + "point": [ + 602, + 369 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "reason": "Identity check failed for step 'step_020' (click 'Confirm Create New Patient'): a target was found positionally (rung 'ocr', confidence 0.92) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Cancel Confirm Create New Patient' (coverage 0.71) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "\u00d7", + "Calendar Finder Flow", + "emograpl", + "1 - 2 of 2", + "Calendar", + "36", + "Mess", + "Preferred", + "Birth", + "External", + "Previous", + "External ID:", + "Hits", + "Name", + "DOB", + "Sex", + "ID", + "Names", + "License/ID:", + "2", + "Avclab,", + "2001-01-", + "Female", + "User Defined:", + "Oaun1000heal", + "01", + "Billing Note:", + "Avclab,Recorded", + "Previous Nam", + "Confirm Create New Patient", + "Cancel", + "Contact", + "Choices", + "Employer", + "Stats", + "Misc", + "Guardian", + "Insurance", + "Search", + "Create New Patient", + "8", + "O Type here to search", + "\u8425", + "2:49 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape", + "click 'Unassigned'", + "type 'F'", + "press enter", + "click 'Billing Note:'", + "press end", + "click 'Create New Patient'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 89.66, + "first_frame_sha256": "319ffd624250aac8d1c5a10203d548153339ef105270848734c01ade7a814569", + "frame_sha256": [ + "d498e57f6af3832bf94fd229469f09443e4fc3e0a07eae464224f1b51b76a53e", + "319ffd624250aac8d1c5a10203d548153339ef105270848734c01ade7a814569", + "6bf59c2015964fd255db498e6dc23c1a1024f64a18a7f63ac01e30f4dedb6de9", + "d498e57f6af3832bf94fd229469f09443e4fc3e0a07eae464224f1b51b76a53e", + "e7749cb25c829b534dd07c9b0b081082aec3f2669bf957169414c3fe78eee8b4", + "6bf59c2015964fd255db498e6dc23c1a1024f64a18a7f63ac01e30f4dedb6de9", + "11dcd7cd0905f5a6080b477172023cfc8bc56ceb9c0bb99a13b4ec82afdc9786", + "e7749cb25c829b534dd07c9b0b081082aec3f2669bf957169414c3fe78eee8b4", + "40e17633ecc1f1cf4357579980c27118ac668bb8a8d5e6e9b999cb67154cc35d", + "efa3189164ee20b75b202dd447618ba16d4caff44a692ff007dfc3db633326ff", + "29cf7e8516ede38f6774bed50df00cba23dec9dfeb983cc79ada9359988598fe", + "5bcec94af93f3c4470086c9f2938e6d87f20040ebf22e449bfaea59a839c30bb", + "2fb7421d62ccc061e2664a346a02c50a15c2673aeffff8339da5315feb029903", + "39f0b7e3b5430613cd8eb3523cc3017e2744246a3000035b5e372b0f8579e6a6", + "815aaba227ea35406b85a73ea537984da5a914e47c1c780bab313393d4fe9276", + "6619854d45ef2fc4ff33da2110caa3b5be493407cf9711fe2bfabdb701feaee2", + "0db61c2eb6ae3b11b4dabea9acf7274d59c801b8a2afb2ac7396d25a8db806e5", + "8e902ee2d0c199a024651941d8efde33fc15942069145a12967a9ecdd78bef0f", + "bfcb4d6e081d8a88f5a1a7b6e24534e868d9d195d5446011ab35c243b1db06f9", + "dd65f620f656f0e40235c6f9a30a7a5ee4c84ecaa59f61c5c9cb7342595384b7", + "a167db9de5847491ecfa76765a329ade6ab9d47afdc637822b28fc4a61e158b3", + "bfcb4d6e081d8a88f5a1a7b6e24534e868d9d195d5446011ab35c243b1db06f9", + "5925d4e5893918e90b0f8fb7d5989318aa4e9dbe91ad2b96978fd4b7d9144c42", + "a167db9de5847491ecfa76765a329ade6ab9d47afdc637822b28fc4a61e158b3", + "644ba9743c027ed875f84f4b56ec6274846b7c3875a7f8244b448da5a90ff586", + "df7c28c059a774cfd733ff20e18727bbb3938059f36724cfd49bbbf8770210ef", + "a52cabf84ad53e408967304a959f9239a76b96d9a879118f4235da785b691230", + "644ba9743c027ed875f84f4b56ec6274846b7c3875a7f8244b448da5a90ff586", + "1bad66701ac9e9e520ecb6ef9a8887ed790450de6feeca5beacbd1a03e1d54bb", + "a52cabf84ad53e408967304a959f9239a76b96d9a879118f4235da785b691230", + "209bc4a75a9a0c5d8ee1ff1e134002b0906ec9cd4283adf3a7d99b7ec5d97ba5", + "1bad66701ac9e9e520ecb6ef9a8887ed790450de6feeca5beacbd1a03e1d54bb", + "7b76f69bbd64aa6ddc58afa993eedf42260214728e0928b765de3e1a17f6a768", + "23914a8c4967fea6040886322e415c44401544cad8b8bda351e68952161a8b1d", + "120429a139a4b54394fd70b8a02f9308ffa07c9813ee95f168dc923f5d10ea4d", + "7b76f69bbd64aa6ddc58afa993eedf42260214728e0928b765de3e1a17f6a768", + "2ad6cc089789bd6e21fea01765d3fd3c137d6c4d0425ce3060a0db735e79c218", + "120429a139a4b54394fd70b8a02f9308ffa07c9813ee95f168dc923f5d10ea4d", + "15a5fb7a60ebbfed98180d884f5a32fae44dfac25a332ad4473d63fce5c29d4f", + "c67173fa485b90d9cfda96a8201a0225e7609382f4dd4e622005e22c8b2290ad", + "15a5fb7a60ebbfed98180d884f5a32fae44dfac25a332ad4473d63fce5c29d4f", + "15a5fb7a60ebbfed98180d884f5a32fae44dfac25a332ad4473d63fce5c29d4f" + ] + }, + { + "trial": 16, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun1015desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": {}, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.855, + "point": [ + 782, + 450 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_000", + "intent": "click 'ername'", + "reason": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMRLogin", + "X", + "openEMR", + "The most popular open-source Electronic Health", + "Record and Medical Practice Management", + "solution.", + "Username", + "Password", + "Language", + "Default - English (Standard)", + "Login", + "Acknowledgments, Licensing and Certification", + "8", + "O Type here to search", + "\u8425", + "2:49 AM", + "10/29/2026" + ], + "completed_intents": [] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 12.3, + "first_frame_sha256": "7ced4c6627d1dc4837aa582dacb938e53e7218f9843e98d2b95fb761c23a8220", + "frame_sha256": [ + "c1ca357f8544787b7a276ffdf03409698708def1992df94d249a725520a4ae4e", + "7ced4c6627d1dc4837aa582dacb938e53e7218f9843e98d2b95fb761c23a8220" + ], + "first_frame": { + "file": "emr_desktop1440_first.png", + "sha256": "7ced4c6627d1dc4837aa582dacb938e53e7218f9843e98d2b95fb761c23a8220", + "size": [ + 1440, + 900 + ] + } + }, + { + "trial": 17, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun1016desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": {}, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.855, + "point": [ + 782, + 450 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_000", + "intent": "click 'ername'", + "reason": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMRLogin", + "X", + "openEMR", + "The most popular open-source Electronic Health", + "Record and Medical Practice Management", + "solution.", + "Username", + "Password", + "Language", + "Default - English (Standard)", + "Login", + "Acknowledgments, Licensing and Certification", + "8", + "O Type here to search", + "\u8425", + "2:50 AM", + "10/29/2026" + ], + "completed_intents": [] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 11.27, + "first_frame_sha256": "e566d64bc563a4cdab931717b9da295f8830637b9b61f84cb2bf46b89567cbde", + "frame_sha256": [ + "9cc6bd9f3ce1c6bea0546280fabb8e4a05e267543d43a71730b7395e0189ae9a", + "e566d64bc563a4cdab931717b9da295f8830637b9b61f84cb2bf46b89567cbde" + ] + }, + { + "trial": 18, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun1017desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": {}, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.855, + "point": [ + 782, + 450 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_000", + "intent": "click 'ername'", + "reason": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMRLogin", + "X", + "openEMR", + "The most popular open-source Electronic Health", + "Record and Medical Practice Management", + "solution.", + "Username", + "Password", + "Language", + "Default - English (Standard)", + "Login", + "Acknowledgments, Licensing and Certification", + "8", + "O Type here to search", + "\u8425", + "2:50 AM", + "10/29/2026" + ], + "completed_intents": [] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 11.3, + "first_frame_sha256": "6f270f30b4ee8ba21cbef1b46954955e8aa327f605e0c353243a478d96b10da7", + "frame_sha256": [ + "79c3e5f9ff85b6e30cfe2a318f6740abc8378f213f4d51ec863ae40391b08e56", + "6f270f30b4ee8ba21cbef1b46954955e8aa327f605e0c353243a478d96b10da7" + ] + }, + { + "trial": 19, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun1018desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": {}, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.855, + "point": [ + 782, + 450 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_000", + "intent": "click 'ername'", + "reason": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMRLogin", + "X", + "openEMR", + "The most popular open-source Electronic Health", + "Record and Medical Practice Management", + "solution.", + "Username", + "Password", + "Language", + "Default - English (Standard)", + "Login", + "Acknowledgments, Licensing and Certification", + "8", + "O Type here to search", + "\u8425", + "2:51 AM", + "10/29/2026" + ], + "completed_intents": [] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 11.38, + "first_frame_sha256": "21970615de3d2ac7c86f02b2f7b8bb57be46a1560d84ad37fdaf9154918880bd", + "frame_sha256": [ + "05278862519948a70b6cf3c28e10e47df43f89609b46f1cd112aacfe993a0362", + "21970615de3d2ac7c86f02b2f7b8bb57be46a1560d84ad37fdaf9154918880bd" + ] + }, + { + "trial": 20, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun1019desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": {}, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.855, + "point": [ + 782, + 450 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_000", + "intent": "click 'ername'", + "reason": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMRLogin", + "X", + "openEMR", + "The most popular open-source Electronic Health", + "Record and Medical Practice Management", + "solution.", + "Username", + "Password", + "Language", + "Default - English (Standard)", + "Login", + "Acknowledgments, Licensing and Certification", + "8", + "O Type here to search", + "\u8425", + "2:51 AM", + "10/29/2026" + ], + "completed_intents": [] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 11.22, + "first_frame_sha256": "a6e423b94bcd465363bda50267221dd421b71a2e17d0a48cf3dbd697e52d2576", + "frame_sha256": [ + "1f58a8d130c3043ea6d90f4725b0edf1d9989d9a8963647775e6b7f3fe94f478", + "a6e423b94bcd465363bda50267221dd421b71a2e17d0a48cf3dbd697e52d2576" + ] + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-2/reports.tar.gz b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-2/reports.tar.gz new file mode 100644 index 0000000000..05847cc791 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-2/reports.tar.gz differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-2/trials.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-2/trials.json new file mode 100644 index 0000000000..ad995f3432 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-2/trials.json @@ -0,0 +1,5387 @@ +[ + { + "trial": 1, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2000heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2000heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:58 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 87.67, + "first_frame_sha256": "a01da85833ef6e781feb2c5bec8e276b6e882acf7bbae4a7be524fd7a3a2d1cd", + "frame_sha256": [ + "d4a0a0ccc0aa37741ef7f645250bd3e5d0d31e0d94d9b34d2cece27d0801079e", + "a01da85833ef6e781feb2c5bec8e276b6e882acf7bbae4a7be524fd7a3a2d1cd", + "3eacc37e49ed1710845c738a243b6e61cb76dcdf41262754d0712af3fa8ed5de", + "d4a0a0ccc0aa37741ef7f645250bd3e5d0d31e0d94d9b34d2cece27d0801079e", + "dcdff1830adfdfaa9392fd2d12c659a06a1b52f97e7701c0b5772ee0230d7a3a", + "367ef1fe973055bea358a1114d66019755b8816c4dde2b5dcc5b2a09e9010852", + "d2066becd816bb13f6b9493b1a2954743848d9b9ef2ead009d3e12af9f7fe733", + "dcdff1830adfdfaa9392fd2d12c659a06a1b52f97e7701c0b5772ee0230d7a3a", + "304aa770e32f30d2f164df331e73b700ff035317a19b4fa40e96907c53583f91", + "fd8c02df46c1ac0e415b9b171c9fca2f1a15ac813177d31bf191299f6d0e668c", + "2ae602ada279bf32e6dafba26e349d602fd88aaf6d1c9b40d8524d26ec12b62c", + "0bf45f4780984496c578f005449d6552b449927d41aa9646af9a927b3de0544e", + "5086261a7d570c4c18f80df07db6f724921b71ab8a7d999f279790a294d3a34e", + "2ae602ada279bf32e6dafba26e349d602fd88aaf6d1c9b40d8524d26ec12b62c", + "5299da99e72c72fa5b769446ca6032e35c16ffed179ec5b660d6fd1e4e4038ad", + "9bf917bd628d1c3a126eec6fa1756ae2b81c684a6670bd52eca4edb0a3171419", + "6d17d58b560f1ff434cc8642d641ad816d32495c70e56ac9848c2e3e2ff47584", + "3e03a4b43ca5f636250a8c405eb3b6ed5b97e32d8d28a7c00c1c3d7248240d26", + "c111b0b529a98516aa1446bad2ba82d91f7fd1e73330aefc6f1d7bb0cb98e207", + "e78435ebbd8b9ed8051785776084ed4c1ffd40459d7aefc44a2c3e34b61e8107", + "733bd3f00e8f7479c78df160a8705e6c0391f8021e35ccca752d05cfa6bade0f", + "88ff82e497839622a31249b08fec0b0e420dc32f8984dd72b1ff3141b7902c01", + "62249df0444073a0fc577383b9f67cc896120cfbc6f99f33bc4494dc197b4084", + "733bd3f00e8f7479c78df160a8705e6c0391f8021e35ccca752d05cfa6bade0f", + "1abb39b997da538b5d1893e0710d89831d5ee85cb22001eabd6da2d903c7f079", + "e54af2c48e33b287fa539ed709b7d6a51c0a7f63b38e5fdabd1adf8ec7e136d7", + "3808804491deb056b3fb05fcbfcb4a5da31409453f53a8fad2a2326ef7d8c0f5", + "1abb39b997da538b5d1893e0710d89831d5ee85cb22001eabd6da2d903c7f079", + "3808804491deb056b3fb05fcbfcb4a5da31409453f53a8fad2a2326ef7d8c0f5", + "3808804491deb056b3fb05fcbfcb4a5da31409453f53a8fad2a2326ef7d8c0f5" + ], + "first_frame": { + "file": "emr_healthy_first.png", + "sha256": "a01da85833ef6e781feb2c5bec8e276b6e882acf7bbae4a7be524fd7a3a2d1cd", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 2, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2001heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2001heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "2:59 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 80.58, + "first_frame_sha256": "ddc09b18bbe4399a9c4ec374850030147e06d33eb5b5da4e47e79040f3a575c8", + "frame_sha256": [ + "40d8d42ddf23b947dcaf18227d1fb37eefd2ae152c3c50fd057202cc678d16d1", + "ddc09b18bbe4399a9c4ec374850030147e06d33eb5b5da4e47e79040f3a575c8", + "97f8abb2e0881a23581354b5f4800cc59b3d06dcb0a5961d0cac15e72a60a2d4", + "40d8d42ddf23b947dcaf18227d1fb37eefd2ae152c3c50fd057202cc678d16d1", + "d9f6e03f9ad8dfbeb102574df6f32b4a44732afa3b8aff05961039843ad28997", + "6a57a0b9a3c66703759baea5cecf419bfc47ca79ac016b4de4389dcecb029a39", + "533d602b51b80a6a67ad7d87729346045531bed7f1507e81e3a58b92b756ce3c", + "d9f6e03f9ad8dfbeb102574df6f32b4a44732afa3b8aff05961039843ad28997", + "d0c414a8f133521c9fd62aa5b8b266250d9c48d8ebce22ae85ffd3e1a22765db", + "533d602b51b80a6a67ad7d87729346045531bed7f1507e81e3a58b92b756ce3c", + "7662674168c25db125277d4f5982b503c492529db1c9e681a60a61c73414da6c", + "0ac43926dfe9d51cb09a42a2b6a5b9a775898163ae30185aa250566dd00eefb8", + "a9a21fe69c1a39719afe114c105b1e759945743dbb5a4c416d49d4c40efce668", + "7662674168c25db125277d4f5982b503c492529db1c9e681a60a61c73414da6c", + "a842981446b0f3bcf235dfcdbd73eb07308a8e54970535a407cef21d3ea99031", + "f9a50de88f0615baf4e5d5533a9633e021c74143cfb708846c5c5f22d8f96bd1", + "85d9de1c153e64c7024bba6d50617ebf979676808889698acd0e3de14bdf65aa", + "c7c4ed06fba71f2d812fee7a1f1b52637093ead82df4dcab7adbc7bcf867bf19", + "7b75d9ace2ddbd274f8fbe2f8b64acd8e114297b422b7373398928fe1a9b97db", + "d8f66b81410e380abd80450c0d304cc1931a32a08c3a34f1b515f6d7586e5001", + "beddb406c3e183871be7c27e3d630d321f9a2f72a513afd93e646392424575a2", + "89d2033fda780082843c449464afd3204e6bc71386056ce48f0061308854b05d", + "3377d901f3b36a171dcc0a48c7fa9f300356007ae175e342b32a4b47a0f78ad4", + "beddb406c3e183871be7c27e3d630d321f9a2f72a513afd93e646392424575a2", + "abcc8881d5f719c99efb2be4c9e09a17b9895e39397b6cf07e289b23a59d7715", + "2465b8fcbec44d6b90c2f4fb92192ed1f6a60f1d7c421d0846b53cc683a4155b", + "1dc17d112a9897f72079f6b68b49527cce5d1f9648a574dc2b8f96e2540f08b7", + "abcc8881d5f719c99efb2be4c9e09a17b9895e39397b6cf07e289b23a59d7715", + "1dc17d112a9897f72079f6b68b49527cce5d1f9648a574dc2b8f96e2540f08b7", + "1dc17d112a9897f72079f6b68b49527cce5d1f9648a574dc2b8f96e2540f08b7" + ] + }, + { + "trial": 3, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2002heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun2002heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ] + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 129.19, + "first_frame_sha256": "6a70a46839a866b8d07658d9252f3fe50537cff35f9c8480a422de2ed7bb9bb0", + "frame_sha256": [ + "60825f11572a60da26cbb3fcba7eff538ba957fcf52fea85c00f0d37e6222dab", + "6a70a46839a866b8d07658d9252f3fe50537cff35f9c8480a422de2ed7bb9bb0", + "cc2c2453e2fe1077e42f67f801b0329be720e56c78916aaa65902054452a6823", + "60825f11572a60da26cbb3fcba7eff538ba957fcf52fea85c00f0d37e6222dab", + "50ee725ac4e50c991627f3cd62fedc480a2c58e8db6a9f27321c7786891afced", + "36c366394d9f8a9d2f4b605ba844ce20d686dc6cb2425e11c1be554abe694107", + "c18d3b10e308de370ae3154ce68287024d7392fe001f6224f8ab69814b3abd7f", + "50ee725ac4e50c991627f3cd62fedc480a2c58e8db6a9f27321c7786891afced", + "09de968e553fef5fe9969b5d584184f26736af3278652ccdee2f4b01686f0b77", + "c18d3b10e308de370ae3154ce68287024d7392fe001f6224f8ab69814b3abd7f", + "69454927af4e34b9313c5399ba259243fbf5acc0e36e16bed773a1c1db5d7843", + "09de968e553fef5fe9969b5d584184f26736af3278652ccdee2f4b01686f0b77", + "66319ea50f44a33841bca8e18ea6901256e1ba350a5c91b2b21ca6096bbb43a9", + "69454927af4e34b9313c5399ba259243fbf5acc0e36e16bed773a1c1db5d7843", + "c536f73e1e39efbdaab99b53d3df1fdf36a4cadfc7202492d362810faac51cee", + "f91b97090d75f421e07d6dad9c92d4abd4110cbfea5784ac5e9f58d8b39c939e", + "992466874df21f0e609766a01c228822368214f2cc52265fd4d3acfc9dde54fe", + "67ac7f74e79788f98af702abeb7ef6c4b72b2e036568cf72ee1e9b2a45a8f5e8", + "57ab8d056145a3928c5e8d3d6b5a8119f1ad611c7dea45b574074431cc846cc8", + "a13334ed3c1ca7c77403815d55f66f016e65d2d112fab29cfe79b214fd77fcb1", + "4010e44f5e465d7049e86ec07c5f6d26b898f7711c2657b334f26697cd55fd31", + "e3ce833237d450af289bce65ec1fcf40fc2688a0f5060d894d2fb7ba858983f1", + "a416e76a449511d6a563a109e26975575cda2d88704768412a4def9744983437", + "4010e44f5e465d7049e86ec07c5f6d26b898f7711c2657b334f26697cd55fd31", + "48433c96b32703795d811e8f1a6a7e4f410e7e7cbf9a2c977c69684aac855fb4", + "91b52c861501064066b67ea71205a425c86ebe1a75564fc3d310af2f1c6a30e2", + "4b80f4dd5e1cae1909c4fac54ab2a2faefe25793b12ced279511a28e487a009f", + "fb68320c85000bd797288f9188414cdf90f379abfbf09e2f9b72e5e1cfd9bb84", + "af7b3c28d9bed8900ccaac34e041bd660e8499feb66b85234a7eb358faaeec64", + "4b80f4dd5e1cae1909c4fac54ab2a2faefe25793b12ced279511a28e487a009f", + "46f2475e388b37175a21c611e62b87ca63360c59da43b41876def10cb7d4bd66", + "af7b3c28d9bed8900ccaac34e041bd660e8499feb66b85234a7eb358faaeec64", + "bc14145507644ea0fd08b04442b6c4926de2a5959d5ddbfed59c4b527317755c", + "46f2475e388b37175a21c611e62b87ca63360c59da43b41876def10cb7d4bd66", + "c8209cdac9321d21d3cd77675afd7a650768643b002ba6b051f9127b97b068a1", + "bc14145507644ea0fd08b04442b6c4926de2a5959d5ddbfed59c4b527317755c", + "fd7b86d5a0b9d6455aad03cc7eec89a4000fb2578951696912f83f36cf8b8faa", + "c8209cdac9321d21d3cd77675afd7a650768643b002ba6b051f9127b97b068a1", + "a3ef07487e3054701637cc4c75e203ce0f5bb6fd595112c8e6c3fd04a377ce75", + "f55a2f7d44620f2d6826f208c882a5d6cb0b16056b9a2dfdb5bedb96b227f7af", + "8bc700f877572512c1ee8d869bfcbcfb92b3544a820e88727ec0d78bab27d94b", + "a2a917706cf8127cedb40c3e0682f7bbfc49f2d22a9da4e896a08218dbfe2d0a" + ] + }, + { + "trial": 4, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2003heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2003heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "3:04 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 78.5, + "first_frame_sha256": "59ffec5ac5b505e3d3d4596de16a0a9e9959aaca76332982a86316d5ffc780bf", + "frame_sha256": [ + "d744443f1de7e08f344b0dcfe73d619b8e79ae088e4c9a91fe4f1b5daaa4ecb0", + "59ffec5ac5b505e3d3d4596de16a0a9e9959aaca76332982a86316d5ffc780bf", + "072a84619f0acb5d88397b0c3a4f28b2faa937e05aef8b9122deba08bb974724", + "d744443f1de7e08f344b0dcfe73d619b8e79ae088e4c9a91fe4f1b5daaa4ecb0", + "766b2c57631e2e734a67243e913f35c2d3db574f369a0253b2db332012166b4d", + "072a84619f0acb5d88397b0c3a4f28b2faa937e05aef8b9122deba08bb974724", + "ab13027b072dc9966ab326706a3f2fe6b7ea6ee0b17ebe0ab69acdd4ab2168d7", + "766b2c57631e2e734a67243e913f35c2d3db574f369a0253b2db332012166b4d", + "a2bc810535c545aa68efcaa5ef7d868c670b2c80f29bf5d3b050b35c0c026b40", + "ab13027b072dc9966ab326706a3f2fe6b7ea6ee0b17ebe0ab69acdd4ab2168d7", + "62d8c1ada5492aa284da1d8d2cda265a32e02f6d20dcc680bb1c6fd0ff3cadb8", + "d1b6ca23bb2e19d6acee77b5043bdfa9d3b4e50a2a4ccc7e1a9fbe4f08d73185", + "c8025f2fcf2ec3cd26fb9b701c6a1adf633102fdee40b567ecafd480df8e56cd", + "c4621a3fce920b7e2309a139300dc9e57ef25850090242fcf9e4e38dd06f8604", + "af336b3148c67a582b244e558658f388bc186235aa984c07f68e180ab170cdbc", + "476521109dea44dd1e448b14da76287b0e1872d7f57d72c6bb96c8903cbf9ded", + "f464e4dcc9a31d7c0d77dd785de3b4b395b6ea89a0d8d9cc95f68db6df28a831", + "2c7f6ada9b3e41ba4754d943cd241b362e2a0610a47516ff116fb9af4957e861", + "ba2b08450a4f1fff81357dbe3413207d31cc9162399375a5a4fa818c2f70b2e8", + "44620f11f1302c67535c1d7946e7b30f5549fb2c4d3b6ac8db3ed2a6f4fa7851", + "3567d82aa7cd6f9e376dfe8b6fb157797e547b9caae09855984b4adec86976da", + "ba2b08450a4f1fff81357dbe3413207d31cc9162399375a5a4fa818c2f70b2e8", + "41e6e48133d131d3b2ac9d5c0e54c5867f7ed6f6955f5e661a0e7ce7c39a1e9c", + "3567d82aa7cd6f9e376dfe8b6fb157797e547b9caae09855984b4adec86976da", + "87b29b14bc2e0047dc679346ade9a584336b69a23cde14f0174def66fbedd8a1", + "150795f0b35a9ab221aeba2f7cd5b7e3ddd80abbc5834393961e57064e74ddb3", + "691edf848035580f5e31d44904d2b115aa7f1ee197500ccd616a062617633b42", + "87b29b14bc2e0047dc679346ade9a584336b69a23cde14f0174def66fbedd8a1", + "691edf848035580f5e31d44904d2b115aa7f1ee197500ccd616a062617633b42", + "691edf848035580f5e31d44904d2b115aa7f1ee197500ccd616a062617633b42" + ] + }, + { + "trial": 5, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2004heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun2004heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ] + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 92.75, + "first_frame_sha256": "95be5d7b06ab0bdfddfaf1f094a5094ec01a15f97245b4c97547ca5b4afcdc29", + "frame_sha256": [ + "4fc5945f14e8cf69eb1756093cf2a1091949622852a3046248fbf339fbabf2d6", + "95be5d7b06ab0bdfddfaf1f094a5094ec01a15f97245b4c97547ca5b4afcdc29", + "014fad78252095291ca41b956570fd79984d24c857f0fbc071da571dee6cb0ae", + "4fc5945f14e8cf69eb1756093cf2a1091949622852a3046248fbf339fbabf2d6", + "bc8e43ac0f7d48e0cf7fd8691f9cb5c8f8c08f0891c0130545d828933236545b", + "014fad78252095291ca41b956570fd79984d24c857f0fbc071da571dee6cb0ae", + "0626a9921ce40246a3add3bf81d4b21734639f2f5b560ef54470ccebc1f0ae7c", + "bc8e43ac0f7d48e0cf7fd8691f9cb5c8f8c08f0891c0130545d828933236545b", + "dd22a0f347c39116f685a2c5d526e516b12c1206b51b49a2d500b6bf37edd1da", + "0626a9921ce40246a3add3bf81d4b21734639f2f5b560ef54470ccebc1f0ae7c", + "0bd07c85a8b0ad2554cc7c78f2d1c7c3696ca512744fbe3b9f94c767bbe1d596", + "36c6c8e658c87586b6ee00fb21872a865a81b38295a86630195bf7b4dc3e04af", + "86ff66f6c85bb5e52149407a827862393001e5e4b2f46611ae9d8490af6ee15a", + "0bd07c85a8b0ad2554cc7c78f2d1c7c3696ca512744fbe3b9f94c767bbe1d596", + "3e11d6a8d149bdf60dde33cc927cc167e9949f2cd63b2847a12985dd128e169b", + "d2e7834582e563c095f119237ccf50c460c5eb7f8560de5e79eedb2c16e1e93d", + "4985e0102e2304460109ec1b93bbcc12b288fb7bebe5ab99f3af577e58546906", + "e18bd6d4e2e1e47c4efbb6d49db0bd233b30a9a4d5341afc88030f54c6addf93", + "ef5abe357fdced46b2f221fb264efc1fe823b5869cbd542f1050b19932f8fd52", + "46f140052dbc331112de5bbc7385504fe6067572f71bac7301f0408f3ef0a3d4", + "cfbf3bc507adf60084a54da216ac2bc25ffb11b3ff98dfec0e48cf448e014ae8", + "ef5abe357fdced46b2f221fb264efc1fe823b5869cbd542f1050b19932f8fd52", + "9788da58bd2878362aaabec3940ca1dd32d745ead149a082024a42035cac494d", + "cfbf3bc507adf60084a54da216ac2bc25ffb11b3ff98dfec0e48cf448e014ae8", + "7cb8e4e0bc05c17fe4b8070ca110bd4e03254253f2342b0e58d1d5cebbc1ea0e", + "b6dded96e4ce6eef5b7072dbc33f6a735f7aeb972bbd38b8bc429eefdde91ddc", + "b83622d2c16f9d8b1cb39b7a5f5e697d2389785feeedf5d31e72a19be19ef55d", + "b83622d2c16f9d8b1cb39b7a5f5e697d2389785feeedf5d31e72a19be19ef55d", + "5e8221ece547ee012a5f144249806aaaf1730a2bcbb752882a7eaae427447d58", + "b481d118a2875be9a9108bac14659fd54b5aa6c2e7a5fbbacc389437f17619ed", + "678b6835595cec1898310e533f11590e7aec6db2c295c305caba6d40bf54db5d", + "5e8221ece547ee012a5f144249806aaaf1730a2bcbb752882a7eaae427447d58", + "88a452428cb79b4d71a3f8bcd8860d1c85d93e7b381914296898985910ebeac8", + "678b6835595cec1898310e533f11590e7aec6db2c295c305caba6d40bf54db5d", + "98f25d997edbdb6488e552e09d1a432d31008a0ced3bbde965fb4d281793137c", + "88a452428cb79b4d71a3f8bcd8860d1c85d93e7b381914296898985910ebeac8", + "e518e9b48043ecb08e244f6975aa33148babf4f6db7f5b1c1d759022a367c9e3", + "98f25d997edbdb6488e552e09d1a432d31008a0ced3bbde965fb4d281793137c", + "42359c8365cb8ae52712263b4e6088feadd16d48ae66262326c9511d1cc49b37", + "7dc4d498b6ffc2a066968b274b2ad68f810cfb3782982869aa1121297e958b50", + "619ba3856d1ceca31dff19012b9d804080ccb7e3b022e8d43c58b95e9f18529b", + "14dc25cb9459afe7023083aa42eec4056ed6fab99302bc31cf3a0d3f666f324a" + ] + }, + { + "trial": 6, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2005heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 6 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_009", + "intent": "click 'First Name'", + "reason": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Searchbyanydemograph", + "Calendar", + "Message Center", + "SearchorAddPatient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "First Name", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Middle Ne", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ontart", + "8", + "OType here to search", + "3:07 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 53.5, + "first_frame_sha256": "57756950dbe4345bb0f08536bcc3fbe180e268661620e367b4577653a1b14810", + "frame_sha256": [ + "724d79c739c40aab577f228091849c89a8ea55fb86520c149e4c104242173c87", + "57756950dbe4345bb0f08536bcc3fbe180e268661620e367b4577653a1b14810", + "9edd54047bd1c07630d71d20150ea84c7506bf20e144d372dca43a68ca265e83", + "724d79c739c40aab577f228091849c89a8ea55fb86520c149e4c104242173c87", + "7da54a8cd22ed93ea79f0eac01f711dd6c3f8535617cdee1024ded7d48c93f99", + "e700da09c16eff25b24db6a80a7d2744bbb05cd307be436d3be77bd595e85237", + "f3a8e6e99a4874a01226daab800122b65e1794d4efd84a3787625c6ced37a760", + "7da54a8cd22ed93ea79f0eac01f711dd6c3f8535617cdee1024ded7d48c93f99", + "5122781f2da273b67cec0800d462a8d89e0e683debb806b389ddcd347aab213b", + "a0a237da9f3aabcb4a8242c3fb34c0680da31232239be04774bdd252df883638", + "748614deae3fbf232d82611b3510eaac5059a5090e33acfde910a103b370307e", + "bce6348196cf9d0ce8e13860761e3def7ba658b4227196652c692663c915bebd", + "1f4af941301409591e96eead43e7a88fcf53a00c1118448609e222d9dc5e9c45", + "748614deae3fbf232d82611b3510eaac5059a5090e33acfde910a103b370307e", + "5799d3a0544b2b30a7a59a4c129c29147f8b586f482e7c110c4114a7edcf6540", + "5eb6c0f8bd1d0da92cb43ee541af1746e149d7ee623514279f599bd065ce2862", + "10996fe4d687f4630b7362210f9c4a04ac9b4f9f59675e074e0fa9920f034681", + "153336bd45dc70ff08de1a0e262de099844e14b8e86af188b4c265e6ae3ce5c8", + "2c3b19224f8c8d913d48386ca201014e72ffb7dd41db05a473e2c04b8d4899c8", + "2c3b19224f8c8d913d48386ca201014e72ffb7dd41db05a473e2c04b8d4899c8" + ] + }, + { + "trial": 7, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2006heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun2006heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ] + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 98.08, + "first_frame_sha256": "0296961ec710a14bb1260f62984e50fcecd98ca61481e47cd417bba824c2e215", + "frame_sha256": [ + "357f2790f8d9193fb9948a3138752b1753f87a7581228e14ef9ad40931708a76", + "0296961ec710a14bb1260f62984e50fcecd98ca61481e47cd417bba824c2e215", + "4e9bccaae348ef778f65a48689253bd1e683e6da92bc5c0c0fd087e6707130ff", + "a0401785860acdfd8b51e8d0ad51ac04879290b6538bfca4ce0461cd1283fdb7", + "71d8cb22b48dfbe14f02df1c551f35423694f60313ffcc0fee2fbad386ea360a", + "4e9bccaae348ef778f65a48689253bd1e683e6da92bc5c0c0fd087e6707130ff", + "a499324361a544a6116cf61f1da272f9f0791d86d8cd0a843addee0cb7b0e78f", + "71d8cb22b48dfbe14f02df1c551f35423694f60313ffcc0fee2fbad386ea360a", + "48898d6649b0e49deb96898893267e1104751cf6796a6fc8d5b8e95dfbba8d29", + "a499324361a544a6116cf61f1da272f9f0791d86d8cd0a843addee0cb7b0e78f", + "47c4223727e94f262d123a201abee2ce6e466d5072df13592b35af15d210b3c4", + "c5e631cced3f3364ca4036754a39452496cdb074f648f57889c4c1c415c18a4a", + "e3798f7209c42c107340a93cf2bec3c079479dcbe407d5e8ce2df4948bc18d88", + "47c4223727e94f262d123a201abee2ce6e466d5072df13592b35af15d210b3c4", + "6461ba5e237c363952373ac922882584f2cf4773ba7e1cf70199827c765efa44", + "804b05848045cfdf102f876941fc2956a3b3f625cec8664bc31f171e8a312aa5", + "d633fda3884cfa9d8bfff0165ac93f84219547fbc2e9485d7a219792c10d24fb", + "5ece745e1cacdfce775a39036b72ce4813052f12033d25b901a26e8055bc98e2", + "107932f725db50eee960feefb72ad61708176ad25860048bb1cfd64928a9bf0c", + "1fd2d1b8e9c08f639eef305cc7dccab726b5f8dd3a250e0f0f0a1c43a9eb3668", + "b18e03399e9f32d47c84fdbf98a698b7484e5be61a2f15168ca57eee0a782777", + "a95cdd2e709173c2c21b3e72dae5d8b9f7de704c7372a1d49ae96c57957231ac", + "7fc25a0d5d06536cd052faa6609a3acd9683783f66ac4e873c5e79e005526602", + "b18e03399e9f32d47c84fdbf98a698b7484e5be61a2f15168ca57eee0a782777", + "bdc6516b6bf65e95df3a75024ee0a97b0d89899716b324eea8acc7070cd5b6ff", + "47662d8ed87f5c85605ae16e0361be32928eada3e2811093f6bff65154844b8f", + "e22f606bdab46219a8731c4705c1583b41394fe9d64359dc86cf7cbda5ef9a32", + "bdc6516b6bf65e95df3a75024ee0a97b0d89899716b324eea8acc7070cd5b6ff", + "14fa02377e6df16ec5cd69a92b507459d3326c1425209963d19d3caa2ca0f4c7", + "e22f606bdab46219a8731c4705c1583b41394fe9d64359dc86cf7cbda5ef9a32", + "fdf3e5a5b68617dccf1403201ec3e383b001515c83103f1c6e0a4827cccef43f", + "14fa02377e6df16ec5cd69a92b507459d3326c1425209963d19d3caa2ca0f4c7", + "ff2d722cab208b69f959e5be9b427af55c467e89c3724da90ff08501d082d903", + "fdf3e5a5b68617dccf1403201ec3e383b001515c83103f1c6e0a4827cccef43f", + "bdb38edac162ab37e7eca1d72ffdc31f1f4171fb4baf89f7e926b3f51e752080", + "ff2d722cab208b69f959e5be9b427af55c467e89c3724da90ff08501d082d903", + "5cd61870057fdd2b60e1475cec049cb0cd2a7d79f7d5d45a5d21736736dc8452", + "bdb38edac162ab37e7eca1d72ffdc31f1f4171fb4baf89f7e926b3f51e752080", + "d6708589c3465b33e06cdc6f924aabec6bd97785e46df2f8a9ddc14f01973678", + "03aa3c3ab669dc040d92a7658971e3614e37ebb5846ec4e4950e0e1b8c0c394a", + "58a6dbeafe40fca19e922de25fa5c7993d0e0f20c782dc65c0b77d373b026c4a", + "82b0a1c725e85b7435c0ab43cf2956e5b691808b7dbf9f090d29c023f1997f55" + ] + }, + { + "trial": 8, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2007heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2007heal", + "\u2460", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Middle Ne", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "3:10 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 76.75, + "first_frame_sha256": "6c393ed49f5469d37076145813b308f3d748e912f1364cb5bcae7a264887f1c4", + "frame_sha256": [ + "4279c612aa33130d2b9ddfc016dd943d74e9bdb266935df8ab8cdf01bcbeb87d", + "6c393ed49f5469d37076145813b308f3d748e912f1364cb5bcae7a264887f1c4", + "c4e9bcc42a560da205f270a008d01c64e3bd5e40834a58f178076b43a72253e4", + "4279c612aa33130d2b9ddfc016dd943d74e9bdb266935df8ab8cdf01bcbeb87d", + "59f3037bbb4089dc15b3e88fecf425f661f2572a8e208b60048527e080127c41", + "c4e9bcc42a560da205f270a008d01c64e3bd5e40834a58f178076b43a72253e4", + "8c2ac314faba152ac8e1f00d7c386c104f48603fde5aaa4cbee5f978ac0339a8", + "59f3037bbb4089dc15b3e88fecf425f661f2572a8e208b60048527e080127c41", + "7426c3fea7e6e69bb9ced1d0e9aac8a925ee0950f0b6c1e8b274702433f89756", + "6b027b8ef917670423e1bc8581486f6940f84942b9e258d4ca7b93f489602acd", + "deb05a1ffbc517e5524ee5bccea7cac9c04998b26652b0fed02e9979cc6a8453", + "be23a58242638bd608754e7cf1391569c4e2dd2608e12056d0b0b713218145ba", + "5820d1fec171eaa7aca3aa38fbfdd3e03d3bee648bcf14a4a496e68874512648", + "deb05a1ffbc517e5524ee5bccea7cac9c04998b26652b0fed02e9979cc6a8453", + "b67dd6a65157827ed5d4b22341998ab0e09ee1f93296c58c3e84072b549dfbba", + "20c2455ffe562975c9ed3e682f9974558a7bceaaa94bead551dd2decadd7ef49", + "8a8dd0e2db4f6b943028481e76bf883d7bebda7b2fad8cfd2d7ee5aab67e74ff", + "27a85ef3c8f45474354ad2a253a8aae93fbc38cc5a374090899f9a618ade76b3", + "9589008b3d611b6652a916c84198a4f435d53b0c72d1439e44afbb3b0a558e9f", + "a3be9ac59dce805312fed2cd3e110eb55b67450480343d62124ec546ac3659fe", + "03e345a2d06561e77de7ec939372837b0a47f3e91ca5a5325cd1f20d98723891", + "9589008b3d611b6652a916c84198a4f435d53b0c72d1439e44afbb3b0a558e9f", + "b33c5a86413fb6ece9dae536c1fd3f9b36eba616d1df04069f3a3dab11b57d32", + "03e345a2d06561e77de7ec939372837b0a47f3e91ca5a5325cd1f20d98723891", + "9aa64cdd61cf5592f72e687a839342aa453ab6657e5ff8c827ae39dddb87139f", + "13db0133e218b5cc9796f187b3089a5f262ed20f7dc932c980eb7cf6d1f76cca", + "3a2b6a889d62a34d37a2fc573d7fdb50fe809418157d62e6b2d0a8bf84a98c72", + "9aa64cdd61cf5592f72e687a839342aa453ab6657e5ff8c827ae39dddb87139f", + "3a2b6a889d62a34d37a2fc573d7fdb50fe809418157d62e6b2d0a8bf84a98c72", + "3a2b6a889d62a34d37a2fc573d7fdb50fe809418157d62e6b2d0a8bf84a98c72" + ] + }, + { + "trial": 9, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2008heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2008heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "3:12 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 81.88, + "first_frame_sha256": "0b9b8012d4fd0cb70d38e084f4dc5490dba94d73c1b76874f53ca7b930c100ce", + "frame_sha256": [ + "003aaa4a7672a4ec111ce9d8f3424c573c0df9f924241bbaae9648ce6ea4bfad", + "0b9b8012d4fd0cb70d38e084f4dc5490dba94d73c1b76874f53ca7b930c100ce", + "c4992738c405a20969c8d20ecfddf5eab6b7a11f1a3512da13b6489d907f2522", + "003aaa4a7672a4ec111ce9d8f3424c573c0df9f924241bbaae9648ce6ea4bfad", + "654aa15ed333421ee6d382ce155a640fa5f51ea889b10e0f2ba740ac442ae348", + "c4992738c405a20969c8d20ecfddf5eab6b7a11f1a3512da13b6489d907f2522", + "545118c86be7a55b8bc722d1f4c52bbbbd602bfc1b3809ddc354f476db0d74c4", + "654aa15ed333421ee6d382ce155a640fa5f51ea889b10e0f2ba740ac442ae348", + "cebf2d58bf0f317f858196edabb1f71475525cd96a8025ecae7e38b2708e577f", + "7ff9e80a8a47b223b397cbe0302258ceff02cd8c90d53ed6a8be40472a0860a6", + "2e8c3a0f0b65919d573e4647f2df3b487ed88373d62a65156d70311a87058b28", + "f1a03874e934277f7e660b6712222ef52f8d10737d4024c42fbca95ed9009a53", + "906b428b9af5af67cd69aef3b5c431c332273f92327de03caa356c74bb07351a", + "5f87b633a5124a51419e209e9cd8986058c89b8374b7c9eeb0805166bd1fec46", + "2e8c3a0f0b65919d573e4647f2df3b487ed88373d62a65156d70311a87058b28", + "b3de45fd112eb0c153109d7d037615aca5f6e2d32c56d4da97913f93bd9297d7", + "01936fb6852c67842bc2a87f83b860d65013373c2c62e1e34a0e0e37e1d3d24e", + "a2bc3ec76a9895502c5bfeb9cd4de270e73cf70116f34ac75b4dc1f3f038628b", + "c6f40fac190a31f326dcd0652b812727df9948661d8662aaa71d8f8a246ed91b", + "67696cda659ba859b671ac8b0760aa2e545c58f06514fd3f69885fd1611e9d07", + "6cf26796461b390943aabb2c9db97110e1b2b0b3ea55f1d79ed1964a57742638", + "976c5ff49081cab4e3bf9e38cb6584195babc03b9805e1f35275634a9e50b484", + "67696cda659ba859b671ac8b0760aa2e545c58f06514fd3f69885fd1611e9d07", + "ace73914fe842d78277a6ecfe5ef1ea4eee587d462c122be029259ac737cb46d", + "976c5ff49081cab4e3bf9e38cb6584195babc03b9805e1f35275634a9e50b484", + "a7290b594783bb886aae56491d402cf0b19b05927e998b075edbf0b74257b697", + "11af8439ba8fd49e6a8ea643a3f4daa94fa1ac87595093e61df0977634a8a9b7", + "70bf223acf6370bc485b7a0e5798dd93c142337b273f823bde6a53da30b0a75b", + "a7290b594783bb886aae56491d402cf0b19b05927e998b075edbf0b74257b697", + "70bf223acf6370bc485b7a0e5798dd93c142337b273f823bde6a53da30b0a75b", + "70bf223acf6370bc485b7a0e5798dd93c142337b273f823bde6a53da30b0a75b" + ] + }, + { + "trial": 10, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2009heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 7 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 454, + 349 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_011' (click recorded visual target): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Preferred Name: Patient preferred name or name patier' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_011", + "intent": "click recorded visual target", + "reason": "Identity check failed for step 'step_011' (click recorded visual target): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Preferred Name: Patient preferred name or name patier' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpb", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2009heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Middle N\u03b5", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ontart", + "8", + "O Type here to search", + "3:13 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 53.3, + "first_frame_sha256": "16f894d1e67127d6635b6a4d29647f3fff2c3478268c09b1b7644c08dfa28d26", + "frame_sha256": [ + "3be7d4bc1ecbfa201953f0e2dced22f920b9fe37f00a16561a8bd99792db2267", + "16f894d1e67127d6635b6a4d29647f3fff2c3478268c09b1b7644c08dfa28d26", + "7337630ef1f395a7538e3983706d8ca3420f206d32da1f7503beef4a6c9ab368", + "3be7d4bc1ecbfa201953f0e2dced22f920b9fe37f00a16561a8bd99792db2267", + "341f315f81f34042387ff674eaac7093c43a518ddaed88eb1537cee08480b87d", + "a48e71c1e39635ab89c2bca3abddce58fe381496bb27bd55cf29ea238cfc2a09", + "461286052d59471e7820a9b8f0472231f5673946ecfab8c1dcf1d0507dcdfadf", + "341f315f81f34042387ff674eaac7093c43a518ddaed88eb1537cee08480b87d", + "2f64fd66c9e8c55f3b116d28e48b8ef2505f8a3ae29d1f1e11a8ba10236a4f4a", + "105a6912c6ec60cbd98499e4ad4213064c7310d79c9c6946dd128a45c4f87e20", + "e890ddc879159d0259e1a11168238628d383d5987ef67733a25eaca4c2f778c1", + "232d64a73f40152c360f1e1bc9ddd25c6fc227d55ecb93e45f0dd395971b3a21", + "5b645d810f6721af970b6db41fe08bbad93aca1ff0cb8e36d20ad111675a73f2", + "6b7040474deaecd60b50d5ff7555eeb75a1387def40249e7a5e4ba27630f50f1", + "e890ddc879159d0259e1a11168238628d383d5987ef67733a25eaca4c2f778c1", + "fceccef506b977e9bf27690bc81b758ef433acb251211d7377baa17b79802777", + "33540d510a8cc781c7c7fb963830c0a6d34fad445dceaed80de693a0785e7c22", + "a105dc9052dae62f59a6051aa4cafc37eb89bf8ba0b8260b3dd2b66bf9cc65d9", + "c819722921e6accf303f609950c5850922571e9e82aa0dca2086e2b221434817", + "ef9faabde08c739ece8dc7268c7f1200cadf00951496c93f56ddea99225ce91f", + "d2e73cb0843ecc4e84cfbb6c805222b0db6966794f6e8e195455ce095068873e", + "84c9c958d1ac33bcb57805ad803add27ddcc528baab4b2e9aac9b000cde5996e", + "ef9faabde08c739ece8dc7268c7f1200cadf00951496c93f56ddea99225ce91f", + "1ff20f6aae35c6fcd958a6719ea49e1aa16b075a5c96d66de433c13f6bfab461", + "1ff20f6aae35c6fcd958a6719ea49e1aa16b075a5c96d66de433c13f6bfab461", + "84c9c958d1ac33bcb57805ad803add27ddcc528baab4b2e9aac9b000cde5996e" + ] + }, + { + "trial": 11, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2010bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun2010bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ] + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 102.73, + "first_frame_sha256": "818de0dd7fe12070db472daf791ed626267d3e32bbcdb62cddf6514304db7487", + "frame_sha256": [ + "0132f574c83f7ffab11f438c48997008930191d48f4746b0ef0cd10ae0d1dde7", + "818de0dd7fe12070db472daf791ed626267d3e32bbcdb62cddf6514304db7487", + "c4539440bb55ab9d20fde2d27b373101f3236afbe2c8418925f421d176146ae8", + "0132f574c83f7ffab11f438c48997008930191d48f4746b0ef0cd10ae0d1dde7", + "570d56661ceec62e849f1f23a92c6ddc6f9ea2830f28a2729a00468023dd58fd", + "c4539440bb55ab9d20fde2d27b373101f3236afbe2c8418925f421d176146ae8", + "68d93d70c2c9a319f7f3c3ebe9a8bf50b76645d5514dea5dd3e2d8e3fe21b29d", + "570d56661ceec62e849f1f23a92c6ddc6f9ea2830f28a2729a00468023dd58fd", + "f9aac995efc66ec114e69ae66000b49137786a9fffb554223ac33388f7d4e2d7", + "50f211cb7ee23d89eecbf099e88546b8dd08ce1f94f286679730aa08384fc289", + "04c9fb53bf7980867b0af395ed72a41deb922054c8c0f79a36e0f19a965f1b4b", + "89f3718f1099f3dd8224b9d01e14ef7d430697c129321a973d94448c3ac63734", + "70395484a0bb2c3bf9da6442b674971a48f121c11227609ece97830ddb21914b", + "04c9fb53bf7980867b0af395ed72a41deb922054c8c0f79a36e0f19a965f1b4b", + "a6fb97b635144e71f1d522662d9a967ccf95f55af066d3823c1ebab27f98359f", + "f8553fa330d25d8686bad34596df895162a803e19d1c10d6075eb9c7acb7cb08", + "cb6b7cc22e1ea7b0511236a729b63cbe0ec7aee4a16ba0affac5050f1c4ef3ab", + "a0c5eea379c3b3f2d74539e81a8570885691f4deaa5c3eaeb11f23925208a336", + "8b0403d5d3978bb45082d7bbbc41c55771dbfb1c10a3be5fceabd4457780c280", + "ce5de8dea246f1a67b89ee7075f0f6a7913122fcbd380d9ad9ce20e218f4cf8a", + "9896f60542432f37b2d90d0f5e0fda279f653159b8ba16275c4e82eabf2196f3", + "8b0403d5d3978bb45082d7bbbc41c55771dbfb1c10a3be5fceabd4457780c280", + "05e370c4bdfea832931d3cde16c12c6a2085ff3bbe30adf1c128df3ca87b107f", + "9896f60542432f37b2d90d0f5e0fda279f653159b8ba16275c4e82eabf2196f3", + "7df42d5b5bf0b3410d25e2481fa12b6abfc7ddce9bfa5d66da7478eb780f9536", + "4e7313d5973e3895da2098f23f3cc67c89b293a1650ac8cf34084e1639499588", + "457f86e45a9d3e52a20f2647c5baaf697eabd1b80b9ea91af4bd1d4b049e9dfd", + "7df42d5b5bf0b3410d25e2481fa12b6abfc7ddce9bfa5d66da7478eb780f9536", + "dae4e8a7f4ce0af5fc7a7ae6051350778c2d9518a72c0507a24d1fa1e656e959", + "457f86e45a9d3e52a20f2647c5baaf697eabd1b80b9ea91af4bd1d4b049e9dfd", + "2ee904f3c7f61088a1fde4595ea561dbe0bece3bb7d8a40c2c99106301638c29", + "dae4e8a7f4ce0af5fc7a7ae6051350778c2d9518a72c0507a24d1fa1e656e959", + "a2ba84db55b48c4b699ca2c73ceb3a45288edee571c67a0bc75f64bdd0d54bf8", + "2ee904f3c7f61088a1fde4595ea561dbe0bece3bb7d8a40c2c99106301638c29", + "45d51ab3009cadd1ed55817206185f2528b561312827ddd2e649702c54e1a678", + "a2ba84db55b48c4b699ca2c73ceb3a45288edee571c67a0bc75f64bdd0d54bf8", + "d9ee914e2a7290c08af18d063faafbbcd9d5ec86e89254c3b07e16642bafce0a", + "45d51ab3009cadd1ed55817206185f2528b561312827ddd2e649702c54e1a678", + "1dda3e4a74b2c30edbc7da2619e593c6176ffcc19fe753391936a05b9876bdf5", + "67e71a54855343c9be3abade7cbb06558daeab4719514e9f8cb2690b79a58631", + "24705062aa2eff3e229effd93056aebb3f95d5f7f4d2d8aa6d5afc9dcbea02d7", + "7a8258600623323f643571598729ba3a01d7e4594374b2df6e78735ce10fcbb8" + ], + "first_frame": { + "file": "emr_bpp16_first.png", + "sha256": "818de0dd7fe12070db472daf791ed626267d3e32bbcdb62cddf6514304db7487", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 12, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2011bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2011bpp1", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "3:17 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 77.45, + "first_frame_sha256": "e472869318563b77e600b112e452d9ab7f8354a8f134ecd044162d99e1f0101c", + "frame_sha256": [ + "067e966eabce965820a43116ada87b764dc78ce0f1f046a96889ada595673fa5", + "e472869318563b77e600b112e452d9ab7f8354a8f134ecd044162d99e1f0101c", + "0c8a3f68c121716e2d4fe8edbc2a97e9958ff16c626f55d5c4d273d0124621ad", + "067e966eabce965820a43116ada87b764dc78ce0f1f046a96889ada595673fa5", + "33d311529d67bc37ef5de187434c35f7a4f670de50bdcef1b229e1ef35431e41", + "0c8a3f68c121716e2d4fe8edbc2a97e9958ff16c626f55d5c4d273d0124621ad", + "81fbd500e9f9cb2eb6210eacb5795d79c2547298e7d608e81f57c2971c94484c", + "33d311529d67bc37ef5de187434c35f7a4f670de50bdcef1b229e1ef35431e41", + "935a922e100e69f4db746dbd17860cc71f0cb47c35df6799c10441b68fd722cb", + "21dc17a63c45752da2e7d3b722783c1f05e07390c1b6634508a6122587e03b30", + "10d43c1104bbe7cec1c2bb46ef710ce2a034edb00689af12b38e9046cb4820e2", + "3efe2c5a0bad8951c5592cce66d6bf64f0871bee62d176097323411e40b27499", + "3778c1d0152cca596c7fe65ddda429aa564591e3d0c31f157e3ac9d3042bb41d", + "c29ebf319bbe4f3f9e85bd9356f6d716466670713e107ee14c597eca50377a2a", + "10d43c1104bbe7cec1c2bb46ef710ce2a034edb00689af12b38e9046cb4820e2", + "509c6a861149ae13f2dad8bff41e7b65028fd7b752f3f601c3bbe268116bf6b0", + "2804cace45e4f01c5c232963729bab34bc094479aa0fdd6e8ad69c73d91d0d6e", + "1ab366c5fbd4a84fa7476b55326843097962b26d1e133f7660f26cb3ddfb96d8", + "9b8f68c7a8390c138bc9f77ad9a32728546b8496c6c8ebf7fd227c0fe2b06814", + "f01b82f51688b328f5c0f9dbae97e33ad80a66cd2108eede732e652bb8fd702a", + "6e51665f196634dce366490fa4b98285f61f39b238dff6dbbf4ad2070999f1cf", + "11df682c1ebd5a06e59e854fc8a1b271e65707000ffd73db5d43a3eac5d286d4", + "f01b82f51688b328f5c0f9dbae97e33ad80a66cd2108eede732e652bb8fd702a", + "b810164c1096c2d2f3eef8cf4c879ec24873b9501af6aecaff3bfacdff000fb9", + "dfaa10b2ac24350aa529b620554ec184ba1ab498c04ae1bae5ac85bb11cde7f4", + "1526aff43f60f6f56d7cd9909c6cd8cb8111815b0e9ff26925e578be008f194f", + "4e98681ae41c7d283ce3f637c493d4884b32459600526ba711481b321d21f7f9", + "8b5baac59769f5eabd8b2948e6cbb7959d09dbc57be55ef8dcd2bf8fba7cc33b", + "1526aff43f60f6f56d7cd9909c6cd8cb8111815b0e9ff26925e578be008f194f", + "8b5baac59769f5eabd8b2948e6cbb7959d09dbc57be55ef8dcd2bf8fba7cc33b", + "8b5baac59769f5eabd8b2948e6cbb7959d09dbc57be55ef8dcd2bf8fba7cc33b" + ] + }, + { + "trial": 13, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2012bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2012bpp1", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "3:18 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 76.47, + "first_frame_sha256": "4633f79f3368dad46ba4682fad9a987722069c7e554308142a600a14f9f10983", + "frame_sha256": [ + "a4d219d6652cfaed0aeb4f02c392f740bf97906bdc0deea1063868c0d5fabfbf", + "4633f79f3368dad46ba4682fad9a987722069c7e554308142a600a14f9f10983", + "3272b93a0ea633efecf5d720357bd56705d40189e0061f2e3e4fbc7fcac2ee31", + "a4d219d6652cfaed0aeb4f02c392f740bf97906bdc0deea1063868c0d5fabfbf", + "587d6c074e87962f75e586f8985c3b7c4d7630852c98736f6b14b2b738ef8c80", + "3272b93a0ea633efecf5d720357bd56705d40189e0061f2e3e4fbc7fcac2ee31", + "728dc158351ec5a17a82bb5105c58892189038dbc8f28e2b9d952f5e77689cb3", + "587d6c074e87962f75e586f8985c3b7c4d7630852c98736f6b14b2b738ef8c80", + "db9026b6510819871578e9104c7ac0ec4b87beeba3a521d4f95b11e8760a9e13", + "c120e7239f59a509a1aedbebfc444037bb0ac5323be8572ea000655fad5e8dc1", + "09a5f0dc9d83db35c003199cd2189abc98742fcde453def63beb68dc37df229b", + "e1e849e0001ffde9b2c0b8c8567f428b41b71079c4379bc0d98a9d38e4830635", + "045ed17a8476fd963eabbfc24075a6903743981a6a9dbf7c9704bf3640816c5c", + "09a5f0dc9d83db35c003199cd2189abc98742fcde453def63beb68dc37df229b", + "a54f1f1f41f22f69cd702db713e5e326c5a932262db9f5ec9292427994a55feb", + "dc20516c893d6cec9e3baa4c6cd2e78c57b521e800ffcc29cfdcdb855d07ee93", + "bce59e8342e45eaae6d6f081bedc51d719bf00538b1ef35a4c7a4d36f37f2bdd", + "e9d5600ae52427eb6c817d277e5746c1a099eb4f1d42b0b9fdf31bce7ee8c6fb", + "d9b3a33c9de64e94d4cf4d3d72c55ebd74a2e1cac242ea8ef30ac2f47ebba6ac", + "4c4d8daec6d9adf0aa0d360d11e92536d125dd3070e112f67b358d8223922d0a", + "22929bd3a3f2c996f71b9598b37ed62160ef38498b34c90fcb4c721a3e09a1ef", + "43966822f684e1180ab513da0de89890bced0192659546f8e49a3e8793a4b2fd", + "04968c72b0c1e0fb84ca93b7913ac30d9a2b5ee20c097f035947018c2cf8e83d", + "22929bd3a3f2c996f71b9598b37ed62160ef38498b34c90fcb4c721a3e09a1ef", + "94b22eb938f8f65ce20a4964d4c8cdd90a981a7c767959376b902ea89dd8e9e8", + "0d45869a1b052dbb2f358c29d0e6cf156a18d7ecfd11f2ae9bcab9fc007f2f69", + "f397f01daaeb1f77a4214b2e5c5bdc3beaad48702bea8af4e89dc403f94d85b0", + "94b22eb938f8f65ce20a4964d4c8cdd90a981a7c767959376b902ea89dd8e9e8", + "f397f01daaeb1f77a4214b2e5c5bdc3beaad48702bea8af4e89dc403f94d85b0", + "f397f01daaeb1f77a4214b2e5c5bdc3beaad48702bea8af4e89dc403f94d85b0" + ] + }, + { + "trial": 14, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2013bpp1", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 8 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_014", + "intent": "click 'Unassigned'", + "reason": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "D", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun2013bpp1", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or name patier", + "Birth Name:", + "Birth First Name", + "Birth Last Name", + "DOB:", + "2001-01-01", + "Sex:", + "Gender Identity:", + "October-2026-", + "Sexual Orientation:", + "External ID:", + "Sun Mon", + "TueWed", + "Thu", + "Fri", + "Sat", + "S.S.:", + "27", + "28", + "29", + "30", + "1", + "2", + "3", + "License/ID:", + "Marital Status:", + "5", + "6", + "10", + "4", + "7", + "8", + "9", + "User Defined:", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "Billing Note:", + "25", + "26", + "31", + "Previous Names:", + "Add", + "ontart", + "O Type here to search", + "3:20 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type ", + "click recorded visual target", + "type '2001-01-01'", + "press escape" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [] + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 80.05, + "first_frame_sha256": "e326da3452d75e45cc15f0934d19cd1a5e63be43136a26bbc9dc12cb0d15fb9a", + "frame_sha256": [ + "dbaff38debd87f5eb8e867cef0971b10a7a6248d315767addc9a4c9d68a464dd", + "e326da3452d75e45cc15f0934d19cd1a5e63be43136a26bbc9dc12cb0d15fb9a", + "ee029037a57da8fca161d550b44ee30070975b556aadb82ed48fef7e57c6d11f", + "dbaff38debd87f5eb8e867cef0971b10a7a6248d315767addc9a4c9d68a464dd", + "dcb11aa25faab5253d6cea6640b313b049a0466e79d724d3163a542a083030bd", + "ee029037a57da8fca161d550b44ee30070975b556aadb82ed48fef7e57c6d11f", + "5a483c891389b37d3ed9b6ac63ef5ea78b91efc9ca8ec1fc87d343f732196123", + "dcb11aa25faab5253d6cea6640b313b049a0466e79d724d3163a542a083030bd", + "e91bc80c44947f7a658d6fb78f0be4f83c4b1f6ede3ad380ae70fb3f4cd81a9b", + "bceb1c6a7ad6acbd921195ae15b622d34cf84c791779248ea22721d4fb04630a", + "f0eb1f2f712f1ad29271e4986d99e849f6546edb589dc329323da5be040edd8a", + "9a2533236dbcb68c0b8806a7812fc9ff15e195c5bcab97a28f1503179a4d2ab2", + "2ea3426dc986bdbd5ff62b74e454a2cbeca74e7c0734607f90afa4012bddb5b5", + "f0eb1f2f712f1ad29271e4986d99e849f6546edb589dc329323da5be040edd8a", + "a75c2ed66918882dc6156f844c45bf4178118a0c4a0250846f1e3900809fc1ae", + "dd5179f77343bc17fb97f6c9e3a259d01f17e5ad12a6deff50b28015fe32bf43", + "c4cbb5528fd19fadf3372f1d70b4e8e35d3b5af00fad5f8c440234ab7e3535ca", + "1828941e99ddcbbce3942e6fbf47db3e0cd1f8ded676c4a35873c04080a3c3d5", + "29a3e22aeb17d02a60e3c9123f6ffbff1d06869c83ecb3957ec64979659760b1", + "0cb13a0c7561c850b5b1308b2577355248d3f25550f4b80cc8cf79cf2f31f82e", + "1af73ebf6e24b8aa3bdc873bffafd01abc8254df6794eac631568834d594feac", + "1309440a7d8fb60b0bc26cc54b8137e3f22e6f34dd1a15421441dea7468df4d8", + "a6843fe6dd6e8072822d49c8ca02281e5660355929a72698937515c568ee21f0", + "1af73ebf6e24b8aa3bdc873bffafd01abc8254df6794eac631568834d594feac", + "7bcd03bb0f86447595063555dd9e6ffc4b29ca5e926770a3f3977de13d8afd63", + "7b93f11a29688e419748ce17065e1136a6b0280276a03de1cf2a435c2b3fdec1", + "9c8c79227a1a6a1a48b1e4397d475077a1c8f9d7d0fd7a2c6915a0890c22911a", + "7b7a799affa638d56a507826c8aed436f3dc20a79c431aae52aff08c5c2baabc", + "9c8c79227a1a6a1a48b1e4397d475077a1c8f9d7d0fd7a2c6915a0890c22911a", + "9c8c79227a1a6a1a48b1e4397d475077a1c8f9d7d0fd7a2c6915a0890c22911a" + ] + }, + { + "trial": 15, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun2014bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun2014bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ] + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 107.94, + "first_frame_sha256": "35bba20a4068756676d06e057ba47110e73056f2862b35c9b91803985bede067", + "frame_sha256": [ + "56060647f0ec4050a0e476fe253989f647d58501f6bda8cf72a01547a500fae7", + "35bba20a4068756676d06e057ba47110e73056f2862b35c9b91803985bede067", + "938ec2655de4d6982506c9027a6c3684ed29ac8fdf48474ea9e80a20d756c594", + "56060647f0ec4050a0e476fe253989f647d58501f6bda8cf72a01547a500fae7", + "2d1796828f92eeb87be347ca80d7f079a9e7ba8f59d24c9f7c616587e84af775", + "938ec2655de4d6982506c9027a6c3684ed29ac8fdf48474ea9e80a20d756c594", + "b22cbeefa04bc29832fb23eb32e14ad958446cd3914273af9728468ec15c45dd", + "2d1796828f92eeb87be347ca80d7f079a9e7ba8f59d24c9f7c616587e84af775", + "1a76eb6b13c1995e315c913ca1b92ae81cdd8c62b62736d949cb9c086f051963", + "4fe4cdb3e75422d25c52a652ee979aa27f859258bcc17fe084e147966a423502", + "45866d2056b387d09ef2ba12fde39720b8803b07c31d694fcf5a7608c62cabe2", + "e995b3d890828d33ff4c59de502f51bb8b8ac162f663229fe19e2ac39170a26b", + "206a750e80582ce1b587cd43d1750db9cff224783ae2ac8d6a28aede7c827a2f", + "a7b051cf9dc36e482d1da95e5c03c697e10272a27435f33e82fe973b3f985a2f", + "d19dcb17aca2c2747aa4d294ae5d7acb0dfe781c53b2cd62dd9c5d668d35d68a", + "9ef9b863dc11e7f053926271e91db3b3d3c45e3b6ff7c656c771de82c3a39721", + "393088ef959b558f9def8ed74eb4a420dd4e65e5d18fa0da0a3b366ec8abcb15", + "b512e2a6cd4c5459041ef401999d30c5a4452dce0a5a2d85f6bdc98c96737027", + "2eb2f08b22f687ae07afc42b61cb686b1360f9de8541ffc9bffa68bc332d61e7", + "b456f20af39f05ac429f56982722009ea44dfede8c845ece94d374231d5585e7", + "04d4af4d1c83b8818d9e1c818032a6de7839aad6b463b8a32cf7841f834e6280", + "2eb2f08b22f687ae07afc42b61cb686b1360f9de8541ffc9bffa68bc332d61e7", + "5d91afc0b1a5c6a20e7cd8f867101c0ca13eb6734e244b6119a2ce6f33afa31d", + "f50f855e7ea75a9269c5837652b712a6cf95307a6bc03dc44ada7a051b471534", + "ff9e057e474cdbbb5a9139b2434caee4dbec64227d623792e5a342d3087f0d61", + "452f3f3e545acd517a33b48c110ada78cc115e2c19224fd7b46784e7c8451d21", + "8a7082d315c2713ce2151d871dcb23540961121f54dc736ca910520771fba58f", + "ff9e057e474cdbbb5a9139b2434caee4dbec64227d623792e5a342d3087f0d61", + "72ac7d0d8112f22b39826fe286463b9f62468d2c2a82d397fd857e5370a678a0", + "8a7082d315c2713ce2151d871dcb23540961121f54dc736ca910520771fba58f", + "68691058ae45e1aaf64b647955b09fe2de96fce195d468b309fdfcd08d911cb1", + "72ac7d0d8112f22b39826fe286463b9f62468d2c2a82d397fd857e5370a678a0", + "2fb20a3d74dc696c107a80916d9ee83fe21404632e2d74797dc1b428105c946c", + "68691058ae45e1aaf64b647955b09fe2de96fce195d468b309fdfcd08d911cb1", + "04126417aef71e313ae33ed91b830af51b57b160f0063f8a971bbe4e5a352a7a", + "2fb20a3d74dc696c107a80916d9ee83fe21404632e2d74797dc1b428105c946c", + "246cce3a3c87e37aaf41205d17853c70d41b528c95025007a1b9ff7b64a5def9", + "04126417aef71e313ae33ed91b830af51b57b160f0063f8a971bbe4e5a352a7a", + "df0ffe28baf1dc887ca06e5ac3d1e1197ab795585c7ae50958805f16699e4f4c", + "caf82d0c9c1e8c248bab221ac968311ed595577e005f00272f1cea2833e51a9f", + "11349ca491c1a41e1459dce2b5a8ff9fd849b7e0ada8b76ad7e6a5225f135102", + "52187f10265a7a139693a77890fa6fd7212272ee4696db2fe3301bdb624ddc64" + ] + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-3/reports.tar.gz b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-3/reports.tar.gz new file mode 100644 index 0000000000..dba8942470 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-3/reports.tar.gz differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-3/trials.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-3/trials.json new file mode 100644 index 0000000000..a7ab87401e --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/emr-run-3/trials.json @@ -0,0 +1,5136 @@ +[ + { + "trial": 1, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3000heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 360, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.991, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3000heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 115.23, + "first_frame_sha256": "52f4c1d7f58a8a83cf28ca457d198d0e29132eeadb4f74f2d6c3a40708e5a7ec", + "frame_sha256": [ + "57f7bf61a60421a2235de93522d10ae7411d0ca4dcd424a9e0bab1d4e24abf19", + "52f4c1d7f58a8a83cf28ca457d198d0e29132eeadb4f74f2d6c3a40708e5a7ec", + "012b48fa2bb31fcd2633949872cbd7f6e61da7b603d806d61849f9416b0be8a4", + "57f7bf61a60421a2235de93522d10ae7411d0ca4dcd424a9e0bab1d4e24abf19", + "31feda3d6550a76d8d2be1f46ea4a80f6589aab7f100cf2238830986cd0468c1", + "1194d95a1bcbded92be35c73f442dc2dfe7181060d3384aa42e359496a070185", + "31c6f385f10a20c0ef9ec6df52e0414bebe80bd6fc00bc14e4ce29b6387abc7d", + "31feda3d6550a76d8d2be1f46ea4a80f6589aab7f100cf2238830986cd0468c1", + "a2b11ca024f128a911a807dbd3d0592c95d1f25cd97e740521eec2ac611bdc77", + "31c6f385f10a20c0ef9ec6df52e0414bebe80bd6fc00bc14e4ce29b6387abc7d", + "35210a4da11e93e2ff79eb8da4453aafbfcc9b514c3f93e8b05646195cf857af", + "7f8ec25c8363c0a559b7503fe76e9edb316b02c3d997461b90241ffd5ba7c4bf", + "c0a777e0ad853bf68033b97b121da40bc9f27e604c13608b603816a47f754c44", + "35210a4da11e93e2ff79eb8da4453aafbfcc9b514c3f93e8b05646195cf857af", + "cf3045be79945d3c62a4f9d2c3dc32558ab75eaa28f9344d4dae9160c67564cd", + "9ad22aac15c13dab8b0ece3c334919ba22c61d7c5f2a593dac2a43b8b1e3e700", + "5133b8c80e0d0661d15882080254e469dfc89ff86398b533428e1b9697e75da4", + "ba68f1464cbb5bd0cfcf70652a8614804c1d8709e6cabf864990ec7ac8bc563a", + "ad607c3e329a8245ea3808e55ced5c8bc0a82f8af43eced720224479567ffecd", + "069f5dd2b2ab4076074f452e935a40c4873d72be4847ecee72b5d324db2017a1", + "dd5d9ee787e273ffa263918a8022ba5c9d341cccee0026d512789fe21b665a9d", + "2985df0416a805f7bc8200d2df9b2aa66f075aaf1e095e71cf7e785c81e1f46e", + "37484b5e57b28e03e6f724124791cb891b65f236eed18e92278c2bde7288d04b", + "dd5d9ee787e273ffa263918a8022ba5c9d341cccee0026d512789fe21b665a9d", + "84f16d44de2f4c81c38d7a8882d48da957ba953d342d8195b4bf7a33fefa9788", + "219dbf9ff917babb96d8e390d790c7d37079754e72de58f5ddf31434997aaddf", + "d96bee174d56912ef687374bfbe6a7141cbb62b9cb5135281f433467eda7d7d8", + "84f16d44de2f4c81c38d7a8882d48da957ba953d342d8195b4bf7a33fefa9788", + "79b637bd45ab3bd4da1fb660482457293664d29adaa010fb956141c7411085e3", + "d96bee174d56912ef687374bfbe6a7141cbb62b9cb5135281f433467eda7d7d8", + "db337a394c452188564a4d94563791eb7c4263bec3f833726412f4ae490b06f9", + "79b637bd45ab3bd4da1fb660482457293664d29adaa010fb956141c7411085e3", + "af25ab5453c12dbb5543b6d0b79c687be9071d1778d7f89b59f8fb64347fd38b", + "db337a394c452188564a4d94563791eb7c4263bec3f833726412f4ae490b06f9", + "c913af4901f0b8e18e49d02327afa470615724a5955956671d2164d8d6f06d45", + "af25ab5453c12dbb5543b6d0b79c687be9071d1778d7f89b59f8fb64347fd38b", + "60ea940ba6a4f00b974bed142457c998333499ab46fc43e00ffb9345a49c4e99", + "c913af4901f0b8e18e49d02327afa470615724a5955956671d2164d8d6f06d45", + "89e0a0c11668a6f1d20f4053383104e9bd1973c28bc7aa992c73c01fb6087f7c", + "279b5d564c3a663ee80785c6052a75e791de2d49fb6f7f6ba71f2b989e9c1ef1", + "cd2152c059ed589043a9b54b5618b36b79b955aa6291e5c21ee113fb648077a2", + "c7a30ba1dc60651aaaa3a8f9975f8d0f4d003c435e51376a98670b9a9f9d4a96" + ], + "first_frame": { + "file": "emr_healthy_first.png", + "sha256": "52f4c1d7f58a8a83cf28ca457d198d0e29132eeadb4f74f2d6c3a40708e5a7ec", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 2, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3001heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3001heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 96.78, + "first_frame_sha256": "04b7f9c33d383368eb0347dc1e48d5320455f86af33ce654573f316fa78153b6", + "frame_sha256": [ + "659a6d517ac5b287186d5e04d334e2b14dc50600f8ca6483d02c0e9b153e4bdd", + "04b7f9c33d383368eb0347dc1e48d5320455f86af33ce654573f316fa78153b6", + "674c236e2bb1b546c2b70e8ef739dff2306f33949d3d5f9b8456d5b8c6485aee", + "659a6d517ac5b287186d5e04d334e2b14dc50600f8ca6483d02c0e9b153e4bdd", + "1d4fcfd842300542e401cc577a7fddbbc25cfec75a405d52a857d3ae2763fb48", + "674c236e2bb1b546c2b70e8ef739dff2306f33949d3d5f9b8456d5b8c6485aee", + "079f090f8efbd2671949a951e8f3fbf669366078aa752b76edd85300bf0620b5", + "1d4fcfd842300542e401cc577a7fddbbc25cfec75a405d52a857d3ae2763fb48", + "6ccfaf5363e3fe42e042d0f31647d467e246757d39502c074fdd9abbf25efac2", + "9c8f43964098ba10782673514dc6763b6fe56f81e51d8e8444d12609acc379ba", + "3e891ba89da7ec6082a6685205818276a3ea552b384193e2924c21cbac763ac1", + "d570fb6e5d07ad71e0500e033024c19feeeebcd68511c8512409a27bb8ae40cb", + "31078aaaf6d2dc955ea4ee9e33b30116cd97137619919a8c99d5ac6eb3e2eb97", + "3e891ba89da7ec6082a6685205818276a3ea552b384193e2924c21cbac763ac1", + "89c14217456d1a80b13dfae436661234585bc839765328d76b4a5fdeb8b7b5c1", + "81ef6430272d027a8a181f9d6efcb7ace0f82c843ae0213db3cacbadd4d52b7e", + "7e485524276560a223e1cd89edec5cb60c18795c29651dd09b1db7d8c5f28ac7", + "80adb2547305e3f46d63884016bbcd3c31f8b4792bd652c75bc6a4795de192cd", + "3ce9ebf7bfdac315176605209fe5d76ff69317dc8ad3a053962b639d9f7ae820", + "9a4375191c91716799b2a769584bdb42dc1cdd59505bef993de651742d6a81be", + "f758c55c00a740e883595c700509216bb135aad2c7b62d0b01d91732fe3e0ef1", + "3ce9ebf7bfdac315176605209fe5d76ff69317dc8ad3a053962b639d9f7ae820", + "a615b430dedad4fc9a7957e0bc76878d1f140ea2b91198a81bf60f1adfbc3d9f", + "f758c55c00a740e883595c700509216bb135aad2c7b62d0b01d91732fe3e0ef1", + "f426c187b26b1fee34891b66baa682d2c2733cc39410fe7277933e36d42bee6e", + "64fdf165367a2e3e7c3fbe8d62fd35686fac1856e4dbba346b97f2ac5f9abc79", + "c1a332ef30197113f7ad7684a66e605a1124caf8d51b84a81af4ee7782f0557a", + "c1a332ef30197113f7ad7684a66e605a1124caf8d51b84a81af4ee7782f0557a", + "315bc41d353275e2446c50dda371ec3df31c3f6b7c77c8e3ec70b08e22889f56", + "d9ae2c9a5e00b7f15d7f53f749b1246a73c4571417bec4976ce19f2f4573fbf1", + "97925ba68e923aba494ddc93b9ee9c6ad078191749d6e29b56ee36460a99ffec", + "aaae668005d748afd5bf7e94eae155fe7fd27257e55d10aa2501cd43f1b23f9c", + "3e3a3487b8f6a40943c4164b34e6d9c09ee696c4b4774ce386a3bbcb09e1ab11", + "97925ba68e923aba494ddc93b9ee9c6ad078191749d6e29b56ee36460a99ffec", + "9475ce04a8aec2fbdb569d7d4e6caec7b7e12630d04ef99357ff5b323184e03d", + "3e3a3487b8f6a40943c4164b34e6d9c09ee696c4b4774ce386a3bbcb09e1ab11", + "7ac81c595a36ee666d6762d210f4feb80021caceee7a3152ee05ff3d208da9e9", + "9475ce04a8aec2fbdb569d7d4e6caec7b7e12630d04ef99357ff5b323184e03d", + "66bb031de2e0d57be97e3746ea3baf2a69f9925a4169a2c70db853eadec31e7c", + "8588b9450b30822f250eda050c015cc9bb4b4d464c0e9c2a87e771704d2f47d6", + "d5dd0a1df84f9dfd2487dbc777a94eeea81412a06f831fa9dc6efa437bbdcd4d", + "852e0b0ea69bd79ea6fd7d9dac7b1b144b489f3f9923220da28f4a8c9f0d184a" + ] + }, + { + "trial": 3, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3002heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 6 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_009", + "intent": "click 'First Name'", + "reason": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "Calendar", + "X", + "Message Center", + "Search orAdd Patient", + "Search orAddPatient", + "Who", + "Title:", + "Unassigned", + "Name:", + "First Name", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or namepatier", + "Birth Name:", + "Birth First Name", + "Middle N", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ActivateWindows", + "Go toSettings toactivateWindows.", + "ontact", + "8", + "O Type here to search", + "4:05 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "halted_before_effect", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 49.64, + "first_frame_sha256": "e775922ab2f938ff8eb346d88aeeacaa48099c0db1d1d54ad071eead9fc903bd", + "frame_sha256": [ + "aa5b04d34173f69830f6455975d67b9dd091bb9df626484501cb26f2024f9842", + "e775922ab2f938ff8eb346d88aeeacaa48099c0db1d1d54ad071eead9fc903bd", + "6ee8a46d3e7e0df6d50bb071438c7d202a1bc6f274b04e15c748f51e5b92c5e8", + "aa5b04d34173f69830f6455975d67b9dd091bb9df626484501cb26f2024f9842", + "1f304f3e1ccb43573c9489ea05dad4eb3b5c5305af49243dcebe1b941f6a739a", + "6ee8a46d3e7e0df6d50bb071438c7d202a1bc6f274b04e15c748f51e5b92c5e8", + "820fd1d5e42f5b4052cd320d0c9ebcfbe0d3e3be4dbd81331c6a660afdf59aad", + "1f304f3e1ccb43573c9489ea05dad4eb3b5c5305af49243dcebe1b941f6a739a", + "a38a6264c4294e1fdf956b1834d2544d4a38c810da9e567f505b1e04ad563782", + "4da51b12ae0b9aee633ed68e2580e4262a09a86609958ca3ec17aa8b7ff22c7c", + "400b3e7b99074a0f33313d31e0b74e610553b2d86263664c86c1711f4ecccc73", + "85052a49e9b2e61578761424edaee86553422ddc6e784894b37b3c993b659eda", + "1a5bf5370756c7cb2b5e3b7202de58a838a0a2d81c155fd72e78df94a62bb172", + "400b3e7b99074a0f33313d31e0b74e610553b2d86263664c86c1711f4ecccc73", + "30b761ff0507fb35df2433bab052ec79f84850bff373bb5afda4d06182580944", + "a16c2f9baea2f99beb2db457784a4d2c7821e6847f2c84793bad29df6966e099", + "6609908f003607f83e12e9ba2bdb144853db84488843cd746beb22de8c1acafa", + "4c041802503e5f73824a514d958ae3a8ae5d64a6afd56ab14bbe87933423052a", + "ace04ea2245b772e5f3966700c7081618bb795b24e15bb664d7c0e821cfe7285", + "ace04ea2245b772e5f3966700c7081618bb795b24e15bb664d7c0e821cfe7285" + ] + }, + { + "trial": 4, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3003heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 6 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_009", + "intent": "click 'First Name'", + "reason": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search orAddPatient", + "Who", + "Title:", + "Unassigned", + "Name:", + "First Name", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or namepatier", + "Birth Name:", + "Birth First Name", + "Middle Ne", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ActivateWindows", + "Go toSettings toactivateWindows.", + "ontact", + "8", + "O Type here to search", + "\u8425", + "4:07 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "halted_before_effect", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 55.55, + "first_frame_sha256": "fe971029859dabd4854a7ce6d40e573973f47090bffc045b5706c164fe4c9528", + "frame_sha256": [ + "b905b05722a1bd0e49277fa028852ad9c5ac98ec9d575c5d793747945ecbe618", + "fe971029859dabd4854a7ce6d40e573973f47090bffc045b5706c164fe4c9528", + "f17f9cca6136cf17ff15a78ff2af28666ca5ae90a8239ebadab9555110377c6b", + "b905b05722a1bd0e49277fa028852ad9c5ac98ec9d575c5d793747945ecbe618", + "c34dc963d1df28155fddd859de2e2057fcce882140abe1bc8109bf0dd9a01361", + "f17f9cca6136cf17ff15a78ff2af28666ca5ae90a8239ebadab9555110377c6b", + "d833780aeb304a9b76f4d69aa893ddfb9d277f915c1974719c0969f8a7ead9ff", + "c34dc963d1df28155fddd859de2e2057fcce882140abe1bc8109bf0dd9a01361", + "621a19143a228a6a82b287f2cd7e76bce200cbd2ac5ffad755d7c958250ab988", + "fdfa92e7665490a5c7ef89a49bf9edb1fbbdae74bf38ac8cf9978c04d8b9ae6a", + "d1b75ffff8b383965acf3d43baf47955993c245d0c6224825fa1b151e5bfc08f", + "af520f4daf28fb779f50c5a77d880c1c0fe7a0f24c964f19e5181f8896a02651", + "db93263252baadb502d3d2a3c0bd156f158f031ff0d0a33462a44ff35cf4fc53", + "d1b75ffff8b383965acf3d43baf47955993c245d0c6224825fa1b151e5bfc08f", + "21ed5cf2c0701420f280edbf9e0bba443022b2403392460dd0f9b9384aec449f", + "b2f18b930589913d47e70f61d40778cecb6ee570fe4b733dad35afd3a299a5ad", + "42f3ea7cbc5f8b8959efaab42b24721fb47b313b6fb5a40cc9916a85ddabee8b", + "d7bb62debe1eafc8853307d0941e3a357c1e06c0dfe90048de903cbc858ed71b", + "cc43aedeaf4110870fc94d1630aa244df7139f61e35c0c1288107f4a1b0fa944", + "cc43aedeaf4110870fc94d1630aa244df7139f61e35c0c1288107f4a1b0fa944" + ] + }, + { + "trial": 5, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3004heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 7 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 454, + 349 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_011' (click recorded visual target): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Preferred Name: Patient preferred name or namepatier' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_011", + "intent": "click recorded visual target", + "reason": "Identity check failed for step 'step_011' (click recorded visual target): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Preferred Name: Patient preferred name or namepatier' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "Calendar", + "X", + "Message Center", + "Search orAdd Patient", + "Search or Add Patient", + "Who", + "Title:", + "Unassigned", + "Name:", + "Oaun3004heal", + "\u2460", + "Middle Ne", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or namepatier", + "Birth Name:", + "Birth First Name", + "Middle N\u03b5", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ActivateWindows", + "Go toSettings toactivateWindows.", + "ontact", + "8", + "O Type here to search", + "4:08 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'", + "click 'First Name'", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "halted_before_effect", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 55.8, + "first_frame_sha256": "4dbf59452ed73dc99aa679332ad28fed6ab8724e3b16580685d2fb278a1413db", + "frame_sha256": [ + "c21c07c89b0c0587cc4b961d4eb61f73af9632d4b3dfcc5a1d32f7bf7acc9381", + "4dbf59452ed73dc99aa679332ad28fed6ab8724e3b16580685d2fb278a1413db", + "eaa5e1b4622fda95a83df37f3e98a82bc59f5a47865e50d7be6606e368607e60", + "c21c07c89b0c0587cc4b961d4eb61f73af9632d4b3dfcc5a1d32f7bf7acc9381", + "93b47267eb9dd5c53b33cc4f845a705c15bc180c05fba6e9521c7401c47abefa", + "eaa5e1b4622fda95a83df37f3e98a82bc59f5a47865e50d7be6606e368607e60", + "e1fa4345b487285034b593a37d23a7a5d40e95052ec16f210e0c86a5b06ced1b", + "93b47267eb9dd5c53b33cc4f845a705c15bc180c05fba6e9521c7401c47abefa", + "4ea3bfaa1554513f6777e984980bd5a3609f9bad606b94827eb97efe8aef88a4", + "e1fa4345b487285034b593a37d23a7a5d40e95052ec16f210e0c86a5b06ced1b", + "a0377b3e4392ff84fc2a8092250aa0aa070dc8b07d9057b86b04bab1834a091e", + "d784bac3131088acaef349b2e6c945e8aa8e113e60eba3238247485cde2c006b", + "fafc3b4b577e5fda2de07b8fe3c01f1447c3882c1021c7167bd1c37467c6362e", + "bd91f1b593c02e2d9385644c42b647ac3f2cd427e4699616c39757df2d55a228", + "a6d9541463bd28e63d91595673925cf27a99ab3a72e10dab404dd7a8afcb5898", + "f1e2f6fa66b3a7e0044d43e18c7039143fdc3962527becea749bdd310a5c7abc", + "9de0c1e82ec6242635b2766a6d4534208f0dcf7d2940f7337cf7caff8f4cae72", + "e715a3e1e27220ea18e68a723925a59bf70764cd67fbc8d95568efc8e5dd852c", + "8227c1086d79181978e619bc60a4e73c3357d414bb2d71cd12eff873c22d7f05", + "f952092235c3fef281fba0a4288755300b0c6c56e0f474958624ef3938430d57", + "774ef1eac39750d343d5586448f3dc721599942efff47a621b7c81facb4fa363", + "8227c1086d79181978e619bc60a4e73c3357d414bb2d71cd12eff873c22d7f05", + "454664df26fecce5ab7a6fed231f0bccf351727462dc351feee0f3e062784940", + "454664df26fecce5ab7a6fed231f0bccf351727462dc351feee0f3e062784940", + "0ed7d930631b0d2c807c0f598c249e8f28226fa073355cca9445f454528fb53b" + ] + }, + { + "trial": 6, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3005heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 6 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_009", + "intent": "click 'First Name'", + "reason": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "X", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "Calendar", + "Message Center", + "Search orAdd Patient", + "Search orAddPatient", + "Who", + "Title:", + "Unassigned", + "Name:", + "First Name", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or namepatier", + "Birth Name:", + "Birth First Name", + "Middle Ne", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ActivateWindows", + "Go toSettings toactivateWindows.", + "ontact", + "8", + "O Type here to search", + "\u8425", + "4:10 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "halted_before_effect", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 51.78, + "first_frame_sha256": "28cda8b10c780e8ed1250bfdf0e5ae8313631081fded6762da3d8ccb36ff264e", + "frame_sha256": [ + "25fdbf948128a68301f2883c1a4da88c6e7d0d8279acdd3a60fc720cd064804d", + "28cda8b10c780e8ed1250bfdf0e5ae8313631081fded6762da3d8ccb36ff264e", + "30080c7491b391e890b1603c69d0f7288b12e9bdcb9f490e3e42a56dbedfd314", + "25fdbf948128a68301f2883c1a4da88c6e7d0d8279acdd3a60fc720cd064804d", + "aa1823195ca32c842775bf3cb3a45e9cec2d6c42ab76ef1d3217a85836671af2", + "30080c7491b391e890b1603c69d0f7288b12e9bdcb9f490e3e42a56dbedfd314", + "3571766eb9e623fff826597dd516357070c923ef41d1faa829ec73f3d0ae4dfc", + "aa1823195ca32c842775bf3cb3a45e9cec2d6c42ab76ef1d3217a85836671af2", + "dbc34bbed40fc999dd908c760d8e31e0653be79ffb8f48e95e8eec11eb425afe", + "3571766eb9e623fff826597dd516357070c923ef41d1faa829ec73f3d0ae4dfc", + "3d2a6cc8ba502804edd6f68ead410e683516d390a61af1533a943dae7c7fd831", + "5be78a771122241ed9718c6401014243fddcedbfc6407bfa732ec5cbf45cf2f9", + "f55054591cb0ccf9b5c45dc744aef752d5f5a62b22bfe2813ad26aac5930543d", + "3d2a6cc8ba502804edd6f68ead410e683516d390a61af1533a943dae7c7fd831", + "434c394b30b3a6ee65062e5385c1675a8d5c4149018e631f2d8fd4bac0be1ea9", + "bae064ec893c40fcea27dd4bfd8cf118d36e5ea1b9b2cac7080c721ed25cc1f8", + "3174ec0a7fe2e326618fce5e3f6a3f8b954f6ccb520f01351d399e3d0d821866", + "3cfd6a0a6a9bef33f5fa05bf37ffb71a0f383dc412b92537a7ec4637bd0cc509", + "1719600aba62218a4dc4eeb5fc7b7cbdae370862b16bd54459b0296891917b26", + "1719600aba62218a4dc4eeb5fc7b7cbdae370862b16bd54459b0296891917b26" + ] + }, + { + "trial": 7, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3006heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.993, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3006heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 92.67, + "first_frame_sha256": "4f8afc01f8efef99779e323d1834651033283a3225805ce320341d542b901141", + "frame_sha256": [ + "91aee97fc6f6422d2c2d91ca0347f510b8f3dedeae652a5f4225a92ef5e47b1e", + "4f8afc01f8efef99779e323d1834651033283a3225805ce320341d542b901141", + "f57f5186883a156aa3d1c43e378dcbe5fcb96dc31f380edf284da10ed4e2d321", + "91aee97fc6f6422d2c2d91ca0347f510b8f3dedeae652a5f4225a92ef5e47b1e", + "72b6dc2a068731b8c4993712024d667984dbcee857ed087aeadc2f89de9e006b", + "f57f5186883a156aa3d1c43e378dcbe5fcb96dc31f380edf284da10ed4e2d321", + "f41b1cfd817e68a5f8799d330a6c7f715cae91912b9f6c624fd6328436dca46a", + "72b6dc2a068731b8c4993712024d667984dbcee857ed087aeadc2f89de9e006b", + "51088df886236b699233bf54660faafc1ac87229d7abb57989f34196bfc211f4", + "8f6d2629a73d3b122998695a9cb50cf6e0cc4709d5021b57c2306ab52a05e318", + "6c3fae709fd10c9aef840e1fd3ee45817593b7cd41a17d280f06c8408f4026c1", + "78f6b981ab21dc225adf8f0a75bec0c4920238c2ea33d9e1be02c8640e124413", + "4457d6e7a5a1d96f389024eb3ff7adf35b92dd355c86d79b9cfc351ef53443dc", + "6c3fae709fd10c9aef840e1fd3ee45817593b7cd41a17d280f06c8408f4026c1", + "8e7c28ee1eeaaa760bb5563dc4a6c83ac01eccf2e9ee6dbfc704d43fdcd6ae77", + "1cc6a4d0eaaa37df34b175d46451fa5560c20a1376b5d8ef72542f74c299eeef", + "e25e6c8be4e8ad176c34c7dfc0350fc7466a8873b7affbd8063dde22d4e2786e", + "eae27588bf0c8a36e13e6a8f47583b1c9e3a0d1c75cadad90e0d3e19e1bb451a", + "94becda2aa47ec937d4523b27ba5a4030263174b0ea4cc3335c29821ee1e84f7", + "843f5d17532b4b577577e3c458ef63241160bff26001b1036963d25f1c3e8270", + "a31b7df9a889162a570ba162ab79459e4da55aada2ef915f93d0d85ec95b4f60", + "94becda2aa47ec937d4523b27ba5a4030263174b0ea4cc3335c29821ee1e84f7", + "3a0134d84c5742e37e7b618343298d9662efc64c1da2556b80f1a5748747ade3", + "a31b7df9a889162a570ba162ab79459e4da55aada2ef915f93d0d85ec95b4f60", + "a4d90d36a45df96cfbad3375c69595f6284d8f936e7afa41372d4e814f6fbef1", + "43830b0cb05c54e95a13cdaaffb9265f6514c7aa53b436519c0c2f7c0a10eb9d", + "16ef4591f949fd311acb2cc935c5f6dcd26402b2da6a1b5d338a4a6310877b11", + "a4d90d36a45df96cfbad3375c69595f6284d8f936e7afa41372d4e814f6fbef1", + "4d88d33bc7d9688d99bad4bf52068efae23ce18a131821ebd7ae25a6d48db387", + "16ef4591f949fd311acb2cc935c5f6dcd26402b2da6a1b5d338a4a6310877b11", + "80b1d42715a562a3525b08b95d8d8543b1ee5bd0f4737b1088febaa2dd000f49", + "4d88d33bc7d9688d99bad4bf52068efae23ce18a131821ebd7ae25a6d48db387", + "d8dde810d393dbfcb067e49d1460db336a7c07308d28355e4249d3a359cf204e", + "80b1d42715a562a3525b08b95d8d8543b1ee5bd0f4737b1088febaa2dd000f49", + "0feb4297acf6e92e6fd28d6edc2899372a16961dce905d79840e70db13672113", + "d8dde810d393dbfcb067e49d1460db336a7c07308d28355e4249d3a359cf204e", + "d41659300898a60cd1c905ec8c7517f88fa8407b8bd3361a52f942b9cfb4c762", + "0feb4297acf6e92e6fd28d6edc2899372a16961dce905d79840e70db13672113", + "4f4173421c5cbdd245f1b89ece5d8c25818003bdbada700bc172835b183503fd", + "379074669e896fe4fb4614b1917094531503e1f1fd17adc7044a85f83801b83c", + "b338177a2bdd67f0e5dceeea1405ab1370a8404e23b42a1c2544d2eface68565", + "214dfa8343dfdc1f36332b60e752f785555ec07b87935c1bd0f80b5826bc484b" + ] + }, + { + "trial": 8, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3007heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.989, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3007heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 101.17, + "first_frame_sha256": "32b8aea202009f66b836643e629418b0767406636ab7bf26239aa2544123ca45", + "frame_sha256": [ + "4d3521fe61d59d8a9fa2edeca24d928655d6f593b32a13c8c6173b6d9e53fe76", + "32b8aea202009f66b836643e629418b0767406636ab7bf26239aa2544123ca45", + "34a7bae747771eed385bc760e5dc206efe5438bbadf45312ee78ef07cdc14354", + "4d3521fe61d59d8a9fa2edeca24d928655d6f593b32a13c8c6173b6d9e53fe76", + "991df1d435c9b1d210f7d2ccfd7c91e1798249be5313b3c9a5eb5d8237984584", + "34a7bae747771eed385bc760e5dc206efe5438bbadf45312ee78ef07cdc14354", + "0ccd826c4a89f5c55c607b656eab56f3700da25ada614f9fb7a75f0945c5d4be", + "991df1d435c9b1d210f7d2ccfd7c91e1798249be5313b3c9a5eb5d8237984584", + "2dc8971a5c7cd49cb3b9d0738720c4a09b6eca50f560e6977bcd391fe427f8af", + "a4d35398577467f4f0611287f80f1a5aa69b09eb027cd009963b1a567f273210", + "31bf4a7edbc85eb944b638e26e0f54ba05faabeb681b695ea84ca77426605be4", + "c3fcdeca5e156f1ab3cd188470a3aa2e815aff1f0969afd96d92e87318fb4707", + "168563a5892118eb1b4fbe3704330f5db03915ffaf3d2e8cd5eae560671a6acd", + "31bf4a7edbc85eb944b638e26e0f54ba05faabeb681b695ea84ca77426605be4", + "05c97dfde8ce6c3edd4116a3e421f593058cd78b606a650d5ecf0909528bc303", + "af3f3c4cdce755b0202f4ef977c0a5145737eae1fd931483f15ef10c17cc1589", + "699dae0576059bde466cfcca4504c51f045e37b13516d2203d6a236c3e3e3423", + "83133b303e6433d0a8a26c890a8b85a94222106f896c5e3e7dc5c211984a35bc", + "b9ae258559eaa6cae54238bb3b58fb5408d3df341c5f465dab1b7ac26da1f04c", + "cbac5e44f0a199551fe0bbf4d39e2abad18581a80d7c69f8c900cdaa032856ac", + "3a77b382fa69a073c6c5d4df741ed5710f9e46c31503e5901ec81905a6328555", + "b9ae258559eaa6cae54238bb3b58fb5408d3df341c5f465dab1b7ac26da1f04c", + "f29b9a1701d63f7075074c78f20911d3cbb11d61e0a3979f4f4452185382c865", + "3a77b382fa69a073c6c5d4df741ed5710f9e46c31503e5901ec81905a6328555", + "9a62690c813ebd4b0eec08e66449c35e3b865a0e866f366f83dcae1d97cafd60", + "fd27ea7dad58d3e2da63b9e0dac5f6678118b2dd01eed5c16d166a1b22538255", + "ccfa591336bca708bd6712bb8a4ca8c77e5ae18e7d2be69fb22466bf6c2035e2", + "9a62690c813ebd4b0eec08e66449c35e3b865a0e866f366f83dcae1d97cafd60", + "6e5fc3f52d96ec5a1177d4b5b3f001cc7bacd4175483ab32c7452a69c5e10bba", + "ccfa591336bca708bd6712bb8a4ca8c77e5ae18e7d2be69fb22466bf6c2035e2", + "3ba7b78ef195f3293db4c2369dd55bb6251e1029eaebcf3a535a8d560426cc7f", + "6e5fc3f52d96ec5a1177d4b5b3f001cc7bacd4175483ab32c7452a69c5e10bba", + "71957a84e4036c1dfedbe1bc33407dc05d8f0bc93e94f2ec4d0fc287e3877a4e", + "3ba7b78ef195f3293db4c2369dd55bb6251e1029eaebcf3a535a8d560426cc7f", + "42a33c9b2d97eaf976231f4238d3e6d4e56b1fec271d6de0cec3b114919d19d4", + "71957a84e4036c1dfedbe1bc33407dc05d8f0bc93e94f2ec4d0fc287e3877a4e", + "576bc2f16d169f659a6d753e374a6b083ffa9bdbbf533fea63c954173ff0d28b", + "42a33c9b2d97eaf976231f4238d3e6d4e56b1fec271d6de0cec3b114919d19d4", + "d730026913cac72c55fc2f073fb6195c427ff034e0fc4dc3e4b4db4ec996b8ef", + "e9e92dc766d66c6342ae56d9f3cf2dae3764afffcab49915e6c4a59075b943b4", + "28bb3ccee4e6a59f77bb766f74ccb642c7e5a61c07eaa17c93b9ec339e519cf5", + "e8ab3150069f5b736b1d767050fa7a3b26b063139c200eb95e8a3827336c33f1" + ] + }, + { + "trial": 9, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3008heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 261 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3008heal", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 94.33, + "first_frame_sha256": "19a4020444b486483da3901f0e8519f34bf61b63d2cae546a7ecfcaf359c1a54", + "frame_sha256": [ + "c6d4dae09477dc3c1b2dda8cf7f858cacbb170835d23f9220149308d04e3f267", + "19a4020444b486483da3901f0e8519f34bf61b63d2cae546a7ecfcaf359c1a54", + "2509eee68d88c3dd619fcefdfbcac642b6856bfcdb5ddc481868ad69a96de18f", + "c6d4dae09477dc3c1b2dda8cf7f858cacbb170835d23f9220149308d04e3f267", + "349833914e269df189d8de9e3d0bb38b09e9a9c7b563f296a1c8436495df4435", + "2509eee68d88c3dd619fcefdfbcac642b6856bfcdb5ddc481868ad69a96de18f", + "6375fda0fcf46becea336288dd6afdf100ed76e40e0c2342729462e898e50881", + "349833914e269df189d8de9e3d0bb38b09e9a9c7b563f296a1c8436495df4435", + "d13f00a48d101741a7328c96a27016ebef672160818966ca1c778613832b5f57", + "6375fda0fcf46becea336288dd6afdf100ed76e40e0c2342729462e898e50881", + "c599c4092c43a03c78c60c750affabfa27de7005e78c3282ad0222546d792a2e", + "b13f597d7287e7faf4897734da06c0bc712c8b64382016f1fbc6a1cfddf0004c", + "58acbd38d30fa6606efcf7986c2a1950ce23d276e39d5ea967b231560f8ccb84", + "c599c4092c43a03c78c60c750affabfa27de7005e78c3282ad0222546d792a2e", + "1a4ec057aed3e88934f40aa598d4e11e9b55a326eb907c8e4dd444fda6032355", + "71e65d1dc5981c40f0c6b87fbedab0c7747b129587c7970fa6f439d478296a31", + "91db549735af61b93835e65fb7cb72307687064b17c54cc17ba108ada9b13db8", + "be69186573622dae2408cb38199c4e90e0c56c8742f35037f830214ec5332e40", + "366fea65cbae02a4f47db8d8781cac115c60144975d81e85a1927efde1e4da92", + "7e611823449cde5a35ccb2054ebdbb6b1dca27192cc8a73434db4977692f2a22", + "293d22b314714c9f9c0793fbc348bdd99dac5459aa47724178857f7c1b63ff79", + "366fea65cbae02a4f47db8d8781cac115c60144975d81e85a1927efde1e4da92", + "f750ad366386554fd0079a6aa5412f82ef01f95bde5c343838f855747b7bb9fd", + "293d22b314714c9f9c0793fbc348bdd99dac5459aa47724178857f7c1b63ff79", + "64eb89c92db09a57ab66070429c1287c220c684002328b13483350a57478a59d", + "ed892db69cb01aeedded28e313fcd5d309ad01ceed3bd768b957c5a46659f1cb", + "ba54d62123c31507758cc0dafeb9ab2e2ae748c9424773978e61cf82f53001c1", + "ba54d62123c31507758cc0dafeb9ab2e2ae748c9424773978e61cf82f53001c1", + "0edbdb5d29f590a6c59fd313213c4d8351bd4d377bce8eea6afc1497e8dffe99", + "63a0bc1988d48e44feb8b4db05a51aa4cfb45b4f5854738a346030cd1da088c2", + "6bacf31b22d3a2fcdf19570b93d640d28b293da6112ef071e7465d8561d3a41b", + "0edbdb5d29f590a6c59fd313213c4d8351bd4d377bce8eea6afc1497e8dffe99", + "b95db5efabef25aea4e1f90e0efa79691189b3fc0b206e590faea94cfd7d2987", + "6bacf31b22d3a2fcdf19570b93d640d28b293da6112ef071e7465d8561d3a41b", + "4681d2319c69510121de29bbbc0307eaf6c374bc2d67502079513cf0eace01ce", + "b95db5efabef25aea4e1f90e0efa79691189b3fc0b206e590faea94cfd7d2987", + "81c9403734f61b66093757400b608f7aa18e633e47fe9468da5bd8a406dca43b", + "4681d2319c69510121de29bbbc0307eaf6c374bc2d67502079513cf0eace01ce", + "8b75b5949ff1c8a6ed1ccc4cd14afff7321d31580cf24bf1b871b26e1c42b156", + "2ddafd3db25ed866e8233497c9e127e38a063d33299e4cf30bae00b15982191a", + "a4ba82f60bca3fe019983ea2b9380155107856289f433ce60afa4cda2d770547", + "1e1705221806917d2a703091e36eb560c58b75581026c6df8643d917474253c0" + ] + }, + { + "trial": 10, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3009heal", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 6 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_009", + "intent": "click 'First Name'", + "reason": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted", + "outcome": "halt", + "observed_texts": [ + "OpenEMR", + "Calendar Finder Flow Recalls Messages Patient Fees Modules Procedures Admin Reports Miscellaneous Popups", + "Search by any demograpl", + "Calendar", + "X", + "Message Center", + "Search orAdd Patient", + "Search orAddPatient", + "Who", + "Title:", + "Unassigned", + "Name:", + "First Name", + "Middle N\u03b5", + "Avclab", + "Name Sufi", + "Preferred Name:", + "Patient preferred name or namepatier", + "Birth Name:", + "Birth First Name", + "Middle Ne", + "Birth Last Name", + "DOB:", + "Sex:", + "Gender Identity:", + "Sexual Orientation:", + "External ID:", + "S.S.:", + "License/ID:", + "Marital Status:", + "User Defined:", + "Billing Note:", + "Previous Names:", + "Add", + "ActivateWindows", + "Go toSettings toactivateWindows.", + "ontact", + "8", + "O Type here to search", + "4:17 AM", + "10/29/2026" + ], + "completed_intents": [ + "click 'ername'", + "type ", + "click recorded visual target", + "type ", + "click 'Login'", + "click recorded visual target", + "click 'New/Search'", + "click 'Last Name'", + "type 'Avclab'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "rows": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "halted_before_effect", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 57.73, + "first_frame_sha256": "7e7ac19890f418a06a0923e812302afd1792590cabbe0bc9f92aa921b293c4a4", + "frame_sha256": [ + "638bab7350ec3b1952945e11aaf76c22a914454eabe4928074f0147df3dbec5f", + "7e7ac19890f418a06a0923e812302afd1792590cabbe0bc9f92aa921b293c4a4", + "b5d21485260c9b3396ae0350ade4cedeb84eac5adf70377576cd88d9a61cd90d", + "638bab7350ec3b1952945e11aaf76c22a914454eabe4928074f0147df3dbec5f", + "73a24642ef46183bd17f9afd7246787254642e55df586924f16efc782e5b2a1a", + "b5d21485260c9b3396ae0350ade4cedeb84eac5adf70377576cd88d9a61cd90d", + "f5c1f57fb633fb7518fbe420e7ce7efc9db97ebf2ade9e6f8ad75ebae50d73d3", + "73a24642ef46183bd17f9afd7246787254642e55df586924f16efc782e5b2a1a", + "bf2eeeff5a700ab5a49e7c9d295e6deecfa4678374a4bd5e145422ddf5d13270", + "f5c1f57fb633fb7518fbe420e7ce7efc9db97ebf2ade9e6f8ad75ebae50d73d3", + "a0a75fa41249cd024f454046a4098975108b62589d65bb3607915f8d61e6d34e", + "93d7608f56df35ec05643ea02ecf809c5d8dfc1700417a130df1b36eca3efedb", + "54787b76d6e2741da57f726c6f14ebe4da6cf5c7277be1a6a2af375eaee65fb1", + "4f633529ce781d4ea3496382742f9389a645c56bdc31b414f36c660007866c71", + "7e3cdfaca132e88aac5559b5d49db6a4affcf6669cfc3ef94f996aefc23dec27", + "6e8cfad67a59658930510d055ce79a6e4db7c650572a0e26236408d46a1f636b", + "328272efad5dcb78d072d99cd2dffa6259380b0b6fa91291bc316abe944a3c73", + "94c7034880e52635e1866fc4102be74ac9135d51b125e9bb44b64c51f1d81a23", + "5fa7143f6aef5435313ecc84b14c77ff917554e0d160d13cd52303f76905ecc1", + "d15e053b7526190031bc131bc30f9abc1405908c27e8e78ba29395148c6d4812", + "d15e053b7526190031bc131bc30f9abc1405908c27e8e78ba29395148c6d4812" + ] + }, + { + "trial": 11, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3010bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.988, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 260 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3010bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 100.83, + "first_frame_sha256": "2ffdfe3b20e89465f73786b17fff77def1b119237cecd429e072675c8ecda04c", + "frame_sha256": [ + "929d877e695e463482bf5bd234a87fea2b16c67e1cdfb625e9ccba75a0176604", + "2ffdfe3b20e89465f73786b17fff77def1b119237cecd429e072675c8ecda04c", + "6db28f36bae04caab0453b0d0ca0e528aa2c25836abfc048699ed13b3de86c4c", + "929d877e695e463482bf5bd234a87fea2b16c67e1cdfb625e9ccba75a0176604", + "8157fa9ddbb85974ff40c64c01eff7f88a39d2234025e57773ce8ae16a4f7d2d", + "6db28f36bae04caab0453b0d0ca0e528aa2c25836abfc048699ed13b3de86c4c", + "d78996c09fec788532e291d2fa2b43765ebeb33b3a42de2ff8e515c3132816d9", + "8157fa9ddbb85974ff40c64c01eff7f88a39d2234025e57773ce8ae16a4f7d2d", + "dfa46d5da41df44075d43e8b461c83051aedb40ee09145804a42b2f3d5afce67", + "d78996c09fec788532e291d2fa2b43765ebeb33b3a42de2ff8e515c3132816d9", + "582ab47023d4a0c243f3b8a72684795b83458811e60a779421a224e42e99f972", + "40b3ba6371bf1ea151244841930e361191bcee343ae7beab62e14365c213db5c", + "b3df7eaae0018df48ca0d15ec84e71d945dd806b8ffa07c5f9162c2c9e4f9319", + "582ab47023d4a0c243f3b8a72684795b83458811e60a779421a224e42e99f972", + "f99f09b6f30f1c1f5a6f573752f6e614c7cd3919483c4d2041217ead984ccad9", + "a3e48f46f66b1bfe3755988035a9886dc256d5c0cb9bec6a0090173b2ccec906", + "0bd926d6a8616872a5be28c4c1cb449539fd80b5672c537d4aebab7964803795", + "953ad3308d32186aa2b04352c3c6b8040782504ccf660d0033c35e0472cd2bf1", + "6f729402e771648aef5813b41db6951eefe7c4fe9edc7f9ca19e95d812157090", + "572c5971d1dd6b603d46d624628f247da50638843cd531a25862c78e10cbeb47", + "d49e9e4688b3de984a7aad2f41f3f6bda14a9cbc2da3bdc5d1cfc6a5231263d5", + "6f729402e771648aef5813b41db6951eefe7c4fe9edc7f9ca19e95d812157090", + "b06c7430c580490a3726ad55d2e749730eff8dcaed00cc2189607b418edd815f", + "d49e9e4688b3de984a7aad2f41f3f6bda14a9cbc2da3bdc5d1cfc6a5231263d5", + "4608f6462193ccabb506d4d22607e20dfb7e39d822d1a3a1c53545c4cf7aef3c", + "d43b869b8be5eff489489690d5f7fadb018faa0001d798bfc33eeb1a7a12dcf6", + "3f7c580fa2a17e6c51969f08112cfffbfb3d654e7c6f54d197e3334b9eeeff43", + "4608f6462193ccabb506d4d22607e20dfb7e39d822d1a3a1c53545c4cf7aef3c", + "63cdc8b8fdf03324cfba5f65f552e64f8ffcd2f8ec5ed04c87b2e801246e9727", + "3f7c580fa2a17e6c51969f08112cfffbfb3d654e7c6f54d197e3334b9eeeff43", + "6761960a16ee043135df3a9c7e7c00df2b5b7dffcfc43f131ad1a9ec4a6d197c", + "63cdc8b8fdf03324cfba5f65f552e64f8ffcd2f8ec5ed04c87b2e801246e9727", + "2a06efe8d0f839aa3a67dc6fc19f2c246b119a7aba79c8fbdd93085912c81ad4", + "6761960a16ee043135df3a9c7e7c00df2b5b7dffcfc43f131ad1a9ec4a6d197c", + "46248def00cc4688504f0784e414cd5eea0cbe315f89058a5ed1ab2ac3c72fd5", + "2a06efe8d0f839aa3a67dc6fc19f2c246b119a7aba79c8fbdd93085912c81ad4", + "45411386a1eb7dee06bf155a70cfa7238bd78d74aea6a06080a30018c0c78efa", + "46248def00cc4688504f0784e414cd5eea0cbe315f89058a5ed1ab2ac3c72fd5", + "e2b035c2a561379d244b1b9883d64cf940a4329e29d4e4353245752642e56b33", + "800f6829c16537600bfbc38c66574e94b9f3950772809c3cd28e0742ac4c6ebf", + "1a90abcfad97e3984fe5d1735ee31a5727e823cbeed022faaf66ec226c8fbfad", + "251af80edf78f37b9a45ee3998d047b3038235b8850e0aa19b6b080ec3d22981" + ], + "first_frame": { + "file": "emr_bpp16_first.png", + "sha256": "2ffdfe3b20e89465f73786b17fff77def1b119237cecd429e072675c8ecda04c", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 12, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3011bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.989, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 260 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3011bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 87.25, + "first_frame_sha256": "9bfe1420c9617b5395f22f7f91112f4642bdfa48acc5968c089169a7470926c5", + "frame_sha256": [ + "891aadd85230d14360db43d1eff7b14f24da21cf72f18cf9175ef4039983c539", + "9bfe1420c9617b5395f22f7f91112f4642bdfa48acc5968c089169a7470926c5", + "19ad56b11bab1f436a45eba64977648933c8ef96f556cdccccd761d3341e9e10", + "891aadd85230d14360db43d1eff7b14f24da21cf72f18cf9175ef4039983c539", + "eb172dde398573e9790a683d9121c6ab6f049a10778ec538e295d753f4e4ee29", + "19ad56b11bab1f436a45eba64977648933c8ef96f556cdccccd761d3341e9e10", + "27402e209a66a66328e7623f6a661f67cd2b90ab859c88b37d1fa684487eb988", + "eb172dde398573e9790a683d9121c6ab6f049a10778ec538e295d753f4e4ee29", + "e11ec79013ce38096436b2dc916ee51411f323b4cf3114c5fc6d54950ffa71d8", + "27402e209a66a66328e7623f6a661f67cd2b90ab859c88b37d1fa684487eb988", + "374d40cd0fc929500313380011569aaec15f0661aad520f89c2765a57089ea0f", + "e4107a0c5bedcf93f9e34ad88f7f4a603831ee1c2b41d9afdbe78662dc7f7513", + "bcb5c644933fb09761083f7d1fa7bcfa7308d6772a50238c2841ee9885d77ffd", + "374d40cd0fc929500313380011569aaec15f0661aad520f89c2765a57089ea0f", + "4093baf3214ae110368cfa4fdb322369e77f09dcb0693262746e12f9813193f1", + "bcb5c644933fb09761083f7d1fa7bcfa7308d6772a50238c2841ee9885d77ffd", + "448251b9621536abdc888c858f9fee3413876fe924badaf106b890cccb911848", + "58c6cdcefa431d1e1aa7e05a0ccfc051ca39ecbe954bdb2d2dc3451f82854b92", + "a90e9d833e2246ac6068ac523e5e1e469c0ef053c127d23d69d1746ee58fc228", + "b5820fd29de2ded28baad891957f72acafdbf2eb688a9906eb506747029a8b6c", + "e2fec035726554ffd6a6820f6b3b43e092835524985b5b0fb9a9622ced31f781", + "4882e36173ba72958fcf348e58a93030d3299294318adf1600e5002d4cb97c63", + "c19291c7cad20ac44c608bc043c34dfaef4cf77d065e7cf256237aad5e1deff4", + "e2fec035726554ffd6a6820f6b3b43e092835524985b5b0fb9a9622ced31f781", + "8cd7634d8f2cb8009bd9d2c6adf79608efd833078875db37f1b15ddbb140f06c", + "6ce6b96bcabbe1b407009453390a865336aade92d2d57daec890db7ed8ee767e", + "64ce0f2a90d441d35ba17f61f4c154b394533e32af3fa80c25e033f74ec75477", + "64ce0f2a90d441d35ba17f61f4c154b394533e32af3fa80c25e033f74ec75477", + "d9efb2e3e8142e27bce6ba204f1fa8d009a2bbee592e77932d073fb25b933fad", + "ea933f37f52a481a0e45e724854d9e98daf2edd283938267d226c0ec2ea5dd1f", + "4ee6159ad24e1fe8ebcc3fb7827f8871f7b5c66e000a2ebdcde788fbd6df6dc7", + "d9efb2e3e8142e27bce6ba204f1fa8d009a2bbee592e77932d073fb25b933fad", + "ecd5f569376b3a326069dbb866c0519ef8a2e842a4ea07125a5a3a584994a8fb", + "4ee6159ad24e1fe8ebcc3fb7827f8871f7b5c66e000a2ebdcde788fbd6df6dc7", + "cc7bdcabd2a33b2aeefb19523d5c947630c87537756676a5a87aa70e17ce2a2b", + "ecd5f569376b3a326069dbb866c0519ef8a2e842a4ea07125a5a3a584994a8fb", + "283395972dd27b88a59c7e92163284b94766fa3933af529fcc4beb583594e5c8", + "cc7bdcabd2a33b2aeefb19523d5c947630c87537756676a5a87aa70e17ce2a2b", + "d0d852d753db4e02642cfb206d70f3223b3892f6f1ae563d85340e464c882e77", + "cc1019a7e9aed4ad90281749ba444a2aa6f140fab0dfd78b9e055f87201f6000", + "53135622821b20239f910c182d51f1bdf681f90aec607de84c3c3d44d9441717", + "7a24aea9c8e728c22fb442a50dda437e2f0605dfedb36663d6de61806179fede" + ] + }, + { + "trial": 13, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3012bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.989, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 260 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3012bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 81.02, + "first_frame_sha256": "baa3347fc450a49855e6f924aebb2c8a895c80c14bf223376fcfb98c688676d0", + "frame_sha256": [ + "6d0f9a957e6898e290ea6e41ea461c23c05fe04ed7c3607aa5341018a8633828", + "baa3347fc450a49855e6f924aebb2c8a895c80c14bf223376fcfb98c688676d0", + "bc1ebd1e7e5b3e0c270e9e95098f77f028f2b52bc0bf48cbf909dd7454bd1534", + "6d0f9a957e6898e290ea6e41ea461c23c05fe04ed7c3607aa5341018a8633828", + "193be49597200c5f1922e1cb2bab319f7e5672f4ced08d81449933f751be6937", + "4bec76d55ff2e41787bece646a99f473695fc5e958caa64c57a3596a6a77e4fd", + "1eff87eaddc48d6c5040a79085cf69fb465458c4b54f0b3c554bfcd7e89a0fd1", + "193be49597200c5f1922e1cb2bab319f7e5672f4ced08d81449933f751be6937", + "5c649bbf5c3af84b08dcf7d2924a1f7f01609da939789ac38df6e5a5f821b566", + "016ee30257a1f15c55bcfae5b3868200cb4c84056af11463eb886481f9c82d3e", + "afff0b7a26d73a3429030edf6e96b6c02aa077fdb0fcc838c3f0ea44b7f77fc9", + "89e99907911bdeb918fb1b51d9ff30da818c251186ffd0f430fef9a7e9465163", + "02125fa0126769fcd55de6c26e62bbf3efafcbc2fd5511ff15f4ba4473040948", + "afff0b7a26d73a3429030edf6e96b6c02aa077fdb0fcc838c3f0ea44b7f77fc9", + "15de16efd23d9e5f1979a61059ce1dd91fe36c65d0567980981ec4ed0315586b", + "02125fa0126769fcd55de6c26e62bbf3efafcbc2fd5511ff15f4ba4473040948", + "0579126d1ad2c285c2220405b5f9b4dd8ddddcc8014d2e561e35c7decfe15d9c", + "96cd3c0af56e2c3b2890f3adfdf57cff002f1d6aad4c1d69c3c27b570da9d250", + "c7515c5681278dd02a702986adef3b92c09cdd40bfcaa84538fd0299d40f59ca", + "8422c22781c043031c9c7937740e5ef353eed38887be5f174cc54dde45019a63", + "e8d5f428c7c97f5ed20ed138ab910785aa36a33dcac7dfd88a3bbd879b776ca1", + "c3f2d8419768fc1c904a49abc5bd42e7d47efafaa231d2d99c6eed4e797b4ace", + "b90006401a8468eadfd81e5b24cd96084ef4c71bf3bb27c05d34a2e932143da5", + "e8d5f428c7c97f5ed20ed138ab910785aa36a33dcac7dfd88a3bbd879b776ca1", + "6e771380cd46d3ed85bc73307383c92f7fd1341a999ab3cbc00565f0ecd865a5", + "5f3b9c838900c2f35a5d01f651daf7ea7129b310b0a2ea8267e543431699a346", + "74bcac136aeeca81c28450692f5bcb242fa763789bf5a558b5dcaa842a5df7ca", + "74bcac136aeeca81c28450692f5bcb242fa763789bf5a558b5dcaa842a5df7ca", + "f9e3db7a00100d385262262d001522027e8091655256b92dac030fc4ea724032", + "9adb4fbb016dde4ce4209e4b82488ec7f95bdb53dfb11e8116dcfb962dd8b08e", + "133d8fde4d6dfe1314a63eb119b73861dbdbbb6345d89a51397552525bee6efb", + "f9e3db7a00100d385262262d001522027e8091655256b92dac030fc4ea724032", + "f6cf8cfc2808c7fe1113db40a7006640d744b6cd4795c4fcd166117b5259fc8e", + "133d8fde4d6dfe1314a63eb119b73861dbdbbb6345d89a51397552525bee6efb", + "360b80a2687adfd3e9b351534c7166b8dd51771439bc2348591c04d01af0a36f", + "f6cf8cfc2808c7fe1113db40a7006640d744b6cd4795c4fcd166117b5259fc8e", + "46645ea7b5052f3fd49b9bc1a011054763c30ed4648e2fd7cb6e06dc2cc096ac", + "360b80a2687adfd3e9b351534c7166b8dd51771439bc2348591c04d01af0a36f", + "d7b3b1eb024b9489317e2c6121f0ba8d3de63a05429a4133537cc1d8e88b3941", + "a2ce6eb3e4b5a2364c101516a53eed94c7d96c34c669720214136a79e0b81e57", + "69fc219823a8ea7a1df2fe404350c7b760cab43aca40e82c6562dfaf489af1ef", + "d85dcf96f0a34fab4403d9b9ecc2942aef54a71049042a945a6aa54bd51a0684" + ] + }, + { + "trial": 14, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3013bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.997, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.988, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 260 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3013bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 96.01, + "first_frame_sha256": "8e36bef9dfb48d78e56d0e6ff9b38340d325010e01b26e9f587f3c25fe8addc7", + "frame_sha256": [ + "7d42c859677cbbc047acaacabb0b5f1cbae367850daaa52d4afeecfe2be1e0af", + "8e36bef9dfb48d78e56d0e6ff9b38340d325010e01b26e9f587f3c25fe8addc7", + "917b38665d60be0d22aaae02c872029e3062798152df219b92339c6096c40e6d", + "e40ffdcad627feb6833f434f0eda79561fb48781ed342ef3c706cc073726d0fc", + "73d4a532e0c619db77f2c34beb982f42693ee03e16c2f4b913a35c5ad8cc5f9f", + "917b38665d60be0d22aaae02c872029e3062798152df219b92339c6096c40e6d", + "c6e182ca2fed61ff89e3735ea0ff280d931af6d705ce254e9aa947f12c4f3e75", + "73d4a532e0c619db77f2c34beb982f42693ee03e16c2f4b913a35c5ad8cc5f9f", + "7d8c765cd1b35d7a277bbcfc47926125bb9d7f75b8404964298091028ec1c5dd", + "c6e182ca2fed61ff89e3735ea0ff280d931af6d705ce254e9aa947f12c4f3e75", + "9c4693c17fe564f3198b7c1338ecb5b3de3892d78c45b15a9cb7b90c65b9c536", + "913eebb57e94f5b724ce71d2021bfbbf184c299f2cc994e2dae4b7b6614d290c", + "dd9a73c761dffd289b2123d5cea0e347aa1454423c717fe532d86b97b2a9d0d0", + "9c4693c17fe564f3198b7c1338ecb5b3de3892d78c45b15a9cb7b90c65b9c536", + "e39a589548131f912101b4fa23ec55084e8ab985373b68a9b9c4504b22f48dcb", + "dd9a73c761dffd289b2123d5cea0e347aa1454423c717fe532d86b97b2a9d0d0", + "af9a993ca4dddf66d9962b09afcf677d6ef7b319fb83c413c5fe71bc744d4948", + "bf9a9e94979f54e9ba07162d0a2ac175f67cf04c39f389bac2b85dc5a0bfa536", + "970ea76add9df7ab8c16662143d1622cd6e1eec0da3f6594eb021a514b864cbd", + "361b322492ca5ac34bfbc07535fd041a8ac4f0eaf8da459cc783f13fb3e82c73", + "9d6bb691b3c87d4fae6d94e10ed6db4329edd988277ba60199cd98c078ce07fd", + "693cbaa35a289d723149ce4837a365bd2bd711a13bc9c7516bc6fddbc2f07e47", + "5c3a505cd7316913e362b682b912e291ebf198ab8ebaa7a07659d77de878d356", + "9d6bb691b3c87d4fae6d94e10ed6db4329edd988277ba60199cd98c078ce07fd", + "9b85c5cdf2d5770005740b2dad83382635959aba91df3cf4208f0ac4d4b01fca", + "d9278aa60ba3595d8a839457fefad6393f9b61c2ccdcc70f08b133d6971dbd22", + "df24a1e2e9f94af08b6ccf02b74250bd97cec54924c5514a66b588181d36b129", + "9b85c5cdf2d5770005740b2dad83382635959aba91df3cf4208f0ac4d4b01fca", + "db8cba004691cd99426ca2ed9b6bbbba5240c5e4a8e7f17f1f559cd57a0aa556", + "77b55cee3d5a6fe9b1f32d779dd18377d148589861450b6c84f999d0b62ba28c", + "30bc17a4279301f94459b3b8de4ef4b86334ebc833aba106d403aad5403ad4e2", + "db8cba004691cd99426ca2ed9b6bbbba5240c5e4a8e7f17f1f559cd57a0aa556", + "cdd959860c468cf2c14a027ee1691b48e97d3a283b292baf01cfaf76e8508457", + "30bc17a4279301f94459b3b8de4ef4b86334ebc833aba106d403aad5403ad4e2", + "918229909665c2f1971fc4ebb282b8ded7dd1824e90d5da54943210fdaaf0dad", + "cdd959860c468cf2c14a027ee1691b48e97d3a283b292baf01cfaf76e8508457", + "d7958547a7189bbd1a48a02bdb1ead83aac867ccce62998fa941e6f3f9faad00", + "918229909665c2f1971fc4ebb282b8ded7dd1824e90d5da54943210fdaaf0dad", + "0251422a8b63d4a6dc61d78c6e56a261a236d2b5f496bd73f794e78df077097d", + "730a250c23af938c39e7c3731c90d8afe95b891511789b55b162ba3027d36942", + "0b2bf01c417f830546a9c3565535162a2dd5762c8dd3d5998e6e5fa77b6c0f18", + "cceff5cbe838ee55acec73d653f6f43b6ea9b19e7db11c25c835451bcba6840f" + ] + }, + { + "trial": 15, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3014bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 12 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click 'ername'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 705, + 396 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 684, + 450 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click 'Login'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 640, + 558 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 361, + 43 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'New/Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 378, + 71 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_007", + "intent": "click 'Last Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 706, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_008", + "intent": "type 'Avclab'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_009", + "intent": "click 'First Name'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.996, + "point": [ + 445, + 311 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_010", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_011", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.988, + "point": [ + 429, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_012", + "intent": "type '2001-01-01'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_013", + "intent": "press escape", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_014", + "intent": "click 'Unassigned'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.998, + "point": [ + 965, + 423 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_015", + "intent": "type 'F'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_016", + "intent": "press enter", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_017", + "intent": "click 'Billing Note:'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 146, + 599 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_018", + "intent": "press end", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_019", + "intent": "click 'Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 272, + 733 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_020", + "intent": "click 'Confirm Create New Patient'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 600, + 260 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 0, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "rows": [ + { + "pid": "1", + "fname": "Oaun3014bpp1", + "lname": "Avclab", + "DOB": "2001-01-01", + "sex": "Female" + } + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 90.36, + "first_frame_sha256": "ce778ede6adbee0d0fd66b603e652d32588982b3d6f03536438abe5d03668b11", + "frame_sha256": [ + "be6ee76a419f3483d837ec42850c010f3c605fd1c1f9f00c9423af447f5feaa7", + "ce778ede6adbee0d0fd66b603e652d32588982b3d6f03536438abe5d03668b11", + "00c476babebb54d3921ccd7e4990a5d65c565f43c1bc6b8adfc0badd61703dd6", + "e705e0163e87e2ad1dbb00c04976a7196e66820fc9ecf35f76a25d15ada583d6", + "fbb87d93a15682f1cd5508a7efa78663d21373979ff86525042f50de431a327a", + "00c476babebb54d3921ccd7e4990a5d65c565f43c1bc6b8adfc0badd61703dd6", + "d57529c016349a2904e3c22dc96b0e48e7f80ca4f38865c0dc605de6cb1c17b8", + "fbb87d93a15682f1cd5508a7efa78663d21373979ff86525042f50de431a327a", + "c122b1a17681c47f9a02aebe4a72afc0516b79807aa80e50e443b0a61d9c7fad", + "d57529c016349a2904e3c22dc96b0e48e7f80ca4f38865c0dc605de6cb1c17b8", + "e39000ec05fc5ced81ccd154252195f5abe377ee6d4002f6c612dd02b0ef4bab", + "93210024359906fa82297fc13700db33864b1b3a14a7663e2f34a557e6cb46e8", + "d4c2a461d54f98ea052ee2e63c2abd4d3baf805f160d70065bfbf9d9beacc5cf", + "2c13e0ebc72b997d650f68c4b1d1acc471dde38cfebdca188f903d6d2ec24d97", + "954dd830a67120a2bcdc85851f0f3778c9d33fec85ef32600064d49f1d10345f", + "d4c2a461d54f98ea052ee2e63c2abd4d3baf805f160d70065bfbf9d9beacc5cf", + "778f52455b565e442cd99dc37a9d5a4656e56c07636bdef43875a5add3c53c08", + "09f26715c091c4e3e1c9c43357ce15bc560ca3946b12249b604d4a1592c0fffc", + "44081faa5c19bc1fc27f84ebe9706cd78d1ea184d1a064487b124b423e7575a2", + "19477c9118baa2ec5ece74018f1f3c2a028515b35e1160d681f6d0c2984e9686", + "f16d7dbd882d5f9e63cfb891946b9f641775274235b15a1f6885b30a2d739871", + "44081faa5c19bc1fc27f84ebe9706cd78d1ea184d1a064487b124b423e7575a2", + "5f8b1014765bc214486eb2ba4c90c105682787bacf9329b446b5d19a80281b3f", + "f16d7dbd882d5f9e63cfb891946b9f641775274235b15a1f6885b30a2d739871", + "caa192b1f23f734691c9bd80338049041cac8790396d90badfe1ba571690db9c", + "e8c7ae15a8903848c5598c08f930bf873dd9f04dd5d850e0bebda410e5121316", + "cf22ac47f5ed84e1bb97914c2b4bccda686ba3e0ce6e3e8300dbd83881f10e9d", + "cf22ac47f5ed84e1bb97914c2b4bccda686ba3e0ce6e3e8300dbd83881f10e9d", + "b4e9e0d6adbc3fcf4a9e27dc42a4d2fdaa8707ec82dd33a4d8b4e652d96294c6", + "8ac4411f4f77061af92139144a6f30395eee4eb8909201226323b1c02f1373d5", + "c3d337a9b25cb9d71fcc4a1026111fbb78098998a9be19eb9414e32bc6288208", + "b4e9e0d6adbc3fcf4a9e27dc42a4d2fdaa8707ec82dd33a4d8b4e652d96294c6", + "3f48df3f326a7ae5fc20ddc48c2a7f9e03fdc531db160803ad0187099c5ad100", + "c3d337a9b25cb9d71fcc4a1026111fbb78098998a9be19eb9414e32bc6288208", + "a6e8dcc66a003c861e8a2d37a369551b5cef0d2d594ffd35e223f3abe17d4a2b", + "3f48df3f326a7ae5fc20ddc48c2a7f9e03fdc531db160803ad0187099c5ad100", + "a76389fac137a631cfd8dab8872e2a11c0e4686763f678309ca8020e46dcadf9", + "a6e8dcc66a003c861e8a2d37a369551b5cef0d2d594ffd35e223f3abe17d4a2b", + "5a2eb763d3fb8c71882ca59800efd2b8bad0fe6dcae61223fefa1f002bd9874e", + "d01d3b4886538c6304c3147f12aa8a5e4ddfa6b8ce6fa5c9fb506d872c7290e8", + "2fb6a38f8bf2299695ad0ab4e04d1d7980a204589e3ad9e1c9b0e5a8f0c48525", + "11cb99c5229f3ffb03cd41fb69fa929c76cca3921e212084522f7d97fbe0e3cb" + ] + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-1/readback.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-1/readback.json new file mode 100644 index 0000000000..5abd33f815 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-1/readback.json @@ -0,0 +1,122 @@ +[ + { + "case": 1, + "stored": "Rb000note", + "expected": "Rb000note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 35.66 + }, + { + "case": 2, + "stored": "Rb001note", + "expected": "Rb001other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 40.44 + }, + { + "case": 3, + "stored": "Rb002note", + "expected": "Rb002note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.34 + }, + { + "case": 4, + "stored": "Rb003note", + "expected": "Rb003other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 38.84 + }, + { + "case": 5, + "stored": "Rb004note", + "expected": "Rb004note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 40.8 + }, + { + "case": 6, + "stored": "Rb005note", + "expected": "Rb005other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.09 + }, + { + "case": 7, + "stored": "Rb006note", + "expected": "Rb006note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.0 + }, + { + "case": 8, + "stored": "Rb007note", + "expected": "Rb007other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.03 + }, + { + "case": 9, + "stored": "Rb008note", + "expected": "Rb008note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.05 + }, + { + "case": 10, + "stored": "Rb009note", + "expected": "Rb009other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 38.91 + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-1/result.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-1/result.json new file mode 100644 index 0000000000..8f219d5aa0 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-1/result.json @@ -0,0 +1,125 @@ +{ + "tier": "3", + "cases": [ + { + "case": 1, + "stored": "Rb000note", + "expected": "Rb000note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 35.66 + }, + { + "case": 2, + "stored": "Rb001note", + "expected": "Rb001other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 40.44 + }, + { + "case": 3, + "stored": "Rb002note", + "expected": "Rb002note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.34 + }, + { + "case": 4, + "stored": "Rb003note", + "expected": "Rb003other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 38.84 + }, + { + "case": 5, + "stored": "Rb004note", + "expected": "Rb004note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 40.8 + }, + { + "case": 6, + "stored": "Rb005note", + "expected": "Rb005other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.09 + }, + { + "case": 7, + "stored": "Rb006note", + "expected": "Rb006note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.0 + }, + { + "case": 8, + "stored": "Rb007note", + "expected": "Rb007other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.03 + }, + { + "case": 9, + "stored": "Rb008note", + "expected": "Rb008note", + "truth": "confirmed", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 39.05 + }, + { + "case": 10, + "stored": "Rb009note", + "expected": "Rb009other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view still shows what it showed before the search; it may not have refreshed, so it can't be read as this save; cannot certify the save; HALT", + "assurance": "unchecked", + "elapsed_s": 38.91 + } + ] +} \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-2/readback.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-2/readback.json new file mode 100644 index 0000000000..4464515005 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-2/readback.json @@ -0,0 +1,122 @@ +[ + { + "case": 1, + "stored": "Rb000note", + "expected": "Rb000note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 53.76 + }, + { + "case": 2, + "stored": "Rb001note", + "expected": "Rb001other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 43.39 + }, + { + "case": 3, + "stored": "Rb002note", + "expected": "Rb002note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 46.51 + }, + { + "case": 4, + "stored": "Rb003note", + "expected": "Rb003other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 45.22 + }, + { + "case": 5, + "stored": "Rb004note", + "expected": "Rb004note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 45.17 + }, + { + "case": 6, + "stored": "Rb005note", + "expected": "Rb005other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 43.58 + }, + { + "case": 7, + "stored": "Rb006note", + "expected": "Rb006note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 45.58 + }, + { + "case": 8, + "stored": "Rb007note", + "expected": "Rb007other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 45.09 + }, + { + "case": 9, + "stored": "Rb008note", + "expected": "Rb008note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 42.14 + }, + { + "case": 10, + "stored": "Rb009note", + "expected": "Rb009other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 43.27 + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-2/result.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-2/result.json new file mode 100644 index 0000000000..d0247e3005 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-readback-2/result.json @@ -0,0 +1,125 @@ +{ + "tier": "3", + "cases": [ + { + "case": 1, + "stored": "Rb000note", + "expected": "Rb000note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 53.76 + }, + { + "case": 2, + "stored": "Rb001note", + "expected": "Rb001other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 43.39 + }, + { + "case": 3, + "stored": "Rb002note", + "expected": "Rb002note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 46.51 + }, + { + "case": 4, + "stored": "Rb003note", + "expected": "Rb003other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 45.22 + }, + { + "case": 5, + "stored": "Rb004note", + "expected": "Rb004note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 45.17 + }, + { + "case": 6, + "stored": "Rb005note", + "expected": "Rb005other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 43.58 + }, + { + "case": 7, + "stored": "Rb006note", + "expected": "Rb006note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 45.58 + }, + { + "case": 8, + "stored": "Rb007note", + "expected": "Rb007other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 45.09 + }, + { + "case": 9, + "stored": "Rb008note", + "expected": "Rb008note", + "truth": "confirmed", + "verdict": "confirmed", + "correct": true, + "false_confirm": false, + "reason": "field 'note' equals the expected value", + "assurance": "screen_readback", + "elapsed_s": 42.14 + }, + { + "case": 10, + "stored": "Rb009note", + "expected": "Rb009other", + "truth": "refuted", + "verdict": "indeterminate", + "correct": false, + "false_confirm": false, + "reason": "the record view, read by OCR, does not match the input (note); OCR can misread, so this is not proof the save is wrong. Check the record; HALT", + "assurance": "unchecked", + "elapsed_s": 43.27 + } + ] +} \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-1/reports.tar.gz b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-1/reports.tar.gz new file mode 100644 index 0000000000..4b3bef37c3 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-1/reports.tar.gz differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-1/trials.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-1/trials.json new file mode 100644 index 0000000000..9a9f9254cf --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-1/trials.json @@ -0,0 +1,5096 @@ +[ + { + "trial": 1, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 20.98, + "first_frame_sha256": "9c0ac4b1d38cafdeb380be043c31fd3ea2832cae9db24ae2d5d618ae0980e66a", + "frame_sha256": [ + "46043f5203de2858014657c5be8c1c3c8f4b2e048b3f67db141b4501a601a410", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "c016863b0cf029ff8129af248f22e6c33b42b33012aca401f83c107287d7b5ea", + "9c0ac4b1d38cafdeb380be043c31fd3ea2832cae9db24ae2d5d618ae0980e66a", + "dc51309d9fa6832a3d80aabf1c791162a0c65e85d5e4ddf825827fc6ed978263", + "c016863b0cf029ff8129af248f22e6c33b42b33012aca401f83c107287d7b5ea", + "136332e219000af3de3bd589421335999f943e5c72176ee1c76a3cb17239b49a", + "dc51309d9fa6832a3d80aabf1c791162a0c65e85d5e4ddf825827fc6ed978263", + "995da17a22329b6793426daf11d28fe3010245abf1a997109d9b776a31de4850", + "625b432ddaf4609b2c07d70974bd5347deb09f254dbc68848a2c9c321a0d05b9", + "14301d2d5ad44a89a6eb3fed549919160846e821596c1b0e3488195af5651c5b", + "995da17a22329b6793426daf11d28fe3010245abf1a997109d9b776a31de4850", + "fdd0e0cef81b1049e88fec18c3008b77f0edb823b789fe50f7309e20b4fc6537", + "14301d2d5ad44a89a6eb3fed549919160846e821596c1b0e3488195af5651c5b", + "be109ff491bae0adeb3706d74f996106ef00753a49107b859d932fc8292e77be", + "46043f5203de2858014657c5be8c1c3c8f4b2e048b3f67db141b4501a601a410" + ], + "first_frame": { + "file": "pn_healthy_first.png", + "sha256": "9c0ac4b1d38cafdeb380be043c31fd3ea2832cae9db24ae2d5d618ae0980e66a", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 2, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.33, + "first_frame_sha256": "acf8316a7abe133c755679848023932f9baba50d413f4c12df5cd048c8f9c6e1", + "frame_sha256": [ + "c9b2179da50a4e81cd11f3b6a5af1acc3f7f1c77389424b96f39c2f220ea5785", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "3fe1a5e515fb92f9aed85dbb8e4825465b97a7d1cabca131afb280d2eb45cf2c", + "acf8316a7abe133c755679848023932f9baba50d413f4c12df5cd048c8f9c6e1", + "6c2eb0ada783100ae0c958019c1b29aa709f9ade7a2c8f625c6d4087269af5a3", + "3fe1a5e515fb92f9aed85dbb8e4825465b97a7d1cabca131afb280d2eb45cf2c", + "6acdfdc743de75a27b0e7a77b08afc77b16d7fe5b7916c4ed08be40d64b042d7", + "6c2eb0ada783100ae0c958019c1b29aa709f9ade7a2c8f625c6d4087269af5a3", + "86dc2cdd41212e1b4f6ecbc6f297bc307cdd1d681ebe951d04c048f1e908f65d", + "ab0be8c1ee496535f56725d3d1ad3a32270368701e16d659514292b27df9a8c5", + "f7a9f0ba890d0a98fbd44b7e5cbcb1760d709c18fea5ca028e5789773e045ebb", + "86dc2cdd41212e1b4f6ecbc6f297bc307cdd1d681ebe951d04c048f1e908f65d", + "6aa6ca2b17efb13604942c651482c83f9d0d5108cf039fe0114397fd8ecb8dca", + "f7a9f0ba890d0a98fbd44b7e5cbcb1760d709c18fea5ca028e5789773e045ebb", + "755671b237d840878dfffe639baa1198fc318c4a3504f222c2625efb54a36943", + "c9b2179da50a4e81cd11f3b6a5af1acc3f7f1c77389424b96f39c2f220ea5785" + ] + }, + { + "trial": 3, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.41, + "first_frame_sha256": "cbcf8a793c9d629ea18239b44305cca7cc1ba4dcff6669420bfad51cff3563a6", + "frame_sha256": [ + "af4a9a1ee3a464ab010d5756d8942b70bf83c54f87a8df800fd5d6f7a91c058f", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "1c443155c4b156bee9c3c5ae43835126f010621c291d018cfbc40142c9b867ad", + "cbcf8a793c9d629ea18239b44305cca7cc1ba4dcff6669420bfad51cff3563a6", + "d06f9d59a7a969d6c07f70d3c1f6fe172446fae4b8645c8d7f219b76e1c02147", + "1c443155c4b156bee9c3c5ae43835126f010621c291d018cfbc40142c9b867ad", + "33a2b0fb7e02e21be1dbac0a18a65afc560a9b7350806a1e9c38aaee48b7d127", + "d06f9d59a7a969d6c07f70d3c1f6fe172446fae4b8645c8d7f219b76e1c02147", + "3322dbda08a529cff2f10585208217ece27614ff7bb5c244875289c0c93c49b3", + "54527ee2e40eaeb3825b82aab1e9da5bec8fe175c0e736e8ba75b90a2efc2f96", + "7cfb4c1fdb02e33a546ff702086f61cb4e41cbed1fd926f7dada7c829d141815", + "3322dbda08a529cff2f10585208217ece27614ff7bb5c244875289c0c93c49b3", + "d766449919997fa01590f3965dfca5bf85556214e829f5a5c61aa73a4668d5ff", + "7cfb4c1fdb02e33a546ff702086f61cb4e41cbed1fd926f7dada7c829d141815", + "235171e9cc395ac0e459e6b4294ae440498f6e73986978905fc4d48a594ed754", + "af4a9a1ee3a464ab010d5756d8942b70bf83c54f87a8df800fd5d6f7a91c058f" + ] + }, + { + "trial": 4, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.27, + "first_frame_sha256": "deff2df88e625688f574262adc7866523933b48f02f531e4305b10bbb6342256", + "frame_sha256": [ + "6fe7a8a633793de19351cb36836180d20a278cf87a84b1e5aa91368cf711590c", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "24a98b6f25613afe5552f35179b499443fcf85aa1959bacae015311cc43c67bc", + "deff2df88e625688f574262adc7866523933b48f02f531e4305b10bbb6342256", + "5b31670696cd404e727cc3b0d9078af8d3a204ff4f43916708fead912d91c5a2", + "24a98b6f25613afe5552f35179b499443fcf85aa1959bacae015311cc43c67bc", + "526d4c57b7d55efe18496770c4089b7860a3dbfeb70ce617c2df16cf4b6aaa57", + "5b31670696cd404e727cc3b0d9078af8d3a204ff4f43916708fead912d91c5a2", + "5648fe7a897f00c9db7b40a88e4513b2c9b1015bfc517f9038dfa1165fde32fc", + "b5aae3777f7b647293b8a72c631193a5b4cbf85b3e701e75f0efe25befd450de", + "1e68911c0512e2a15632641ddc024b13f16f704d17b7373345f65faba5643854", + "5648fe7a897f00c9db7b40a88e4513b2c9b1015bfc517f9038dfa1165fde32fc", + "f394deec73e7378437c34e6f3cd8c8e76ec96f2adcac432132fd541dca3b5293", + "1e68911c0512e2a15632641ddc024b13f16f704d17b7373345f65faba5643854", + "fcbcf034e5355e07763528b2a6d4fa7db8667689443d87403d966b0deb2af2be", + "6fe7a8a633793de19351cb36836180d20a278cf87a84b1e5aa91368cf711590c" + ] + }, + { + "trial": 5, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.53, + "first_frame_sha256": "4e60b17f2275ed6b2d81e259696c55fc4e006f4422a4c602981f783189a1e3a3", + "frame_sha256": [ + "45c20b781b58bd49051508b02ce1be1ea2dfed59fba4e0e43715e115427aeea2", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "7f4e8f0d203380c76097aaee676c3a8f73aa65ad8061f897f36faaa2c91fa670", + "4e60b17f2275ed6b2d81e259696c55fc4e006f4422a4c602981f783189a1e3a3", + "5a23f4dbce3113fc46726f7a48323e12234acec83110cc918ee985601cf781ae", + "7f4e8f0d203380c76097aaee676c3a8f73aa65ad8061f897f36faaa2c91fa670", + "84eda28b79c1412f7b5252deaddd54bf93f3d0eee8ed9b99fe9328a17e857c74", + "5a23f4dbce3113fc46726f7a48323e12234acec83110cc918ee985601cf781ae", + "607552a35eddd6e2fca49d6e283b4f23c019d4f841dff00fdd1e71321a806589", + "bcb20ec359eb7d55658b017191c9ad490f789c1335c4724300c84255d8fb8164", + "b117b8a22c9d8a645520e892ba1c2d706ccad187eded6bcbf41ce5d8db782c65", + "607552a35eddd6e2fca49d6e283b4f23c019d4f841dff00fdd1e71321a806589", + "dc2fc87361748730ec84ec7071d015418dfd0d8dc21b422fced4de5a60c5102a", + "b117b8a22c9d8a645520e892ba1c2d706ccad187eded6bcbf41ce5d8db782c65", + "64684d94974e78604c77a22eb15f8de1482561cafa517551e44663fc5d02ae89", + "45c20b781b58bd49051508b02ce1be1ea2dfed59fba4e0e43715e115427aeea2" + ] + }, + { + "trial": 6, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.45, + "first_frame_sha256": "6770197356528ff4997fdc274365dfc6a70d0e968ac4c7015034df5c6ff9adb5", + "frame_sha256": [ + "5edca7f0789053519d9e4a53b5226f0a6dbb16f8237f10a62631ae3e01c1181f", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "98de6a7663dbdf918dede3caec0a1b95bbf71fe7daa24d3e325b89826d7423d9", + "6770197356528ff4997fdc274365dfc6a70d0e968ac4c7015034df5c6ff9adb5", + "7853b0dc1be93f9d5d556692a492ee48ea4c1e46e10cde68de8041a8143222fb", + "98de6a7663dbdf918dede3caec0a1b95bbf71fe7daa24d3e325b89826d7423d9", + "11c9c9630f8177a0f450dd196778dc32ad140545a944fbd65aad917c3aab8284", + "7853b0dc1be93f9d5d556692a492ee48ea4c1e46e10cde68de8041a8143222fb", + "923c78bb66a592d97a08c906ed57f76734462c4b4c5521489a5edebeb385a1cd", + "f7acbcba3bf76659b65944882d551a5a3e595fe1ee8ecb37f26489ad07cd6907", + "e681e1e85ee35d848bbc4635fa0d669540de61198db76452c725ed0704aab365", + "923c78bb66a592d97a08c906ed57f76734462c4b4c5521489a5edebeb385a1cd", + "ac3a4d00cf934b18531738cb0a3208ab595a0172e78da8b2daf0eaf93423ff97", + "e681e1e85ee35d848bbc4635fa0d669540de61198db76452c725ed0704aab365", + "a5126654424bf37eef2d27ce8fbc719089f1f405a6714a5679f36dab7ac5cf9d", + "5edca7f0789053519d9e4a53b5226f0a6dbb16f8237f10a62631ae3e01c1181f" + ] + }, + { + "trial": 7, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.5, + "first_frame_sha256": "23286c934833ae484109acce2902c2ec90755a0dcc39868f4bf21f09ff4e6efd", + "frame_sha256": [ + "4ba78a16490a092e7c60df776841c2175fee37aeea848a412794adadb293e3c4", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "70bdb6c9c22c648a17cef5e5f4dfabc658028f5c3f2ac109624e4bf00107cbd5", + "23286c934833ae484109acce2902c2ec90755a0dcc39868f4bf21f09ff4e6efd", + "40139576cb3b5d6a237de3a056c3d41b30fe979c1df95274aa3a8f57ddd35f0a", + "70bdb6c9c22c648a17cef5e5f4dfabc658028f5c3f2ac109624e4bf00107cbd5", + "24814f7a80c9c9cbb9927481daaf2d3e4ff108e492ed2ff3c0eab7cf75a57360", + "40139576cb3b5d6a237de3a056c3d41b30fe979c1df95274aa3a8f57ddd35f0a", + "11a3cc82212be9d1bcfff1ead353f65c6bdc4ec7dde6177d5a52cbf0735218fd", + "e7183a540d802cb56a555a5cfa7a8493c04a51a3d38ca2c1fad1f2cd7be02f61", + "fadc8738c95559927655c550b28cb03efc60ca05a4f0cda30fc4e668b2cc5490", + "11a3cc82212be9d1bcfff1ead353f65c6bdc4ec7dde6177d5a52cbf0735218fd", + "818d197870fb6ee34129a99b96a7095975b7705d23ec9af73ea2d9665cc80d7e", + "fadc8738c95559927655c550b28cb03efc60ca05a4f0cda30fc4e668b2cc5490", + "5d9bc5bc47ac7355529388e515393c6333fb72346b2c7ce2283ceccecc479f6b", + "4ba78a16490a092e7c60df776841c2175fee37aeea848a412794adadb293e3c4" + ] + }, + { + "trial": 8, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.42, + "first_frame_sha256": "e1ed2516a49c5f52d928d5e0cfd0ae8394b6d6045d129cc68bcfa5e4361fcf78", + "frame_sha256": [ + "9122df02906d56785d97cff2b2ef11f661f1214febf8c662035a0b1005b38536", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "548668af8c60c22e2a7a1bed7bd8746c0b494e53499c27c21f0d7e1ef3c8c7f1", + "e1ed2516a49c5f52d928d5e0cfd0ae8394b6d6045d129cc68bcfa5e4361fcf78", + "3813975869577d82d979afbd7181f932f0e12816582c35c0a234929d77f085dd", + "548668af8c60c22e2a7a1bed7bd8746c0b494e53499c27c21f0d7e1ef3c8c7f1", + "df54ad3d98c7eb8774c37a33488b661cf6e8413e3cb9c2667247de3983db5cf0", + "3813975869577d82d979afbd7181f932f0e12816582c35c0a234929d77f085dd", + "9c3bbc58af437793aaab1d7193b105d685a957bf063daf03dbeeb63064efeef2", + "7160e987fe047b1c4bdaff2e09242b78e55fdc976c0ca40832f0a821d09f202e", + "89751b7babf8acd580811c991e6b3c7fe43c5dece2ad3ea3ea27e50d2ce37dab", + "9c3bbc58af437793aaab1d7193b105d685a957bf063daf03dbeeb63064efeef2", + "9e0e80ae02f49fa6f57dd73a98a8e1d8106ec731263a3cb9200d17eb9233ab25", + "89751b7babf8acd580811c991e6b3c7fe43c5dece2ad3ea3ea27e50d2ce37dab", + "707680428199656f7c6e9a4047c23da0384c810a2e34f670762b8911cbeaf30b", + "9122df02906d56785d97cff2b2ef11f661f1214febf8c662035a0b1005b38536" + ] + }, + { + "trial": 9, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 15.51, + "first_frame_sha256": "ded3c1b4db494d453077aa9bce33e9673909afd574bcd16d8001001ed7d43a6e", + "frame_sha256": [ + "3e41f07bb8c6424bb8d3fa1ad75255a77dc3049f4041630b369c85ff8592028a", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "4436b23350b1c7ab56a19fc1913fd9506ca238845f8641307bcd4a16264396f1", + "ded3c1b4db494d453077aa9bce33e9673909afd574bcd16d8001001ed7d43a6e", + "2ae93eb2edbc3efda474b7e6c708cc8cd1a430224018be654c136e11f277be48", + "4436b23350b1c7ab56a19fc1913fd9506ca238845f8641307bcd4a16264396f1", + "82bb996c054f8fd109a597fb5f68931074f70ae97c4bf747897399e8e600dd5f", + "2ae93eb2edbc3efda474b7e6c708cc8cd1a430224018be654c136e11f277be48", + "86903f3bb137077f50fae3638367c2d941bb3525a51cf8db6386e109ae0a0d5f", + "82bb996c054f8fd109a597fb5f68931074f70ae97c4bf747897399e8e600dd5f", + "c8fa1b4a91a1ffc2c4319c945060e39f922c51f57b2224b7e186da7275f88ba2", + "86903f3bb137077f50fae3638367c2d941bb3525a51cf8db6386e109ae0a0d5f", + "7426dfc228970a2fefa26523563cd50db8b7bf7974fd9255fb2fdc688461bb13", + "c8fa1b4a91a1ffc2c4319c945060e39f922c51f57b2224b7e186da7275f88ba2", + "7f50d920a0a499e02f1a33e5c753b16dc4efb821da503e9b2084a77d61fea5ed", + "3e41f07bb8c6424bb8d3fa1ad75255a77dc3049f4041630b369c85ff8592028a" + ] + }, + { + "trial": 10, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 22.5, + "first_frame_sha256": "36dcd9207218219cc73f7eb34473dc129d16b90a534b9b1c3fddc9f30e493923", + "frame_sha256": [ + "0653fac8e2ad830e5cbb1ee5f17f9dad606a5cb051df220f38341d45946caae1", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "29cb4ca6eb3e36b7f7c342bbde002d7fabdcd222246c5acb8777a1dfe9217e69", + "36dcd9207218219cc73f7eb34473dc129d16b90a534b9b1c3fddc9f30e493923", + "c61fe4d5dcd15879d5129f3fc6d78dc9b487e6fb245960b7881fb3ee3601a9d0", + "29cb4ca6eb3e36b7f7c342bbde002d7fabdcd222246c5acb8777a1dfe9217e69", + "fb6995664c78757e79e365327fe851968974ed8f4c1c4bd52cbf2c2663f71745", + "c61fe4d5dcd15879d5129f3fc6d78dc9b487e6fb245960b7881fb3ee3601a9d0", + "171b6e378b4d27b805bf593b3c92f711e35b77f181ab01fb6411cb4ce574c4b1", + "fb6995664c78757e79e365327fe851968974ed8f4c1c4bd52cbf2c2663f71745", + "f4495a0709dc9383f0b3a0e722b0719d1cab88aadb175617129fc4f9962c7697", + "171b6e378b4d27b805bf593b3c92f711e35b77f181ab01fb6411cb4ce574c4b1", + "c683432c93a5f8c8f8c50fba116596d7397831a47d2bc3036abb28c34f015f4b", + "f4495a0709dc9383f0b3a0e722b0719d1cab88aadb175617129fc4f9962c7697", + "80ffec89293b207bc5c7a9360196f4a71aae7eeb3aed885214e71ed363770056", + "0653fac8e2ad830e5cbb1ee5f17f9dad606a5cb051df220f38341d45946caae1" + ] + }, + { + "trial": 11, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.5, + "first_frame_sha256": "bccc9e8e162a07708b14af7f8070b28627a10842661dc3584958c424ac33ff26", + "frame_sha256": [ + "1fbd98e7b9dc177ec3531207f64146d709bf2782c0cf92bd8e409162939b4576", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "7b03bab21fa986b6393e620e387f6a44465ecbf207daf2554095360c82ea9b2f", + "bccc9e8e162a07708b14af7f8070b28627a10842661dc3584958c424ac33ff26", + "b7353a4ec30b0925d6de8bc04377d517f81a09103812139e91372cee6e9a69db", + "7b03bab21fa986b6393e620e387f6a44465ecbf207daf2554095360c82ea9b2f", + "c62ca9bad42812ab861730f3bdd74d1553e4c81d0718056588cef9cbe48c8068", + "b7353a4ec30b0925d6de8bc04377d517f81a09103812139e91372cee6e9a69db", + "adf6fd616b26ee013ee23fb354c553ee861fdbaedd5b55e46d4635e92c783f94", + "08020cd9c93b774e3221f90a8127b22a4b73748e00315fd0754de0a46fe9079d", + "d2db0d8c17775ca5415bc7af71240870ec5e6c11f94a9f39eb0e8f6b8bf31da6", + "adf6fd616b26ee013ee23fb354c553ee861fdbaedd5b55e46d4635e92c783f94", + "172e4bc0dbfb3852cd9ab174a6fc4c1df9416f59dd6c9dc8c48c08f9bd1065a1", + "d2db0d8c17775ca5415bc7af71240870ec5e6c11f94a9f39eb0e8f6b8bf31da6", + "b5f8451cd70e8ad6813162584cc63b0543a392af0ba9dd5b75e4b475dde94b0a", + "1fbd98e7b9dc177ec3531207f64146d709bf2782c0cf92bd8e409162939b4576" + ], + "first_frame": { + "file": "pn_reorder_first.png", + "sha256": "bccc9e8e162a07708b14af7f8070b28627a10842661dc3584958c424ac33ff26", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 12, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.34, + "first_frame_sha256": "d5a6fb027469876f71420359420e2f51e29fb9dac5768f53b56b61bde7c08f52", + "frame_sha256": [ + "be38fbb7582eb2a655c581e7a1b0ccbb59f7c396e2e62ccd4100cd5d98e620bf", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "ceb3e20310c7863db3d62db1275bbb21beca415f2dd755368c5a547c7488c858", + "d5a6fb027469876f71420359420e2f51e29fb9dac5768f53b56b61bde7c08f52", + "64259f47a55f1c6ad4123f13c92fb0c4729fb268215d32846a96a83c0ceb33d1", + "ceb3e20310c7863db3d62db1275bbb21beca415f2dd755368c5a547c7488c858", + "5097ad08146a5aead445254e2b48d1c23b1ba078721f500b00c29f000757482a", + "64259f47a55f1c6ad4123f13c92fb0c4729fb268215d32846a96a83c0ceb33d1", + "3b389fe257c9f83237b0a16e3dafe6d89485ced1cbe0c7e9eeed8ab437abc9de", + "1abdc3354b538590593850bdead7a5ba9f80c77986da383b1bea8e8df237f079", + "187cea055dd58c38466f9ebf0085ee6b24cc4f0ae35367a2134709e91e0694af", + "3b389fe257c9f83237b0a16e3dafe6d89485ced1cbe0c7e9eeed8ab437abc9de", + "67b11ceab68177b0612ae809e37c777c0c17f00a39e106d35b5e66b63bc4449c", + "187cea055dd58c38466f9ebf0085ee6b24cc4f0ae35367a2134709e91e0694af", + "449c1a757a3cc3a64c979695973c4cd68d03442e0458b48699f045f607d47340", + "be38fbb7582eb2a655c581e7a1b0ccbb59f7c396e2e62ccd4100cd5d98e620bf" + ] + }, + { + "trial": 13, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.5, + "first_frame_sha256": "67ae481e2632a4fde41f73ded2aa4fb819992282a3a817d2875ec3c6dfb1f9b6", + "frame_sha256": [ + "0b604a9a61520d7087ffea7e09f5ad14c787eb722b5a6b455cf5aade85763ece", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "2faf5ddf2de67b252f229a57d9f269630b57361c4aa7eedcd2c01a741cf32a18", + "67ae481e2632a4fde41f73ded2aa4fb819992282a3a817d2875ec3c6dfb1f9b6", + "025f7922abaed2f1939f0ce0162a2b7bebba71778c5f34a5aea1d64ad01141f2", + "2faf5ddf2de67b252f229a57d9f269630b57361c4aa7eedcd2c01a741cf32a18", + "1959a4ceba7e90bf76d62875ad1b6870d58f8528f356fd09ceb1e87005030379", + "025f7922abaed2f1939f0ce0162a2b7bebba71778c5f34a5aea1d64ad01141f2", + "173c544676fa385da0de3ad19b3fcfd4515dbf4c9e920a97e28b84c9030f2772", + "c7bad19d06e96223a3545ed275dc8e7f47be1454cd85535da0c3cd4fc3fb64d3", + "bce3c9cde61846d59d97949ed22b563f2021ae703899ae2e7d46f79188d4352d", + "173c544676fa385da0de3ad19b3fcfd4515dbf4c9e920a97e28b84c9030f2772", + "499c0bcef354ac6883f153ca252c8dfeb7a706813682ce3b7259a3e7a6222981", + "bce3c9cde61846d59d97949ed22b563f2021ae703899ae2e7d46f79188d4352d", + "f4b25754eefa38dea7e86b67f8feefbe64069fc281d70167b58939cc1104f6e5", + "0b604a9a61520d7087ffea7e09f5ad14c787eb722b5a6b455cf5aade85763ece" + ] + }, + { + "trial": 14, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.44, + "first_frame_sha256": "a507126b22fbb1947f41af5cbae15b57febcac76d544cbfcb4bba5bc11ab3585", + "frame_sha256": [ + "c8841f3cc04e19a12a53d88b131cd615f3d5dee3337c58606325076f0847163e", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "ecf4cb56d38f79d94bcb448dae7e6a90592aa1ac57136bf35b50d5928f4b9d7f", + "a507126b22fbb1947f41af5cbae15b57febcac76d544cbfcb4bba5bc11ab3585", + "bda8d9efc8764d515ea7ce5d640e0aa6ee09bb3c4535dcfafdb3208815a113c9", + "ecf4cb56d38f79d94bcb448dae7e6a90592aa1ac57136bf35b50d5928f4b9d7f", + "94382b56e326c5c7b9fee478b902437f042c4ab0c50e5f207f4564ba43de9bb5", + "bda8d9efc8764d515ea7ce5d640e0aa6ee09bb3c4535dcfafdb3208815a113c9", + "6485c7d5157b835715bd56b71ccd252d9ecc423603ea5108d94b93aadc167cd3", + "ecb45b98b59be8509764d980625a55095fe61833b1e9c156900ac345a57cc688", + "67d9018b120e379b2fa9288b76df36e7c9bdc66bdc044915cd344d5682fd6ac9", + "6485c7d5157b835715bd56b71ccd252d9ecc423603ea5108d94b93aadc167cd3", + "8b940d4d1578280a8e7e567eb5133846d1eda05678e900aa92ac048ca1401898", + "67d9018b120e379b2fa9288b76df36e7c9bdc66bdc044915cd344d5682fd6ac9", + "964cdbc0d53761162805224fe0f602868a919bcb221de68e98990bd2e91fcdcd", + "c8841f3cc04e19a12a53d88b131cd615f3d5dee3337c58606325076f0847163e" + ] + }, + { + "trial": 15, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.28, + "first_frame_sha256": "04dd3716855b7bb5305090140db367941a6142e4a3b037862d6b9ba1e4771b3e", + "frame_sha256": [ + "12ef736a53be79a471d26c682aab5b62ecc6ff46cd0cb842ba5711e090ff639d", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "aa9d3a994f2b8f25fd5849b83172295252bbc27ed8004bd349f6e8ee61854390", + "04dd3716855b7bb5305090140db367941a6142e4a3b037862d6b9ba1e4771b3e", + "5409a9eee148a6c97ddf4a2837f79d2674393ff4696625414c292b6cbd5a332f", + "aa9d3a994f2b8f25fd5849b83172295252bbc27ed8004bd349f6e8ee61854390", + "4a311dd5719e2be473cbfb0443d96e9901706a226cb67d2de532a45a2245fa57", + "5409a9eee148a6c97ddf4a2837f79d2674393ff4696625414c292b6cbd5a332f", + "e2a05d040bdc253da27d714caf1d2b9b4f3ddb8e0c6d09cf220dbbec8f1dbcbe", + "3c21c024f1db14a1a64185d3297708872c609cea22b40403d4727f5853d4c5d7", + "aff167338c1c4b6f47f040d4a52420da292e5bb5328044e6e279dd974ea78998", + "e2a05d040bdc253da27d714caf1d2b9b4f3ddb8e0c6d09cf220dbbec8f1dbcbe", + "68f82f9b877f83a4012ddc775377e557ba088b1205d06b1a21ac41d76ed927f8", + "aff167338c1c4b6f47f040d4a52420da292e5bb5328044e6e279dd974ea78998", + "40cdfadb5221681a42fd4a7d33dcb4551bd47cf0fc79a26f79ae41e11791d293", + "12ef736a53be79a471d26c682aab5b62ecc6ff46cd0cb842ba5711e090ff639d" + ] + }, + { + "trial": 16, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:54 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 14.91, + "first_frame_sha256": "3accf4a0df5d4d0877fc66adf0c62db72f79339b11efc32d2039192732b352a2", + "frame_sha256": [ + "b4cb30d73e79bdc99dc849a4566e0b438d2d385a7e99742bfdd00e2b09b28977", + "3accf4a0df5d4d0877fc66adf0c62db72f79339b11efc32d2039192732b352a2", + "8e6143d529ee7d18d70035c421c3a5a180b871e2d6af100b33efb9e0128fd6df", + "b4cb30d73e79bdc99dc849a4566e0b438d2d385a7e99742bfdd00e2b09b28977", + "efdc426410128a4861e982acf8ad50f95074dbef9bb3d080bc7a239585a3eb21", + "8e6143d529ee7d18d70035c421c3a5a180b871e2d6af100b33efb9e0128fd6df" + ], + "first_frame": { + "file": "pn_decoy_first.png", + "sha256": "3accf4a0df5d4d0877fc66adf0c62db72f79339b11efc32d2039192732b352a2", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 17, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:55 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 13.39, + "first_frame_sha256": "563a4af35d800eac4f55f1f00d470a40b0831f2cf7a507625c2c56b9b273e42e", + "frame_sha256": [ + "cf28fd3eae141e340586dc59b6cf2456d60301e57d8befc1595bb91cf937b81a", + "563a4af35d800eac4f55f1f00d470a40b0831f2cf7a507625c2c56b9b273e42e", + "c0885ba710fff7fd6b0d25879b8b097c49952114a2e79a03caabc87e0e33fdfe", + "cf28fd3eae141e340586dc59b6cf2456d60301e57d8befc1595bb91cf937b81a", + "38aef8b11618e2de9f944f3044c9568533f6b781c3c707faa1783b5cc68410e6", + "c0885ba710fff7fd6b0d25879b8b097c49952114a2e79a03caabc87e0e33fdfe" + ] + }, + { + "trial": 18, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:55 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 12.56, + "first_frame_sha256": "962826e1565f9af2f17a5e8548fa50a98737f32c95ca7c7eae46408cf09bd504", + "frame_sha256": [ + "78048053295001170f3b8b95a73e858718626ca732648070f130d27168fcdecd", + "962826e1565f9af2f17a5e8548fa50a98737f32c95ca7c7eae46408cf09bd504", + "a155cc4096c6abbba47627dd1a10e2fe03311f34a3b9f0da75baf905307624e4", + "78048053295001170f3b8b95a73e858718626ca732648070f130d27168fcdecd", + "833671a8d7ebb97ec127cdc35fb811e0b2f1dad255d30e6c42b16a7f19262ef5", + "a155cc4096c6abbba47627dd1a10e2fe03311f34a3b9f0da75baf905307624e4" + ] + }, + { + "trial": 19, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:55 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 12.64, + "first_frame_sha256": "8e0ead562a2bcad8272d9babadd35d9364da6ef6b89e75d6824eb29d16a6ec53", + "frame_sha256": [ + "05c5beac5868654229e23744b9d3a766ba3d4542bbb8846cf5d0b9092444f9e9", + "8e0ead562a2bcad8272d9babadd35d9364da6ef6b89e75d6824eb29d16a6ec53", + "f3282f7f063df3abc0dd8d4df20a00cec1adab36396da266c8f6a6a9d7366c5e", + "05c5beac5868654229e23744b9d3a766ba3d4542bbb8846cf5d0b9092444f9e9", + "3c275340ed67f4184e878c6b770559643e1865d72b6f124ce283d92aeedb5538", + "f3282f7f063df3abc0dd8d4df20a00cec1adab36396da266c8f6a6a9d7366c5e" + ] + }, + { + "trial": 20, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:56 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 9.69, + "first_frame_sha256": "982dff43daa19e8c8a6eb339356d5149fef4ee6429ef9fc438c05a9d429335c4", + "frame_sha256": [ + "2255bb97807c4944e7a4739dab7ef9bbfc17fd81c67cfc080fd8ef5758d1d71d", + "982dff43daa19e8c8a6eb339356d5149fef4ee6429ef9fc438c05a9d429335c4", + "5bc372e628032b0b03d3bc57412775e4b6881b1c274facc4be4f193b78c3aed0", + "2255bb97807c4944e7a4739dab7ef9bbfc17fd81c67cfc080fd8ef5758d1d71d", + "5a9ad965cbbea0d139a1e771bbf0d409c0bfc27ed81aa0196bf4021cc234d868", + "5bc372e628032b0b03d3bc57412775e4b6881b1c274facc4be4f193b78c3aed0" + ] + }, + { + "trial": 21, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 115, + 145 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:56 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 7.33, + "first_frame_sha256": "00ab40490413286d9855d72731e835bc0daf21aae73c3ea537a57aec3ed5192b", + "frame_sha256": [ + "45dd4e0c0b9638f2ac15f4b30270f4e0825b10efb9b88971998629d59f6a6e3f", + "00ab40490413286d9855d72731e835bc0daf21aae73c3ea537a57aec3ed5192b", + "c5335785f5f1aecc53afc48e531dee7b61b35e040a2d641f773a3e52e197933a", + "45dd4e0c0b9638f2ac15f4b30270f4e0825b10efb9b88971998629d59f6a6e3f", + "91d66c5eaa2e2c2f5f0655072d346ceeb83153d26f7a94c6d09a63a6f01bb5d5", + "c5335785f5f1aecc53afc48e531dee7b61b35e040a2d641f773a3e52e197933a", + "afb242064086e2013164a0ab72e296124e91a99b066e2a27cf12f1af244fb43c", + "afb242064086e2013164a0ab72e296124e91a99b066e2a27cf12f1af244fb43c" + ], + "first_frame": { + "file": "pn_siblings_first.png", + "sha256": "00ab40490413286d9855d72731e835bc0daf21aae73c3ea537a57aec3ed5192b", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 22, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 115, + 145 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed '1983-01-19 Sorensen' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed '1983-01-19 Sorensen' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:56 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 7.25, + "first_frame_sha256": "d6f770c18043f33090b9d85972de7d24052eaee5a2983c31db2b60c09932d587", + "frame_sha256": [ + "3ce7127000c81d1a243283e395d4222d054580a5a385157744dee9e3588a3795", + "d6f770c18043f33090b9d85972de7d24052eaee5a2983c31db2b60c09932d587", + "b58484509d579d6d5eb7638042853b13b04ee59da128032c76a8132d5b06aea3", + "3ce7127000c81d1a243283e395d4222d054580a5a385157744dee9e3588a3795", + "9a4f211b39015449cab654b7ce966f316ec2baa76f397a8c44a36cd4f93ef27b", + "b58484509d579d6d5eb7638042853b13b04ee59da128032c76a8132d5b06aea3", + "fbbd512fd7574b1c9b512f793c254d7c1e26df53213c6be0020ba56e7e1121b4", + "fbbd512fd7574b1c9b512f793c254d7c1e26df53213c6be0020ba56e7e1121b4" + ] + }, + { + "trial": 23, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 115, + 145 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:57 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 4.42, + "first_frame_sha256": "00ab40490413286d9855d72731e835bc0daf21aae73c3ea537a57aec3ed5192b", + "frame_sha256": [ + "45dd4e0c0b9638f2ac15f4b30270f4e0825b10efb9b88971998629d59f6a6e3f", + "00ab40490413286d9855d72731e835bc0daf21aae73c3ea537a57aec3ed5192b", + "c5335785f5f1aecc53afc48e531dee7b61b35e040a2d641f773a3e52e197933a", + "45dd4e0c0b9638f2ac15f4b30270f4e0825b10efb9b88971998629d59f6a6e3f", + "91d66c5eaa2e2c2f5f0655072d346ceeb83153d26f7a94c6d09a63a6f01bb5d5", + "c5335785f5f1aecc53afc48e531dee7b61b35e040a2d641f773a3e52e197933a", + "afb242064086e2013164a0ab72e296124e91a99b066e2a27cf12f1af244fb43c", + "afb242064086e2013164a0ab72e296124e91a99b066e2a27cf12f1af244fb43c" + ] + }, + { + "trial": 24, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 115, + 145 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:57 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 6.67, + "first_frame_sha256": "6ddf63b3755eb9ace28bfb337c857ddbde2a57350c8f90f2edc8de533f7ca99a", + "frame_sha256": [ + "071c15198cdcb18a3248126f84cc7bd3036dbc8e87dae6699ac50532f1f77ddb", + "6ddf63b3755eb9ace28bfb337c857ddbde2a57350c8f90f2edc8de533f7ca99a", + "b760a82f2247bc0c2992155980bf7e769a9987a4da3ecf82dc544cf750a29db8", + "071c15198cdcb18a3248126f84cc7bd3036dbc8e87dae6699ac50532f1f77ddb", + "dc72bbe28a5f5ee22c398447a321631c4733c4ae798d4a88d26346ce0cbb7b59", + "b760a82f2247bc0c2992155980bf7e769a9987a4da3ecf82dc544cf750a29db8", + "dc72bbe28a5f5ee22c398447a321631c4733c4ae798d4a88d26346ce0cbb7b59", + "dc72bbe28a5f5ee22c398447a321631c4733c4ae798d4a88d26346ce0cbb7b59" + ] + }, + { + "trial": 25, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": "geometry", + "confidence": 0.9, + "point": [ + 115, + 145 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "last", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "8", + "O Type here to search", + "12:57 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 7.92, + "first_frame_sha256": "176dc66ee991bebb95a282eb5c4d5b2efe544772973c25c6b68d5d249accee5a", + "frame_sha256": [ + "7829b038483ee08fd16d5714323d604196da1c033674db37fb2c4e2a5f1847eb", + "176dc66ee991bebb95a282eb5c4d5b2efe544772973c25c6b68d5d249accee5a", + "1775e252d2f9b18ad9f3114edeb865f30317b007c3f80ce667a221c3d64daf91", + "7829b038483ee08fd16d5714323d604196da1c033674db37fb2c4e2a5f1847eb", + "4ebd2e29e6b647383b348be3bcd0f14962387f6a30aa0f5d3081c8b9b43279b6", + "1775e252d2f9b18ad9f3114edeb865f30317b007c3f80ce667a221c3d64daf91", + "37688ccf452df2ebb1008d0197f2342bed459c28309e9eabc3e3297669ab21dc", + "37688ccf452df2ebb1008d0197f2342bed459c28309e9eabc3e3297669ab21dc" + ] + }, + { + "trial": 26, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 17.78, + "first_frame_sha256": "b23db933136ee910ea1264ca2bf2705d371e2f676a5a48b389baeffa76d1eb1c", + "frame_sha256": [ + "b613db3ad41ec315ce3598ad5b9a061b375c00552066162cbf0678611a593934", + "6454559c359ae74ae94ab24740d1db70c206550783dc73bac940a0e3ae15471d", + "1c2d28d1f085db37a06123220e0632b4c28831784268106b579aed1f0c94a791", + "b23db933136ee910ea1264ca2bf2705d371e2f676a5a48b389baeffa76d1eb1c", + "fbfbb6a996dc8636bab6bb2ac589993f21fc5e08739ca7f2eeabed85aef8e9cc", + "1c2d28d1f085db37a06123220e0632b4c28831784268106b579aed1f0c94a791", + "661d61dd4ad08bc4dd1f8f5d41e9f79325dfa81ef52d0b676695b1e6058b5db9", + "fbfbb6a996dc8636bab6bb2ac589993f21fc5e08739ca7f2eeabed85aef8e9cc", + "0d7dae17f982796a466e4356883a0fd83d1d6f02312c906bb3a419de03a3dea1", + "c97f58cbea87ec46b2610f6560b15eda9af91cc4282b944971e71aa692fdeed2", + "831609a21a41f6a46161bb67b0e8fbe22ad99a741e45d5807c6b91a7e3c215d1", + "730f2e7d830be9ba22bab9317ce750ae7f007e4bdce429071bf4b0828ed32edc", + "2d4c35dada2b12e177948b499096145750f8667dc332af6570b7b07e3159f651", + "831609a21a41f6a46161bb67b0e8fbe22ad99a741e45d5807c6b91a7e3c215d1", + "60ad0daccd94c3eba6be79f19f50f73155b583a59731efa3f1c5a9ec41f149e4", + "b613db3ad41ec315ce3598ad5b9a061b375c00552066162cbf0678611a593934" + ], + "first_frame": { + "file": "pn_bpp16_first.png", + "sha256": "b23db933136ee910ea1264ca2bf2705d371e2f676a5a48b389baeffa76d1eb1c", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 27, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.98, + "first_frame_sha256": "e47ce033b63fb0f380b33a73e5c862e38b94db5155277c6a44471b69bf97f856", + "frame_sha256": [ + "7e0c95433c432441183873f403466508a9f41759fea37fd7b582f8e6f8034060", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "774fb71356576f71f0b61345ac0f5ec2906c2402bd08661c17a62bffabdfa9a9", + "e47ce033b63fb0f380b33a73e5c862e38b94db5155277c6a44471b69bf97f856", + "77d94f69c72a75e32f925bd9f6602f51f948182a9f68b461dff9839b1bf8ea42", + "774fb71356576f71f0b61345ac0f5ec2906c2402bd08661c17a62bffabdfa9a9", + "0fc82104cd6c0253f1f07d7d0fd98b9b562bd8e4027d339b6a3156ae4b6f1560", + "77d94f69c72a75e32f925bd9f6602f51f948182a9f68b461dff9839b1bf8ea42", + "089e78ff549df9b4a48e31e414e917b4bb46d604f4990994f86d2db0cbb50e44", + "145e99d154d22b8aac33c04953d414108129e627ca84b6df28bd45d78f1c7d5e", + "bf91e00168ac962f6efdd66e2b414ee057c3ed998cd5e0adbf792a8d990740d9", + "089e78ff549df9b4a48e31e414e917b4bb46d604f4990994f86d2db0cbb50e44", + "026d25d1ecd0e460e18c404edd4eb504f4294ec914d484a758e75d1b90162df3", + "bf91e00168ac962f6efdd66e2b414ee057c3ed998cd5e0adbf792a8d990740d9", + "aac3ddf8cc1e16f593eb46d7346037b5011833f070add905c45e0c984a4e239c", + "7e0c95433c432441183873f403466508a9f41759fea37fd7b582f8e6f8034060" + ] + }, + { + "trial": 28, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 19.2, + "first_frame_sha256": "0c70baa87660434d42db5d7ba8ad0efc8d1d512df1abfdef8dc0e6332f212b21", + "frame_sha256": [ + "9f3a45664b02cdda8913dee3cee2befe70e2e4fc7a26420e8cae08110a5a53c6", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "774fb71356576f71f0b61345ac0f5ec2906c2402bd08661c17a62bffabdfa9a9", + "0c70baa87660434d42db5d7ba8ad0efc8d1d512df1abfdef8dc0e6332f212b21", + "77d94f69c72a75e32f925bd9f6602f51f948182a9f68b461dff9839b1bf8ea42", + "774fb71356576f71f0b61345ac0f5ec2906c2402bd08661c17a62bffabdfa9a9", + "145e99d154d22b8aac33c04953d414108129e627ca84b6df28bd45d78f1c7d5e", + "77d94f69c72a75e32f925bd9f6602f51f948182a9f68b461dff9839b1bf8ea42", + "089e78ff549df9b4a48e31e414e917b4bb46d604f4990994f86d2db0cbb50e44", + "145e99d154d22b8aac33c04953d414108129e627ca84b6df28bd45d78f1c7d5e", + "bf91e00168ac962f6efdd66e2b414ee057c3ed998cd5e0adbf792a8d990740d9", + "089e78ff549df9b4a48e31e414e917b4bb46d604f4990994f86d2db0cbb50e44", + "5bbc2d829630e69ac8aa4288f228c04779a189797a1a3c2260f85f20753b3428", + "bf91e00168ac962f6efdd66e2b414ee057c3ed998cd5e0adbf792a8d990740d9", + "62dc5d94081e641d3ac53689f39a0ac0dffce3298af6a18a6ac050bac5d2d5cc", + "9f3a45664b02cdda8913dee3cee2befe70e2e4fc7a26420e8cae08110a5a53c6" + ] + }, + { + "trial": 29, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.12, + "first_frame_sha256": "05733307b54ea7fcecda6ffbd5efece674acb40c772dfc361f8484eaf2e09061", + "frame_sha256": [ + "02e3076d22a5cd51a3bd8899490f48f79408045d8641d7de9e7d94833e6c6c09", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "e87d171c8fd618b86fcf3cf4b7722ef5c8ecd04202ba9b1291b79e62d8f571c5", + "05733307b54ea7fcecda6ffbd5efece674acb40c772dfc361f8484eaf2e09061", + "ffc4dbf0fa09da881b233ec101718c22dc5a8bde2c1adda704647737bd491a77", + "e87d171c8fd618b86fcf3cf4b7722ef5c8ecd04202ba9b1291b79e62d8f571c5", + "ac53cb4749d0fd137d7a5eb0290b236b4d02a398f2e1384d83d2be5ba69d1b52", + "ffc4dbf0fa09da881b233ec101718c22dc5a8bde2c1adda704647737bd491a77", + "0fbf7c9bd6ab8d3e64b0fe075783f9befe024990ecf96dde7eff22e4c2e1d6ec", + "cd9f446a093556871de173c7d833a5206d6cee1aa080547e8b5a027551da3c53", + "d7c5f66bc9c6085f1eaa64cbc1ba47eacf67025b44ccfa62264f8a0bb389aa4f", + "0fbf7c9bd6ab8d3e64b0fe075783f9befe024990ecf96dde7eff22e4c2e1d6ec", + "4df0114f9c20fd596eb3b607ffea8876be8acfcf64b53c9cb882fb6f81f219ba", + "d7c5f66bc9c6085f1eaa64cbc1ba47eacf67025b44ccfa62264f8a0bb389aa4f", + "0cf7c7983eb4a028659dad04dd2e0c7fe42c427e66d64bf6f979619af289744a", + "02e3076d22a5cd51a3bd8899490f48f79408045d8641d7de9e7d94833e6c6c09" + ] + }, + { + "trial": 30, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "note_holders": [ + 1 + ], + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.02, + "first_frame_sha256": "780be040238d35f54e41e9831dfab53fff53fc21b9161a27e3f68dc7229f8e68", + "frame_sha256": [ + "f0b0acadacf5106be9d32e6e68f10d1da41061b00361de8eb7520ddc8c2cb738", + "916135340152763298bd4e84c7cee9fe0417b3aeb4e745ecf937e865266c3635", + "fcfb4bd359b6e8eb11a21a814f8bf21b5955f51545fdd8962f4443b7e2c8e996", + "780be040238d35f54e41e9831dfab53fff53fc21b9161a27e3f68dc7229f8e68", + "1c1717e06bc4f8da4d4b14ce34946c516965ade9d93c384c603c939b8da0d9d8", + "fcfb4bd359b6e8eb11a21a814f8bf21b5955f51545fdd8962f4443b7e2c8e996", + "94039760711cbdebc2fda02d02c576fe301a87c4ab3dc50bbda1e86570b169b7", + "1c1717e06bc4f8da4d4b14ce34946c516965ade9d93c384c603c939b8da0d9d8", + "6dbf82776b11cd0204ec4d9daabf801e71a047ffe1002f55f579aa702b46de5c", + "ac46b45c4dba2cad72f30f8acd3542092026d32d4008068a4a597828dd7e9955", + "309f38c547f77bdafc740b16d9a5392e621749a9ee85042a72663ab4248a09f8", + "6dbf82776b11cd0204ec4d9daabf801e71a047ffe1002f55f579aa702b46de5c", + "9f0b63b9cd15de5290327698e8beb644bb44bc0cbf2e8af1cf1ebbcbba69f51b", + "309f38c547f77bdafc740b16d9a5392e621749a9ee85042a72663ab4248a09f8", + "3bb92f88292b9909989147d9929ba59f0f6687244d5fdf49d6efd16c4de67bb0", + "f0b0acadacf5106be9d32e6e68f10d1da41061b00361de8eb7520ddc8c2cb738" + ] + }, + { + "trial": 31, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "8", + "O Type here to search", + "1:00 AM", + ">", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 15.22, + "first_frame_sha256": "e6e02a305e7cd6453f328226ebc99b55fe9bd32db59dc3e332f7797df906eb1e", + "frame_sha256": [ + "305aee93e134594c17e17514f304938d8d70ce87089e3168a99e1648f1d4a097", + "e6e02a305e7cd6453f328226ebc99b55fe9bd32db59dc3e332f7797df906eb1e", + "f0d45e2dc21ab81f668cb5107ac2f739d592d06932ba2c469961f84978a1c00c", + "305aee93e134594c17e17514f304938d8d70ce87089e3168a99e1648f1d4a097", + "4bf9438639cf0d3cc22f6f867a45185a19a205d18b000ae01324a43bd4611b53", + "f0d45e2dc21ab81f668cb5107ac2f739d592d06932ba2c469961f84978a1c00c", + "e28fac763bab4a978a878f36d0c4a512919a6717bd8b5545253810ec2a2d9db4", + "4bf9438639cf0d3cc22f6f867a45185a19a205d18b000ae01324a43bd4611b53", + "7d9b88a9634142ecc74d2ef673d76dbbbf7a9a5b3a0e41acd3c6b629af22b115", + "e28fac763bab4a978a878f36d0c4a512919a6717bd8b5545253810ec2a2d9db4", + "2d956651a98de04718d34bd9da0d7e077deef9762c4a01720287cd181017f827", + "7d9b88a9634142ecc74d2ef673d76dbbbf7a9a5b3a0e41acd3c6b629af22b115", + "49b211a2b60a51246fcdf1f3bf71935a9bb2ffcea9dc5807b75a3c3dc12b3e3c", + "49b211a2b60a51246fcdf1f3bf71935a9bb2ffcea9dc5807b75a3c3dc12b3e3c" + ], + "first_frame": { + "file": "pn_desktop1440_first.png", + "sha256": "e6e02a305e7cd6453f328226ebc99b55fe9bd32db59dc3e332f7797df906eb1e", + "size": [ + 1440, + 900 + ] + } + }, + { + "trial": 32, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "N!", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "oaun-1031desk", + "Loaded 1 patients", + "Save Note", + "8", + "O Type here to search", + "1:00 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 15.94, + "first_frame_sha256": "381b2a2177ff0af6c516c23d87d77014790b1fd18f2ae1817d40a71cddae00cb", + "frame_sha256": [ + "3477ab09ccfa9bde71d17c1ed460a6de8970584930bf845cbb701455810784e0", + "381b2a2177ff0af6c516c23d87d77014790b1fd18f2ae1817d40a71cddae00cb", + "9cb256ce30513ba37dd7b4c0fd32d0f170bb60308f31de70204e71d571c5a391", + "3477ab09ccfa9bde71d17c1ed460a6de8970584930bf845cbb701455810784e0", + "2742b834ebb47643a481321a61b1a7e80dd98da0fdbc90208a187bdd7b3bdbf8", + "9cb256ce30513ba37dd7b4c0fd32d0f170bb60308f31de70204e71d571c5a391", + "aa5074c083ab65f4c4e448f9cfbfe1e6c3fa67714c1613f934175467add92c93", + "2742b834ebb47643a481321a61b1a7e80dd98da0fdbc90208a187bdd7b3bdbf8", + "4220accea8e4c3db7c9ed775a949d532f157295a22898f93e6463f7ca4cc85c2", + "aa5074c083ab65f4c4e448f9cfbfe1e6c3fa67714c1613f934175467add92c93", + "a08e7394af65a4a5dbaba7c1c74124e69c6ac126100c0e4372ebd3824f9aa6e2", + "4220accea8e4c3db7c9ed775a949d532f157295a22898f93e6463f7ca4cc85c2", + "26e90ea094c0a3a9ad4f145d83748b81882e62b5740446d6c027c787ba37490f", + "26e90ea094c0a3a9ad4f145d83748b81882e62b5740446d6c027c787ba37490f" + ] + }, + { + "trial": 33, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "N!", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "8", + "O Type here to search", + "1:01AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 14.88, + "first_frame_sha256": "3839ad6758d58c058ab0644693734c89c65e10eac1c0caa15833248a5559a45e", + "frame_sha256": [ + "598a84ae9152ca33039a38c2a20cedfc84d44d8abe2fe4581681c7d6e8d3848a", + "3839ad6758d58c058ab0644693734c89c65e10eac1c0caa15833248a5559a45e", + "32e36c69e8a0df1d42c05e418dcc832242e601eca4176b0689d150ba79a3b87b", + "598a84ae9152ca33039a38c2a20cedfc84d44d8abe2fe4581681c7d6e8d3848a", + "dc8495d049f0795ac81235212ac0b1ae43bdb770e46fdaa3f6028ed67ed8b11e", + "32e36c69e8a0df1d42c05e418dcc832242e601eca4176b0689d150ba79a3b87b", + "0d196768a1dd29926522108386ecb3f592b502e567bea4b9d32f8f85d3f5dc15", + "dc8495d049f0795ac81235212ac0b1ae43bdb770e46fdaa3f6028ed67ed8b11e", + "01ec35a1ecb62c475f816f18c9f6ff345ba87c605f179a964d6aeb0a2e944075", + "0d196768a1dd29926522108386ecb3f592b502e567bea4b9d32f8f85d3f5dc15", + "c461ea82b0b1eb0b9d4103a0b40cfd9248799acbe58dd44eeb214e347509bf03", + "01ec35a1ecb62c475f816f18c9f6ff345ba87c605f179a964d6aeb0a2e944075", + "127eba737f725e913fde0da85cafd2e36b6fc6d823a9fbd8f86cec7354b04f75", + "127eba737f725e913fde0da85cafd2e36b6fc6d823a9fbd8f86cec7354b04f75" + ] + }, + { + "trial": 34, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "N!", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "8", + "O Type here to search", + "1:01AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 15.11, + "first_frame_sha256": "719167c579d4f516af2a4a375af07a9004fe8dfee88a95ae914140b034f79e8e", + "frame_sha256": [ + "3d0122d8997703ba54fa9fcf40fb9e4fc7201eb4267a9e41bf1d71a34914484c", + "719167c579d4f516af2a4a375af07a9004fe8dfee88a95ae914140b034f79e8e", + "c2e182a52f10b0c894da2182f4fe1fdc0575d549336fa3b6d5360ec70d25e4ca", + "3d0122d8997703ba54fa9fcf40fb9e4fc7201eb4267a9e41bf1d71a34914484c", + "01982278baa2913f7256b2373043b11a19ddcbf3ebc305375b7f8de121bab5e3", + "c2e182a52f10b0c894da2182f4fe1fdc0575d549336fa3b6d5360ec70d25e4ca", + "19a07154ad8d66422bb9057a896c49b1eca4254b384f06d40afb19624fab572a", + "01982278baa2913f7256b2373043b11a19ddcbf3ebc305375b7f8de121bab5e3", + "e679a967ab9caaeb30432534644cd1991746d54d01b89217c47822167dab6f5d", + "19a07154ad8d66422bb9057a896c49b1eca4254b384f06d40afb19624fab572a", + "702a2916df3f8975a8eef30cb797039aef6e29dfda6b6c3ecabbd4cf588b17af", + "e679a967ab9caaeb30432534644cd1991746d54d01b89217c47822167dab6f5d", + "73b807ca6931749b0e7f146ef7f9c5867979c19d8ecb8f6658bb071c63788a59", + "73b807ca6931749b0e7f146ef7f9c5867979c19d8ecb8f6658bb071c63788a59" + ] + }, + { + "trial": 35, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes - Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "N!", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "8", + "O Type here to search", + "1:02 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "note_holders": [], + "class": "false_stop", + "must_save": true, + "elapsed_s": 17.06, + "first_frame_sha256": "c0a355b2a2e78687356452f486af02d06aacaf65f2d5ec05ecf67aebcd335df7", + "frame_sha256": [ + "bf967abb3ae4abed6feafcf95072fef7d6b36fc70ba0c85d6614e18d69678db8", + "c0a355b2a2e78687356452f486af02d06aacaf65f2d5ec05ecf67aebcd335df7", + "b6a2212430747ff889caf1c10c5d313f9ae770af072f15e65aeb6005c5bf7a4d", + "bf967abb3ae4abed6feafcf95072fef7d6b36fc70ba0c85d6614e18d69678db8", + "13a4e52fcb6e6483d2edc835fd6d24581e25a529aa584896a1c0dadd080c2245", + "b6a2212430747ff889caf1c10c5d313f9ae770af072f15e65aeb6005c5bf7a4d", + "0d3f2051771f6276ede33582bec1655c37d0a8a1cd748ccce09025c04d3fa0fb", + "1cc8be5c0c9cfe748ed983448bc25a324308ba28cecac09cda22065d6f2be109", + "7a02977c9671d9b16cb41a8ec851e3fb1a1843830d7003f65159003ee7fae196", + "0d3f2051771f6276ede33582bec1655c37d0a8a1cd748ccce09025c04d3fa0fb", + "4bc5de4db7e938ee021d4f02bff44590d98c2297aa0ca896ef0bb7dcc8e2e2d4", + "7a02977c9671d9b16cb41a8ec851e3fb1a1843830d7003f65159003ee7fae196", + "5a9a7f103f42e2e5b50a7d7c215bf0620c35f58358556781a7503f81fd4be17f", + "5a9a7f103f42e2e5b50a7d7c215bf0620c35f58358556781a7503f81fd4be17f" + ] + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-3/reports.tar.gz b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-3/reports.tar.gz new file mode 100644 index 0000000000..dbf40f8203 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-3/reports.tar.gz differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-3/trials.json b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-3/trials.json new file mode 100644 index 0000000000..0f6f0548a5 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/campaigns/pn-run-3/trials.json @@ -0,0 +1,5431 @@ +[ + { + "trial": 1, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3000heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 22.41, + "first_frame_sha256": "af672ce15e7db963d2caff403e9342ae4dc89907c3c85a4b257457dceccc11bc", + "frame_sha256": [ + "da78d7594e0506bdf07dd66263dfa0a53f510ac151e0f5135c9782747187df29", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "9c45a059b76e5ac1dd44216a13e73df46499c5493372e5d4a264fbb1bdf36813", + "af672ce15e7db963d2caff403e9342ae4dc89907c3c85a4b257457dceccc11bc", + "71c5735b7f41b99d3f4506664bcd2777a6eb565c5d81941dbe356a152a8c159e", + "9c45a059b76e5ac1dd44216a13e73df46499c5493372e5d4a264fbb1bdf36813", + "06a545908a0b3ff5567c8fd42e615d0abefca565ad2c8ba24d53c74bf07295cb", + "71c5735b7f41b99d3f4506664bcd2777a6eb565c5d81941dbe356a152a8c159e", + "153b495024d85f6339cb4eeb327f87bed3dbd77c5193b82d9814506b0f1cf8db", + "06a545908a0b3ff5567c8fd42e615d0abefca565ad2c8ba24d53c74bf07295cb", + "89c90b4d90adc4feaa9fb8d7bd372e77e7189077638747eeafa4a5ce6158a49b", + "153b495024d85f6339cb4eeb327f87bed3dbd77c5193b82d9814506b0f1cf8db", + "e06ed027528b26fa16b9a1b3f0788eaf07d3fcc20be32f27520c8cb1b741293e", + "89c90b4d90adc4feaa9fb8d7bd372e77e7189077638747eeafa4a5ce6158a49b", + "d69fb0b4789b96bf75d1d644cad4f3311124a5ec34b5a4943b2014e2d946ee1e", + "da78d7594e0506bdf07dd66263dfa0a53f510ac151e0f5135c9782747187df29" + ], + "first_frame": { + "file": "pn_healthy_first.png", + "sha256": "af672ce15e7db963d2caff403e9342ae4dc89907c3c85a4b257457dceccc11bc", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 2, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3001heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 16.0, + "first_frame_sha256": "fc0259785a3961c91f9aadc8fb6bc01d6a5829d8c875a3d968e41ef39b5877c5", + "frame_sha256": [ + "b5cf9a2443d36f1b3695cc3669dcce5747a47052b2ff603385d9abe46c3221a5", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "abc4150f7027d4795e9a5ee8919848ef327e14857c5e2d21b46bca9158bd8c47", + "fc0259785a3961c91f9aadc8fb6bc01d6a5829d8c875a3d968e41ef39b5877c5", + "b442cd0dfdabeb2d247945484c2e2724cc80a64b0e70ab3bbc61f2c9092e1dea", + "abc4150f7027d4795e9a5ee8919848ef327e14857c5e2d21b46bca9158bd8c47", + "06a545908a0b3ff5567c8fd42e615d0abefca565ad2c8ba24d53c74bf07295cb", + "b442cd0dfdabeb2d247945484c2e2724cc80a64b0e70ab3bbc61f2c9092e1dea", + "153b495024d85f6339cb4eeb327f87bed3dbd77c5193b82d9814506b0f1cf8db", + "06a545908a0b3ff5567c8fd42e615d0abefca565ad2c8ba24d53c74bf07295cb", + "89c90b4d90adc4feaa9fb8d7bd372e77e7189077638747eeafa4a5ce6158a49b", + "153b495024d85f6339cb4eeb327f87bed3dbd77c5193b82d9814506b0f1cf8db", + "a319f447b14453b54c08c0456e61e006717de07eb6c4649d4bfd7df3f51ac770", + "89c90b4d90adc4feaa9fb8d7bd372e77e7189077638747eeafa4a5ce6158a49b", + "77775d94de8c671b4819a643a1707ef5b8b0b09ab4eaa641ef14956011ac674d", + "b5cf9a2443d36f1b3695cc3669dcce5747a47052b2ff603385d9abe46c3221a5" + ] + }, + { + "trial": 3, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3002heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.5, + "first_frame_sha256": "86cc95229f1c0b4ac294e8497ca3b8863dd02ec839480344d57e76030c0b4f76", + "frame_sha256": [ + "cd53a08fe5f352a3abf3a665355c9cd22bd4b4b4ea1ec765c29ee0b20f6fa448", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "e83929200d8ae8186e36ed28c09a1d329bd47c95ec259b68032bf5b5411b2bfe", + "86cc95229f1c0b4ac294e8497ca3b8863dd02ec839480344d57e76030c0b4f76", + "d234ebed55b1170bfbd32328aaf62ddf50292a7f064d7ddff1c120dddf2aef28", + "e83929200d8ae8186e36ed28c09a1d329bd47c95ec259b68032bf5b5411b2bfe", + "fe12947b5e89241e057c813fca2a4590292acd429285f2a7336f5cf3b2c0dc11", + "d234ebed55b1170bfbd32328aaf62ddf50292a7f064d7ddff1c120dddf2aef28", + "743dfb8b93732ec4d6ac402d673e54db3743b7cc187092bce57a7e70d60dd562", + "fe12947b5e89241e057c813fca2a4590292acd429285f2a7336f5cf3b2c0dc11", + "73181201854e49416e029e298ece160ed2a64cc8162c07273412488a1429ac23", + "743dfb8b93732ec4d6ac402d673e54db3743b7cc187092bce57a7e70d60dd562", + "c3d901eb520dc6ca75cb103419d4bd05d198124b8af05f8a86ff61fbd65882be", + "73181201854e49416e029e298ece160ed2a64cc8162c07273412488a1429ac23", + "bdd55e8469c860fb6b954847c47e1a566f05597f95466a9ae4bb951fad634022", + "cd53a08fe5f352a3abf3a665355c9cd22bd4b4b4ea1ec765c29ee0b20f6fa448" + ] + }, + { + "trial": 4, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3003heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.48, + "first_frame_sha256": "36cd732ff395edf205d600b3298bc1c0aa0a2d686e55235ef4e33709b0ed131e", + "frame_sha256": [ + "5f703bb57b6c966e8687ebad52b156b7b1f5b55905a2875a869fff8e4d473c03", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "9cbc40d7d4fe78b5599ae8edc6a6655545ea13c59d60af68b50cbd4c65daed6b", + "36cd732ff395edf205d600b3298bc1c0aa0a2d686e55235ef4e33709b0ed131e", + "a47bb11ff22563d4cb2f0ea2bcee8ea37f995b98bb588e7f99c4da2a218d3b81", + "9cbc40d7d4fe78b5599ae8edc6a6655545ea13c59d60af68b50cbd4c65daed6b", + "ed4c67169839dc8e5d27c401de89461282dbce3511914a54a35f71d338b04f4a", + "a47bb11ff22563d4cb2f0ea2bcee8ea37f995b98bb588e7f99c4da2a218d3b81", + "23ceac974da8e781d33e3e060aeb51db9171c21f090649e291432094f604d156", + "ed4c67169839dc8e5d27c401de89461282dbce3511914a54a35f71d338b04f4a", + "53319747fb45ddf76c5d0bed1a19222dbd2ac8da241413ec188ee4826e83fa5e", + "23ceac974da8e781d33e3e060aeb51db9171c21f090649e291432094f604d156", + "751e5a28efc8a9eb0150afa95fd8c29b49b878ac40c7e2e774dfb676bdf81218", + "53319747fb45ddf76c5d0bed1a19222dbd2ac8da241413ec188ee4826e83fa5e", + "3ac6e100c9fa054f0bd4ecf5ca40e4824e2eb2fa7221169fb94673871c5a1ab3", + "5f703bb57b6c966e8687ebad52b156b7b1f5b55905a2875a869fff8e4d473c03" + ] + }, + { + "trial": 5, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3004heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.28, + "first_frame_sha256": "cf5d86772d9fc91a9aede8d2fb2a64d2fc7167d4fc18b01aa2add2931aa8e378", + "frame_sha256": [ + "7fd4e2a5f8463e9efd4ed2775d9502acdb0470efda84651207f81f6fb1a7ef05", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "516847b2e39fb75845328cc119b6adf00fc2ad279660c7bb3ad4d479bdcc57cf", + "cf5d86772d9fc91a9aede8d2fb2a64d2fc7167d4fc18b01aa2add2931aa8e378", + "46c1ebcbaaf3ea42b201d90ea815410fc3804c79ca805abecc08e2684759c72d", + "516847b2e39fb75845328cc119b6adf00fc2ad279660c7bb3ad4d479bdcc57cf", + "bfbfdfddaaff29fcbfa1f1f3d8859ec80ce878a0f389ed15e39b7b2be00880a4", + "46c1ebcbaaf3ea42b201d90ea815410fc3804c79ca805abecc08e2684759c72d", + "e8e48e1db8cec337e8791cc2e68f9b1fbddc9de6ce93a96418c7eddac5668986", + "bfbfdfddaaff29fcbfa1f1f3d8859ec80ce878a0f389ed15e39b7b2be00880a4", + "cb219a421ac01eeca07a540f39c30cc0b96b4728468967732a20302a8e36adfd", + "e8e48e1db8cec337e8791cc2e68f9b1fbddc9de6ce93a96418c7eddac5668986", + "693dd0be13a93b9d0213e9d63fed6813df44f9868fe5bd3e99ab54774c938171", + "cb219a421ac01eeca07a540f39c30cc0b96b4728468967732a20302a8e36adfd", + "aebb9e0d4bfea43629026e435786d09af3f02137b89234e834484a07915761bf", + "7fd4e2a5f8463e9efd4ed2775d9502acdb0470efda84651207f81f6fb1a7ef05" + ] + }, + { + "trial": 6, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3005heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 14.94, + "first_frame_sha256": "cf5d86772d9fc91a9aede8d2fb2a64d2fc7167d4fc18b01aa2add2931aa8e378", + "frame_sha256": [ + "c950e869bb8a35b41ef1ddd0cef4e6db0f671c19188a25e7de32b6d36f0c6c8d", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "516847b2e39fb75845328cc119b6adf00fc2ad279660c7bb3ad4d479bdcc57cf", + "cf5d86772d9fc91a9aede8d2fb2a64d2fc7167d4fc18b01aa2add2931aa8e378", + "46c1ebcbaaf3ea42b201d90ea815410fc3804c79ca805abecc08e2684759c72d", + "516847b2e39fb75845328cc119b6adf00fc2ad279660c7bb3ad4d479bdcc57cf", + "bfbfdfddaaff29fcbfa1f1f3d8859ec80ce878a0f389ed15e39b7b2be00880a4", + "46c1ebcbaaf3ea42b201d90ea815410fc3804c79ca805abecc08e2684759c72d", + "e8e48e1db8cec337e8791cc2e68f9b1fbddc9de6ce93a96418c7eddac5668986", + "bfbfdfddaaff29fcbfa1f1f3d8859ec80ce878a0f389ed15e39b7b2be00880a4", + "cb219a421ac01eeca07a540f39c30cc0b96b4728468967732a20302a8e36adfd", + "e8e48e1db8cec337e8791cc2e68f9b1fbddc9de6ce93a96418c7eddac5668986", + "4c693376594f51065b3646cc1f959db5aa94748caebce89722cd1a029d6b833c", + "cb219a421ac01eeca07a540f39c30cc0b96b4728468967732a20302a8e36adfd", + "031a387bd85a1134c44f8ec08559def796e77b5babf0ea5a19b32037df55c201", + "c950e869bb8a35b41ef1ddd0cef4e6db0f671c19188a25e7de32b6d36f0c6c8d" + ] + }, + { + "trial": 7, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3006heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.33, + "first_frame_sha256": "4afbd46b4faa0d0a2577eadd00c5da8c019d5e2e9cfed59239c73c247e6bb949", + "frame_sha256": [ + "312a637c500796729d2cf5df2c171f2a09ce2c43edc577b412fa0299162902bd", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "8327481a04458f071727de6d7f6b982663d46b926e8dd0dd51952ba98e94d5da", + "4afbd46b4faa0d0a2577eadd00c5da8c019d5e2e9cfed59239c73c247e6bb949", + "abb6700ad3f0cbf4252459887066762cc8d0b95e99f7203f5c148c231ff614a2", + "8327481a04458f071727de6d7f6b982663d46b926e8dd0dd51952ba98e94d5da", + "3c378da29d8c1e289cad7df71a552cc0009b81becd122788b30b3fa3e52d02b3", + "abb6700ad3f0cbf4252459887066762cc8d0b95e99f7203f5c148c231ff614a2", + "bb1753c97aaf733cce1f2002937bf4296fac907dd20dea1546bf627a4bd6e5a8", + "3c378da29d8c1e289cad7df71a552cc0009b81becd122788b30b3fa3e52d02b3", + "c30a5b4cee1b0a96b7472467bd3b6c8f633d13055755f45097e2b344b05f0c4c", + "bb1753c97aaf733cce1f2002937bf4296fac907dd20dea1546bf627a4bd6e5a8", + "d44a52b74b98da7c24a9170d064947fe0673a1c4f0d28cc915426a1e6148e541", + "c30a5b4cee1b0a96b7472467bd3b6c8f633d13055755f45097e2b344b05f0c4c", + "b55f3ced3a88906592ba776724eea6643b48779a79d16ed93716edb99739a6ac", + "312a637c500796729d2cf5df2c171f2a09ce2c43edc577b412fa0299162902bd" + ] + }, + { + "trial": 8, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3007heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.42, + "first_frame_sha256": "ad6b35236712e6e0edd2d2166a3c5a78cc74d652f373c7a18f56c35fbbd63e4c", + "frame_sha256": [ + "6a9cdde9fe821d040f805683d81d43bc61f612d1edd11e7dbb114427b9be05d0", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "a8b31e1dacc51b7940c9d2a1c5868da58bb9107c22b7e67b6ca22ee2d7101ad7", + "ad6b35236712e6e0edd2d2166a3c5a78cc74d652f373c7a18f56c35fbbd63e4c", + "ef68bc3d3b9662ba1a30f95e250c27fe64344876b96e79e522c259ba1d18782f", + "a8b31e1dacc51b7940c9d2a1c5868da58bb9107c22b7e67b6ca22ee2d7101ad7", + "a65c8dfb10a0d64832ae5cdfc8d8954e910a05349983a5b628a288aa2868bcbb", + "ef68bc3d3b9662ba1a30f95e250c27fe64344876b96e79e522c259ba1d18782f", + "770625a3ee6a70a45fca5af3078800c98052394f3eb42a647951bc0e43aa6a63", + "a65c8dfb10a0d64832ae5cdfc8d8954e910a05349983a5b628a288aa2868bcbb", + "4d50524c35e8a4d3f42ba48438501acb791a01da347a497b359f9a871c8112c6", + "770625a3ee6a70a45fca5af3078800c98052394f3eb42a647951bc0e43aa6a63", + "db78070346f898185846d66370fea41940aea065f173f57c87df4851ed7c0202", + "4d50524c35e8a4d3f42ba48438501acb791a01da347a497b359f9a871c8112c6", + "0c70dbd26ec5b5878c7775d0cb92d665ebb764e10e97a3df8df663711082b521", + "6a9cdde9fe821d040f805683d81d43bc61f612d1edd11e7dbb114427b9be05d0" + ] + }, + { + "trial": 9, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3008heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.3, + "first_frame_sha256": "dde3daa0d44dc940125cac7a9209aca4795d50d2250c81ed010184907ba2a9ec", + "frame_sha256": [ + "49ae39356945c3a67f5f77a866cb7c91c356ff58591daf90caa9702d6041ecde", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "37712eafe472ffdefb85855b1f3f218149da067155cadbd46a0241fa671fc457", + "dde3daa0d44dc940125cac7a9209aca4795d50d2250c81ed010184907ba2a9ec", + "0851e99ed2130fd5e3fcf07908b7d539f1a960c263075f31142682d9544ff5e3", + "37712eafe472ffdefb85855b1f3f218149da067155cadbd46a0241fa671fc457", + "4df9b77c135fd58287f2c29b905a1348f430cf212294a907bf3317d7889d8ff9", + "0851e99ed2130fd5e3fcf07908b7d539f1a960c263075f31142682d9544ff5e3", + "4a659d260cf4078e004d7aaa2d9636c219eb63caef9642b1e82e273ab26c07d0", + "4df9b77c135fd58287f2c29b905a1348f430cf212294a907bf3317d7889d8ff9", + "4fdaa866e5922e5d85ab2af97773bf44371dd2ed8fff4354fac0241e0fdd25a9", + "4a659d260cf4078e004d7aaa2d9636c219eb63caef9642b1e82e273ab26c07d0", + "b22f6d26792cb8d4d58cef46c24354d9287b0b48c15bbcdf38147610c7f1b5c3", + "4fdaa866e5922e5d85ab2af97773bf44371dd2ed8fff4354fac0241e0fdd25a9", + "9035e56c0ddbfe3410c1d3808834af3cdebe0dd055f91e28fd460bea07d77089", + "49ae39356945c3a67f5f77a866cb7c91c356ff58591daf90caa9702d6041ecde" + ] + }, + { + "trial": 10, + "condition": "healthy", + "drift": "none", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3009heal", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.73, + "first_frame_sha256": "011b4ab8bf41413ded5f82d2840c32a0cae3c60c9fcfd1c3dbea0c38165db4a3", + "frame_sha256": [ + "37d9e27777c9860accded8af58fccfa43356b180dda3515243b0ab934944ebba", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "d832c3663ba3ab07747742a404f26a7824f0cf12fb393869487b2e719a0ee377", + "011b4ab8bf41413ded5f82d2840c32a0cae3c60c9fcfd1c3dbea0c38165db4a3", + "2950d33d93a0d020c40fb39e58f4aea3d4b9b3b6bd7028a6d6415cdf6d0149a9", + "d832c3663ba3ab07747742a404f26a7824f0cf12fb393869487b2e719a0ee377", + "9a752501907d94cc7ac625c5dd055e1de913e836a791034abde920e368f0ddfc", + "2950d33d93a0d020c40fb39e58f4aea3d4b9b3b6bd7028a6d6415cdf6d0149a9", + "3639be36d3bc33656c14203257b64102b297e2b25ed1e59054598a4188aab275", + "9a752501907d94cc7ac625c5dd055e1de913e836a791034abde920e368f0ddfc", + "39223360b035c53ce2c71898a45c0b9ccf3ea5eb18a676afebda1b86e4a5fbdf", + "3639be36d3bc33656c14203257b64102b297e2b25ed1e59054598a4188aab275", + "376c26c36004266ced84e9c0dadd81d7cc7c7a8e0eb7d544363ab2bbd9e00d5d", + "39223360b035c53ce2c71898a45c0b9ccf3ea5eb18a676afebda1b86e4a5fbdf", + "1a24bfbc1a18bd183fb5964ae81612f2828c4451fed63f5dbb011500a5519629", + "37d9e27777c9860accded8af58fccfa43356b180dda3515243b0ab934944ebba" + ] + }, + { + "trial": 11, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3010reor", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.48, + "first_frame_sha256": "dfaf1553df01474439d5e2fef5a996085784aafa1452914d4661f736032efcd4", + "frame_sha256": [ + "ecf3b4d0f8c6174a91325473332f67becc95d586bdcfd2fbc2947e1ee2df694e", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "9ff160fc9ee118f63b641214695e6a1c05d0d8ab37a38c133685716755046329", + "dfaf1553df01474439d5e2fef5a996085784aafa1452914d4661f736032efcd4", + "3192ed75de8c89c4b0f6a2a0286ec35ca765f8165d0de5c925bf1001066cf29f", + "9ff160fc9ee118f63b641214695e6a1c05d0d8ab37a38c133685716755046329", + "79628e652a16acdc0d5901fc2071e03a0876e33d1ed699d1aa4291dc441b1b34", + "3192ed75de8c89c4b0f6a2a0286ec35ca765f8165d0de5c925bf1001066cf29f", + "96327d2f4d890935f1988b0653e79e383adb1c87c22a24e21f6706e849f94087", + "79628e652a16acdc0d5901fc2071e03a0876e33d1ed699d1aa4291dc441b1b34", + "1c7535bdf359fa43b07afe45e4aaa2e145e018d416e41da78013f2731e41a0d6", + "96327d2f4d890935f1988b0653e79e383adb1c87c22a24e21f6706e849f94087", + "21bb2a47d7431bfb6b0840d7095ae712ea8af2c5841d1f27017958dac80da056", + "1c7535bdf359fa43b07afe45e4aaa2e145e018d416e41da78013f2731e41a0d6", + "f8c558947d9124dde00d00b93e662f396c6cadd9709a5c7137315d365d35636d", + "ecf3b4d0f8c6174a91325473332f67becc95d586bdcfd2fbc2947e1ee2df694e" + ], + "first_frame": { + "file": "pn_reorder_first.png", + "sha256": "dfaf1553df01474439d5e2fef5a996085784aafa1452914d4661f736032efcd4", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 12, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3011reor", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.5, + "first_frame_sha256": "225c17a95e4b427c3a90b0a99a4fc2db4f854035ea9bd7f0c3836360144e3cdf", + "frame_sha256": [ + "26cdbd7bf2d215a020d0b638f378cec15e7ed46e2c7952dbc7037f97c7fd10b3", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "59f92b41e4a2435948a69827f2ccd1f2e7f4dd9177ab28d8c964dc7fdf0ba752", + "225c17a95e4b427c3a90b0a99a4fc2db4f854035ea9bd7f0c3836360144e3cdf", + "33143ee653f0bb404ec0adbb230db5af005af2ad4a2fc339582957b4f1f816be", + "59f92b41e4a2435948a69827f2ccd1f2e7f4dd9177ab28d8c964dc7fdf0ba752", + "3e4e6340bc2c6dd4f26349c212c4c22bbf828dbd18005e422dec067a6111c248", + "33143ee653f0bb404ec0adbb230db5af005af2ad4a2fc339582957b4f1f816be", + "20c5ca3db093066cb451a64e940e7d6fafae3c0b24baef9e0b0129358cdd26ca", + "3e4e6340bc2c6dd4f26349c212c4c22bbf828dbd18005e422dec067a6111c248", + "c664e891e971ae076d6fc1c2f31d0845540ca32cc3030904fe716595d79cf264", + "20c5ca3db093066cb451a64e940e7d6fafae3c0b24baef9e0b0129358cdd26ca", + "cc3596a9e35abe68af8786ef3f19e3bbf8c3d9844a6adfd5423dea9b1c765118", + "c664e891e971ae076d6fc1c2f31d0845540ca32cc3030904fe716595d79cf264", + "b7d302a936c7859522fda7c9aca9a0f84b38c805b333fbcee92b12d7dcf18ca7", + "26cdbd7bf2d215a020d0b638f378cec15e7ed46e2c7952dbc7037f97c7fd10b3" + ] + }, + { + "trial": 13, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3012reor", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.53, + "first_frame_sha256": "94520bb2a92123ab2998abd0f4b26956a6a28f4905890576e78f21b02361c6b6", + "frame_sha256": [ + "467a151031c7c2fb2feed3b8fd8728965f2b8ff289b5a587f5b3995827fbae6c", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "5f3c8c87934373877926ae76c8ca90972d0276d6a1c1f8558c61e1b8ee3549b5", + "94520bb2a92123ab2998abd0f4b26956a6a28f4905890576e78f21b02361c6b6", + "c36ce01ef188862bce18c680c4f39a50311010775fb05de946f66d990098cc6a", + "5f3c8c87934373877926ae76c8ca90972d0276d6a1c1f8558c61e1b8ee3549b5", + "cb2984ce92ce7c172bb9aa7bfc4516e7a2761c8356d1d8388c795b618f14e374", + "c36ce01ef188862bce18c680c4f39a50311010775fb05de946f66d990098cc6a", + "e921e64d1a436c2f59a943309803c3704087d4788dd3a5e810195da0cf118c7f", + "cb2984ce92ce7c172bb9aa7bfc4516e7a2761c8356d1d8388c795b618f14e374", + "3197ffcfbcd9326201532804b4813fc49d98fed296c543d5c7deafce5f327937", + "e921e64d1a436c2f59a943309803c3704087d4788dd3a5e810195da0cf118c7f", + "a3fe396d1c6b6b23f935f1fc3f96d291b50763bdb082f57128a05b1308b96b59", + "3197ffcfbcd9326201532804b4813fc49d98fed296c543d5c7deafce5f327937", + "4a0c40e3a78ad1daf2bbb5e350b880d908414e2429ca40c0a7c97c0daabd8d1c", + "467a151031c7c2fb2feed3b8fd8728965f2b8ff289b5a587f5b3995827fbae6c" + ] + }, + { + "trial": 14, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3013reor", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.41, + "first_frame_sha256": "0b61061a0fc056abc2df87b46a7adfc1411244bb0d8c8cf4bdacd735bbf80223", + "frame_sha256": [ + "f84688a0e44ca7907eb331f327bac85c86ac8943b2601d32f3256a1c45d629be", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "0e966e1b290c26a2a0ead3179bb7a483679243faeeffc751f55b2c718aad4b8c", + "0b61061a0fc056abc2df87b46a7adfc1411244bb0d8c8cf4bdacd735bbf80223", + "e7b29ed8faf7b692512403d85a8fcc571fb1b57f9960db0c186cf195bbe7796d", + "0e966e1b290c26a2a0ead3179bb7a483679243faeeffc751f55b2c718aad4b8c", + "5ef8cffc908b5e11805c3c221d50c5e74f2370148d1dfa7394431b3f84195d22", + "e7b29ed8faf7b692512403d85a8fcc571fb1b57f9960db0c186cf195bbe7796d", + "6418dbbd8d607757445e18131d0d5421238c444c2a7f41292d8318f864aa0bc5", + "5ef8cffc908b5e11805c3c221d50c5e74f2370148d1dfa7394431b3f84195d22", + "708838256d3f0d28393214a7418aa5aaa3d20e03570d3b05fc4e6b3711e23772", + "6418dbbd8d607757445e18131d0d5421238c444c2a7f41292d8318f864aa0bc5", + "5f19e5f99b839a6f87a239b27f621087b4a79b6121d32be5cb29ae2ea58c767c", + "708838256d3f0d28393214a7418aa5aaa3d20e03570d3b05fc4e6b3711e23772", + "32aa162b1c5255666d876ee44aa8a527afabdfcf08bda61c1c01bee0b91210fe", + "f84688a0e44ca7907eb331f327bac85c86ac8943b2601d32f3256a1c45d629be" + ] + }, + { + "trial": 15, + "condition": "reorder", + "drift": "reorder", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3014reor", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.06, + "first_frame_sha256": "218bd1d6f8cc63b9e32009e02b63c6c26f4186d2712eafbb2fcf575e79aa1b1c", + "frame_sha256": [ + "b21173b4328adbde5e80ee92a5c09290a5b5491600a2c676a1b4831f2cf3f5c6", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "6156fedd8e8869ad16dacbc1ea801bb9f348facfd3094e390daf8013429f1e4a", + "218bd1d6f8cc63b9e32009e02b63c6c26f4186d2712eafbb2fcf575e79aa1b1c", + "5f9b4a6e0eac6c2483ca6e85a6264d5eeb722d612a8e8711c5956079bdc96333", + "6156fedd8e8869ad16dacbc1ea801bb9f348facfd3094e390daf8013429f1e4a", + "75c5fffc14b31dd0f2893c75a83841a8c8d07ff05187e528bed11731e640a711", + "5f9b4a6e0eac6c2483ca6e85a6264d5eeb722d612a8e8711c5956079bdc96333", + "1b0457183ae9161cdc04c638310a1e6fd98911bf498f77066ffc19ac9c5da2af", + "75c5fffc14b31dd0f2893c75a83841a8c8d07ff05187e528bed11731e640a711", + "70ff95359b732da1a0b53019942c4965d469e0b88ea1548f4548ab8d6fab1178", + "1b0457183ae9161cdc04c638310a1e6fd98911bf498f77066ffc19ac9c5da2af", + "d1d3d220b2ac73f24e1b4252cd273796a743fde577127720942c986f96038161", + "70ff95359b732da1a0b53019942c4965d469e0b88ea1548f4548ab8d6fab1178", + "b637b4878426b8e42f30835a61943b7c0ecdb68bec95be10dac452a99274ae36", + "b21173b4328adbde5e80ee92a5c09290a5b5491600a2c676a1b4831f2cf3f5c6" + ] + }, + { + "trial": 16, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3015deco", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:46 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 11.38, + "first_frame_sha256": "0a1e9bbaf4278c6aec136213aee26267321ba3349481464e3dbe3d29c57fc2b0", + "frame_sha256": [ + "a2355b05b3f48010cb8c0f11559259d03d22f90e1071b1bb10c0af8b6136b6ba", + "0a1e9bbaf4278c6aec136213aee26267321ba3349481464e3dbe3d29c57fc2b0", + "1a19e569834f4be9c33fe6be9fe8f2b49ac32cfd2edb140dc2595bf923facdc6", + "a2355b05b3f48010cb8c0f11559259d03d22f90e1071b1bb10c0af8b6136b6ba", + "fe84539126729df28fa8be75977cf0dfd88d6e3ec38d45521546af7fd851eeae", + "1a19e569834f4be9c33fe6be9fe8f2b49ac32cfd2edb140dc2595bf923facdc6" + ], + "first_frame": { + "file": "pn_decoy_first.png", + "sha256": "0a1e9bbaf4278c6aec136213aee26267321ba3349481464e3dbe3d29c57fc2b0", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 17, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3016deco", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:47 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 9.7, + "first_frame_sha256": "7c7fef04960f3f5bb676026eeb2a92a0ddd97ab80b9e9ec540586de351ac7dd3", + "frame_sha256": [ + "e3cd56569c0a9938576d68d126956eadc28698aed3507a98fe57ca0c051664fe", + "7c7fef04960f3f5bb676026eeb2a92a0ddd97ab80b9e9ec540586de351ac7dd3", + "10a76fca5b079f65491aac900073c09592473f82dfccc628cecaa66c46be9886", + "e3cd56569c0a9938576d68d126956eadc28698aed3507a98fe57ca0c051664fe", + "1109ac2023f3738db205696427469bcd0b2c8f9b87c6a68b91c111df04041d9f", + "10a76fca5b079f65491aac900073c09592473f82dfccc628cecaa66c46be9886" + ] + }, + { + "trial": 18, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3017deco", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:47 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 9.47, + "first_frame_sha256": "a88631688e4beb989411d43b4a98d446e07d93ac5c31e15ebbcbcd54ef9b4128", + "frame_sha256": [ + "4f52277a6f7b8389ccc850bb8755af7c8791c0db020034336f3771d0c8ac2857", + "a88631688e4beb989411d43b4a98d446e07d93ac5c31e15ebbcbcd54ef9b4128", + "c4d554baf0c8e2b9bcb7cae724049f988aca0c2651268fe079dd11582d03f2ec", + "4f52277a6f7b8389ccc850bb8755af7c8791c0db020034336f3771d0c8ac2857", + "95c68078c49873e4de880a1d6e749bd45e654948f28ce1667d2d9cb199949d57", + "c4d554baf0c8e2b9bcb7cae724049f988aca0c2651268fe079dd11582d03f2ec" + ] + }, + { + "trial": 19, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3018deco", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:47 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 9.47, + "first_frame_sha256": "e752bbff7798066d57f66be75c38d2019ae492f96f4090be1386cefa0d45b2ee", + "frame_sha256": [ + "afebbe0f6736231f6485fbf853e384fc95f507a4b97d099d3ceffc223eb1b5d6", + "e752bbff7798066d57f66be75c38d2019ae492f96f4090be1386cefa0d45b2ee", + "0b4b41dc99a6ada392e21c569006ba584b878938d1935e6923f6330490b2797f", + "afebbe0f6736231f6485fbf853e384fc95f507a4b97d099d3ceffc223eb1b5d6", + "36d8032f0ea73d3ce5afd7ce85287f2d649512aac66f753975c8a48a3042fc5e", + "0b4b41dc99a6ada392e21c569006ba584b878938d1935e6923f6330490b2797f" + ] + }, + { + "trial": 20, + "condition": "decoy", + "drift": "decoy", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3019deco", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": false, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_002", + "intent": "click 'Search'", + "reason": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Anderson", + "1972-10-04", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Anderson (1972-10-04)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:48 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 9.42, + "first_frame_sha256": "3402c1b2a3a095ef0c8f22dcc1552715bb24aeab492311319413517c140c7faa", + "frame_sha256": [ + "6221c121ce09a011045b63a61040f87f59bf5d8e7096d82463cd555f28a33999", + "3402c1b2a3a095ef0c8f22dcc1552715bb24aeab492311319413517c140c7faa", + "04224f4328ff8f62a4af42f94b94d499195208d3dc77b5bfdcffaac462251b5c", + "6221c121ce09a011045b63a61040f87f59bf5d8e7096d82463cd555f28a33999", + "6eea7629ed6c03b24abe70e6ff59bd16f46c8978d9250e3c10cb05e66c583432", + "04224f4328ff8f62a4af42f94b94d499195208d3dc77b5bfdcffaac462251b5c" + ] + }, + { + "trial": 21, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3020sibl", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:48 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 6.59, + "first_frame_sha256": "7593772fbbf016e5fff1f0c2a3e77ee72a80966672aaecf40c8999e92c358395", + "frame_sha256": [ + "d7c5155659854e04505c1f398e5370bd8c9becf6917aae78bdb4fc516b7cfb93", + "7593772fbbf016e5fff1f0c2a3e77ee72a80966672aaecf40c8999e92c358395", + "28efba5cf0b3e68b620272c33fda82bad4a063b8f5bbacf1643380aab9fa9dc5", + "d7c5155659854e04505c1f398e5370bd8c9becf6917aae78bdb4fc516b7cfb93", + "988f855324771175c4f7d0ffa7ca610511b7c21340f0e156e46f29bef71d21ef", + "28efba5cf0b3e68b620272c33fda82bad4a063b8f5bbacf1643380aab9fa9dc5", + "988f855324771175c4f7d0ffa7ca610511b7c21340f0e156e46f29bef71d21ef", + "988f855324771175c4f7d0ffa7ca610511b7c21340f0e156e46f29bef71d21ef" + ], + "first_frame": { + "file": "pn_siblings_first.png", + "sha256": "7593772fbbf016e5fff1f0c2a3e77ee72a80966672aaecf40c8999e92c358395", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 22, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3021sibl", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:49 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 5.67, + "first_frame_sha256": "e1ad26d9a539c859a0cc07969eef664daf79c59fb52396970d2c2aa6f99ee99c", + "frame_sha256": [ + "1f87d5592e05d86305665749b5cadcc9c47562ea08370a8437cfd51dcf5051c6", + "e1ad26d9a539c859a0cc07969eef664daf79c59fb52396970d2c2aa6f99ee99c", + "cda10a72be70ca10c91025fe1a2a53257c69b9182f2f61db67aa6f7db6a9fa07", + "1f87d5592e05d86305665749b5cadcc9c47562ea08370a8437cfd51dcf5051c6", + "a52317ca8eaf0f8c42da50002a60f53728fc014a7b9ce65518f24b5497218f01", + "cda10a72be70ca10c91025fe1a2a53257c69b9182f2f61db67aa6f7db6a9fa07", + "a52317ca8eaf0f8c42da50002a60f53728fc014a7b9ce65518f24b5497218f01", + "a52317ca8eaf0f8c42da50002a60f53728fc014a7b9ce65518f24b5497218f01" + ] + }, + { + "trial": 23, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3022sibl", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:49 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 5.69, + "first_frame_sha256": "2a7021f6f2c92400942623e009aed919097c64fd2efbcba85b4f5553d2abf090", + "frame_sha256": [ + "7cd6076c4670754172e90085ae296fa3d4ed4a230b5f3bf6898d5a760d5182f9", + "2a7021f6f2c92400942623e009aed919097c64fd2efbcba85b4f5553d2abf090", + "b3b6ad72ae268827d00606adfce3fb53d3e2f06698f2f72702d35eebe760554f", + "7cd6076c4670754172e90085ae296fa3d4ed4a230b5f3bf6898d5a760d5182f9", + "0a61541e5596293f994cbb9d6fb0b6e7d0fd2949e71ff94de5867d66e3235760", + "b3b6ad72ae268827d00606adfce3fb53d3e2f06698f2f72702d35eebe760554f", + "0a61541e5596293f994cbb9d6fb0b6e7d0fd2949e71ff94de5867d66e3235760", + "0a61541e5596293f994cbb9d6fb0b6e7d0fd2949e71ff94de5867d66e3235760" + ] + }, + { + "trial": 24, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3023sibl", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:49 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 2.75, + "first_frame_sha256": "2d957576210e3c7a32b95b7a5528d8c5bcb9311f63f240f498416f6fe87473c4", + "frame_sha256": [ + "7cd6076c4670754172e90085ae296fa3d4ed4a230b5f3bf6898d5a760d5182f9", + "2d957576210e3c7a32b95b7a5528d8c5bcb9311f63f240f498416f6fe87473c4", + "b3b6ad72ae268827d00606adfce3fb53d3e2f06698f2f72702d35eebe760554f", + "7cd6076c4670754172e90085ae296fa3d4ed4a230b5f3bf6898d5a760d5182f9", + "0a61541e5596293f994cbb9d6fb0b6e7d0fd2949e71ff94de5867d66e3235760", + "b3b6ad72ae268827d00606adfce3fb53d3e2f06698f2f72702d35eebe760554f", + "0a61541e5596293f994cbb9d6fb0b6e7d0fd2949e71ff94de5867d66e3235760", + "0a61541e5596293f994cbb9d6fb0b6e7d0fd2949e71ff94de5867d66e3235760" + ] + }, + { + "trial": 25, + "condition": "siblings", + "drift": "siblings", + "bpp": 32, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3024sibl", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 2 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": false, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_003", + "intent": "click 'eil'", + "reason": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted", + "outcome": "halt", + "observed_texts": [ + "Patient Notes - Benchmark Harness", + "X", + "Neil", + "Search", + "15e1", + "qop", + "Sorensen", + "1983-01-19", + "IN", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorensen (1983-01-19)", + "Loaded 2 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "\u8425", + "3:50 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'" + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "stop_on_lookalike", + "must_save": false, + "elapsed_s": 5.77, + "first_frame_sha256": "f12c8a04ac92a5dbc49bf59cb49d983475b6f2d3b46e5200b24e3bb6e6ecc5d8", + "frame_sha256": [ + "6fd54a04e66425f7d244cf095df25fa7f8f0571c90f466bfb6d1bd86e7367b0d", + "f12c8a04ac92a5dbc49bf59cb49d983475b6f2d3b46e5200b24e3bb6e6ecc5d8", + "0087687620414170b1b16dd31d20336da8d02cce5b926c9d3cb97c6c382b9c35", + "6fd54a04e66425f7d244cf095df25fa7f8f0571c90f466bfb6d1bd86e7367b0d", + "52819c97fd19641efa0d0e6b25419e7fb96cf76f5833559d312966d06668d804", + "0087687620414170b1b16dd31d20336da8d02cce5b926c9d3cb97c6c382b9c35", + "52819c97fd19641efa0d0e6b25419e7fb96cf76f5833559d312966d06668d804", + "52819c97fd19641efa0d0e6b25419e7fb96cf76f5833559d312966d06668d804" + ] + }, + { + "trial": 26, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3025bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 19.95, + "first_frame_sha256": "7cbfbd57904b947f3ff915ec4463dc9f064fe04788afbe260c365a9e0b2b389f", + "frame_sha256": [ + "0a3dd2b5e543bca543c000c013521174f51ba603904337d0983c3111c54b1377", + "bd8794e9250645d923ed2eea990c31ef2b420e82a49ec132cbf0758f4a243051", + "174accb255642a84cdb7d8416a2128a3ec161921fbcf26667afdc968a427d827", + "7cbfbd57904b947f3ff915ec4463dc9f064fe04788afbe260c365a9e0b2b389f", + "03d91c85ed658452e2f4575960a69e93cee3cf512be07c94734285c22f0e3e93", + "174accb255642a84cdb7d8416a2128a3ec161921fbcf26667afdc968a427d827", + "31d2cdb3960ca8cdbcbc00350ceb7458e419e5307ab1b23c7a0232ad4f37c1ed", + "03d91c85ed658452e2f4575960a69e93cee3cf512be07c94734285c22f0e3e93", + "d229877fb532d4edd03284c5ab216c63f89fae508b3ca25c67c1c7352bd4d550", + "31d2cdb3960ca8cdbcbc00350ceb7458e419e5307ab1b23c7a0232ad4f37c1ed", + "58ed188eaa5c1a8bd3331792e0fb6b5ffdf932216c53cebfd3e905b6d0c3167d", + "d229877fb532d4edd03284c5ab216c63f89fae508b3ca25c67c1c7352bd4d550", + "19a05a6cb69064e8b36e7fdf9003730f0d1c30b3578191b7fcd7b957ce011f41", + "58ed188eaa5c1a8bd3331792e0fb6b5ffdf932216c53cebfd3e905b6d0c3167d", + "238b2af45a6613aa4a0f43002c1903e2edb0dbd0fc6ee56a14e108e691c4eebb", + "0a3dd2b5e543bca543c000c013521174f51ba603904337d0983c3111c54b1377" + ], + "first_frame": { + "file": "pn_bpp16_first.png", + "sha256": "7cbfbd57904b947f3ff915ec4463dc9f064fe04788afbe260c365a9e0b2b389f", + "size": [ + 1280, + 800 + ] + } + }, + { + "trial": 27, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3026bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 19.5, + "first_frame_sha256": "0090c28e77930a4e0885d5fd24fed6c047821f02fe8f4ccf625a8e74db7ca3a8", + "frame_sha256": [ + "6f4401a53a79672716d83420f1db8cd84352bf377ad6a31084ae16385d4eee73", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "876c3e61a4fdf585699f71a643ef8081b80d7f9a5f460d41613844a7e42768f7", + "0090c28e77930a4e0885d5fd24fed6c047821f02fe8f4ccf625a8e74db7ca3a8", + "70b90db1de701aa30d455238d311cf1974b9f758f80d0fbd9df855c1bdb8985d", + "876c3e61a4fdf585699f71a643ef8081b80d7f9a5f460d41613844a7e42768f7", + "14136852cb19991124c65a8efe4d7f4141d37c16de5cae8dfcac6f97cb1aa475", + "70b90db1de701aa30d455238d311cf1974b9f758f80d0fbd9df855c1bdb8985d", + "76016b9ea783d5e12bf11c06150a0308704cde1bd5546994db16fe477844a815", + "14136852cb19991124c65a8efe4d7f4141d37c16de5cae8dfcac6f97cb1aa475", + "ac1c60ce87b6cea802b218ac6eee9f4140b56ddbe055036e92468d479c3a0011", + "76016b9ea783d5e12bf11c06150a0308704cde1bd5546994db16fe477844a815", + "03fc80e3275a49e8ac3d4766a54182d80563f8a4631d5ad280346761340c3a81", + "ac1c60ce87b6cea802b218ac6eee9f4140b56ddbe055036e92468d479c3a0011", + "60b6b6c0dba24f00d0162a75ae68939bfd5c62b88e60446a9fef365ca2b6a80e", + "6f4401a53a79672716d83420f1db8cd84352bf377ad6a31084ae16385d4eee73" + ] + }, + { + "trial": 28, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3027bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.72, + "first_frame_sha256": "928bf1bc6e5cdf0fa7955503ff9514b77cac095b09f51a70e0e07d84dda1ad1e", + "frame_sha256": [ + "0fa6782c80393c29b372d126110381d9cfdfe32d2188c0c58fccd77f7a5d7d91", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "1bde6c36d0c43bb6092a8ab24dd731c3d42b74ee7b2481c24552bca4f079e9cf", + "928bf1bc6e5cdf0fa7955503ff9514b77cac095b09f51a70e0e07d84dda1ad1e", + "fcebabdaeddc432bb7d1005952e4ce3ae8ef5b8648e36799dca696e78d30b122", + "1bde6c36d0c43bb6092a8ab24dd731c3d42b74ee7b2481c24552bca4f079e9cf", + "c6cbab40b10596e350fd35127a1dde92dd96121c6cbdd3b9d874e5d44a00e570", + "fcebabdaeddc432bb7d1005952e4ce3ae8ef5b8648e36799dca696e78d30b122", + "0cb290ace61e0e793843f8685b4ee7e56155bfbc95e8e1df535ee8d5bc171021", + "c6cbab40b10596e350fd35127a1dde92dd96121c6cbdd3b9d874e5d44a00e570", + "0b8a7268bdfba87f120ac8b9db640a63a91fa13189e6274bb9240d260e3d0b90", + "0cb290ace61e0e793843f8685b4ee7e56155bfbc95e8e1df535ee8d5bc171021", + "10368bda15abf5abc7a53493a98c2eda1505fda499d125b4007368863c8cc608", + "0b8a7268bdfba87f120ac8b9db640a63a91fa13189e6274bb9240d260e3d0b90", + "bab20ebaa201aac4d0a1023d42671b7b1d183668b6ebac6886c2d9f725430879", + "0fa6782c80393c29b372d126110381d9cfdfe32d2188c0c58fccd77f7a5d7d91" + ] + }, + { + "trial": 29, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3028bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 17.19, + "first_frame_sha256": "928bf1bc6e5cdf0fa7955503ff9514b77cac095b09f51a70e0e07d84dda1ad1e", + "frame_sha256": [ + "549c712503f6775c9b9a7bb8215234eff37de2457be5835d216c6f250bc5cc19", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "1bde6c36d0c43bb6092a8ab24dd731c3d42b74ee7b2481c24552bca4f079e9cf", + "928bf1bc6e5cdf0fa7955503ff9514b77cac095b09f51a70e0e07d84dda1ad1e", + "fcebabdaeddc432bb7d1005952e4ce3ae8ef5b8648e36799dca696e78d30b122", + "1bde6c36d0c43bb6092a8ab24dd731c3d42b74ee7b2481c24552bca4f079e9cf", + "9858e1c9183200de8533b37a33728e6a31440edbe186ad52670437ed3d388399", + "fcebabdaeddc432bb7d1005952e4ce3ae8ef5b8648e36799dca696e78d30b122", + "e59ed9ee47b0e00f925f8bd446c86531a705e0429719f243de75fbd8e1701716", + "9858e1c9183200de8533b37a33728e6a31440edbe186ad52670437ed3d388399", + "355eb7e4df26a40427db59e1ee20bb4ce1724354a66de2437f62417f2c31ab68", + "e59ed9ee47b0e00f925f8bd446c86531a705e0429719f243de75fbd8e1701716", + "c3298175d9b153fcdc2be0edebbae225dea4d1f7ebf628a877d8358343670c4c", + "355eb7e4df26a40427db59e1ee20bb4ce1724354a66de2437f62417f2c31ab68", + "bdfa07d651f6fcb62a4274f4ed7a7684c36b124eafbe8a458c3cd81fb010be2f", + "549c712503f6775c9b9a7bb8215234eff37de2457be5835d216c6f250bc5cc19" + ] + }, + { + "trial": 30, + "condition": "bpp16", + "drift": "none", + "bpp": 16, + "desktop": [ + 1280, + 800 + ], + "token": "Oaun3029bpp1", + "success": true, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4, + "ocr": 1 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 145 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": true, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 560, + 489 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + } + ], + "execution_outcome": "COMPLETED_UNVERIFIED", + "heal_count": 1, + "halt": null, + "render_scale": null, + "db_outcome": "correct", + "record_detail": { + "note_holders": [ + 1 + ], + "record_check": { + "effect_verdict": "CONFIRMED", + "outcome": "verified", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "verified_save", + "must_save": true, + "elapsed_s": 18.74, + "first_frame_sha256": "d8addcd0d401d118757d776a13af59ff61dc4beaec0d0d2fc6f56541174a9dd7", + "frame_sha256": [ + "6c68d59410bc11c46b33c6e69b043b030ee66aaca6569954fef2622fac190250", + "7e5e410555d0a5c75be257607d70881b29eb2ad93614feaff34a386f25d3e0b7", + "9e84ca637875910a411ab91e204e97cc7f183e62a19104db578b192e7d09c897", + "d8addcd0d401d118757d776a13af59ff61dc4beaec0d0d2fc6f56541174a9dd7", + "9669942df966d3a287d10d0442e804870f922eda833231d53b031d64315393c5", + "9e84ca637875910a411ab91e204e97cc7f183e62a19104db578b192e7d09c897", + "dda8b0ff662c2fffafa12c8d8370cdf1d790ff31a183267cff852939878cfe3e", + "9669942df966d3a287d10d0442e804870f922eda833231d53b031d64315393c5", + "293241694753d10b4a0205c22f2375d8a0b9f90b5f0de96d2de8bcc04fd7a88a", + "dda8b0ff662c2fffafa12c8d8370cdf1d790ff31a183267cff852939878cfe3e", + "7b1373901f676383baeeb3186dbee096a3a3d36bd51a2d60d5d5b0600761bfd6", + "293241694753d10b4a0205c22f2375d8a0b9f90b5f0de96d2de8bcc04fd7a88a", + "6066cda91352aeb7e32f80d8449b5cd79def7cf1fc03ab21a3215b31d4781e62", + "7b1373901f676383baeeb3186dbee096a3a3d36bd51a2d60d5d5b0600761bfd6", + "8452e3ab2a7cf16340b090b4e10247d4794cc09f8a4241c3c5723deef9fb0660", + "6c68d59410bc11c46b33c6e69b043b030ee66aaca6569954fef2622fac190250" + ] + }, + { + "trial": 31, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun3030desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes-Benchmark Harness", + "X", + "Neil", + "Search", + "first", + "last", + "qop", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "3:53 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 21.81, + "first_frame_sha256": "4179509b305ce041d84e1c97bde962d3ac862fe782aa9b0ca8c151c2bd5e8cdd", + "frame_sha256": [ + "15b3a00073c6ff19bfd9c611d54817eff6164cf1df09a61ea053f14a7e09fa62", + "4179509b305ce041d84e1c97bde962d3ac862fe782aa9b0ca8c151c2bd5e8cdd", + "2ebb5d971b9b10dbd9d8fcb0ce1207a16d77f4581c61ecb8f476f16537dca0d8", + "15b3a00073c6ff19bfd9c611d54817eff6164cf1df09a61ea053f14a7e09fa62", + "9bfc5f74c611b5e818fceaf386fa31c9c409e811dc1a7fcee5e83493542991be", + "2ebb5d971b9b10dbd9d8fcb0ce1207a16d77f4581c61ecb8f476f16537dca0d8", + "17239bbe9cf84ed0d75a70804c274ceefae45c7f4150211c9e9fa56f04445a3c", + "ee034dcbac09bb46186cf20b870ad0eec884198e3e2b8e462157198bb6e1e892", + "cea91bb33132978073a9adbb4828cbe0d26faa3de0fcd5b07035d1d244738603", + "17239bbe9cf84ed0d75a70804c274ceefae45c7f4150211c9e9fa56f04445a3c", + "4f160b330cffbd40fa5504860b26ff7e7ef4590fb18285e6d85aaa137d0b0a45", + "cea91bb33132978073a9adbb4828cbe0d26faa3de0fcd5b07035d1d244738603", + "c1aa1fc772fb84cddff19ed9c7d4abcd4ec1d7a7d9715d577fac9c49e583bccb", + "c1aa1fc772fb84cddff19ed9c7d4abcd4ec1d7a7d9715d577fac9c49e583bccb" + ], + "first_frame": { + "file": "pn_desktop1440_first.png", + "sha256": "4179509b305ce041d84e1c97bde962d3ac862fe782aa9b0ca8c151c2bd5e8cdd", + "size": [ + 1440, + 900 + ] + } + }, + { + "trial": 32, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun3031desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "3:53 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 19.97, + "first_frame_sha256": "f3e5150e635c31556143c41d78e457dfe69e96dc78e679ccdc4b4ede5e552120", + "frame_sha256": [ + "441608141dec9da1f4e592d9207923ae9097c9c8a4ff72fd3805b867915a9018", + "f3e5150e635c31556143c41d78e457dfe69e96dc78e679ccdc4b4ede5e552120", + "9540ea964a6f0c6dff34a4027d1ece019a868cabb6ce16493c191f0902437b38", + "441608141dec9da1f4e592d9207923ae9097c9c8a4ff72fd3805b867915a9018", + "6fd67797733a75ead9f3ce13fef3ca99f1b0c43bfe410f7660ee48e75d9fffe2", + "9540ea964a6f0c6dff34a4027d1ece019a868cabb6ce16493c191f0902437b38", + "4ab67722bf6baee27326a47ea3a91e863b14f751864b354a15e6baf125b2ffbe", + "f53b1f114165c03ad2686f6aca7436bbf2e2cb95f7fc4ceab591c266d72b1d93", + "627ae7ad470144e702d85a16114cc86514e85247628e0c8500749773639ec69f", + "4ab67722bf6baee27326a47ea3a91e863b14f751864b354a15e6baf125b2ffbe", + "0c5cdf4d5e74fad2b9932d817b8fcbcb8fa26edf5b0b7c10c4d5d9d9a8732bc0", + "627ae7ad470144e702d85a16114cc86514e85247628e0c8500749773639ec69f", + "89143c9f816ddc8f1e4c1602f0576cbc8061ac121cadef74fb0ed1f8792afdcd", + "89143c9f816ddc8f1e4c1602f0576cbc8061ac121cadef74fb0ed1f8792afdcd" + ] + }, + { + "trial": 33, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun3032desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "N!", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "3:54 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 18.58, + "first_frame_sha256": "ef1ce280bab0735ce9636b039a3653144e233afd294ce28b04ed338b48dc7e49", + "frame_sha256": [ + "7cfbb7fc6c9af7bd21fea91e8b2ceb73b7535f248eb0b6292fd59d81bc5cb7f1", + "ef1ce280bab0735ce9636b039a3653144e233afd294ce28b04ed338b48dc7e49", + "0c9bb331a54a6612ca70e2ef49815bf2d0135b3d5b3e6507583255bde5b8300b", + "7cfbb7fc6c9af7bd21fea91e8b2ceb73b7535f248eb0b6292fd59d81bc5cb7f1", + "e314742087fc18f95c424dc351d9b68c0ac9ec026f3843c78718a7e1cbe0f1fc", + "0c9bb331a54a6612ca70e2ef49815bf2d0135b3d5b3e6507583255bde5b8300b", + "a1baac8f7a169c512b33a583acd029d13c5149ccc9eb9c486d23ce677dded04c", + "d08d1b7898a6e476295297ed77023164c19b7d82daef49abdf3a68f41f5e27dc", + "94cf738a095395fecb0234d565a14ee648bc21672d49477b84a77fb27f5025d6", + "a1baac8f7a169c512b33a583acd029d13c5149ccc9eb9c486d23ce677dded04c", + "dda74b257418c56b5897b49f97fbf0f52ae2f1ac45654da7a57eade40d3dbc2b", + "94cf738a095395fecb0234d565a14ee648bc21672d49477b84a77fb27f5025d6", + "df72a8f773406db5e631c5e939bda7ee58d3f02ba34245b914f7c8a2058662cc", + "df72a8f773406db5e631c5e939bda7ee58d3f02ba34245b914f7c8a2058662cc" + ] + }, + { + "trial": 34, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun3033desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "Patient Notes -Benchmark Harness", + "Neil", + "Search", + "first", + "last", + "qop", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "3:55 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 20.33, + "first_frame_sha256": "80cdb560cc6757ff759871ba711fa7d40a3998dc6a6ead46bbb584ab5e2b7491", + "frame_sha256": [ + "6ae825dfb1b6d21ab2f1a903c4383c9e13ba62148472f43d067faabfc4d2d986", + "80cdb560cc6757ff759871ba711fa7d40a3998dc6a6ead46bbb584ab5e2b7491", + "0e8605919b1f7f2479a35f70d789035834edcd7f3cb89ae8999c16659c44cd09", + "6ae825dfb1b6d21ab2f1a903c4383c9e13ba62148472f43d067faabfc4d2d986", + "4cc4a29032d8c48c140819c230a828c85fc46948bb1c7c0db0554c941f53d34e", + "0e8605919b1f7f2479a35f70d789035834edcd7f3cb89ae8999c16659c44cd09", + "953ab17fa8b4aba58088bd11f01816cc65f25015473c51bc8a7ef45a483ebbec", + "4cc4a29032d8c48c140819c230a828c85fc46948bb1c7c0db0554c941f53d34e", + "b8604087f5096d49c06d9a264e0ed71966e44f6d7e53505c95dba82c7eac4d4c", + "953ab17fa8b4aba58088bd11f01816cc65f25015473c51bc8a7ef45a483ebbec", + "0fedccdd928a5d8f8e27e29ec64ea12fad8330d92e00f58ad7613425cc416031", + "b8604087f5096d49c06d9a264e0ed71966e44f6d7e53505c95dba82c7eac4d4c", + "67292adaf18c72876e85a34140e8bec259f9257395cc0eee7ffe853072c3232d", + "67292adaf18c72876e85a34140e8bec259f9257395cc0eee7ffe853072c3232d" + ] + }, + { + "trial": 35, + "condition": "desktop1440", + "drift": "none", + "bpp": 32, + "desktop": [ + 1440, + 900 + ], + "token": "Oaun3034desk", + "success": false, + "terminal_outcome": null, + "error": null, + "model_calls": 0, + "rung_counts": { + "template": 4 + }, + "steps": [ + { + "step_id": "step_000", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 260, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_001", + "intent": "type 'Neil'", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_002", + "intent": "click 'Search'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 0.999, + "point": [ + 559, + 55 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_003", + "intent": "click 'eil'", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 115, + 144 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_004", + "intent": "click recorded visual target", + "ok": true, + "skipped": false, + "rung": "template", + "confidence": 1.0, + "point": [ + 311, + 400 + ], + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_005", + "intent": "type ", + "ok": true, + "skipped": false, + "rung": null, + "confidence": null, + "point": null, + "safety_halt": false, + "failure_category": null, + "detail": null + }, + { + "step_id": "step_006", + "intent": "click 'Save Note'", + "ok": false, + "skipped": false, + "rung": "ocr", + "confidence": 1.0, + "point": [ + 561, + 490 + ], + "safety_halt": true, + "failure_category": "safety_halt", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + } + ], + "execution_outcome": "HALTED", + "heal_count": 0, + "halt": { + "state_id": "step_006", + "intent": "click 'Save Note'", + "reason": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation", + "outcome": "halt", + "observed_texts": [ + "\u54c1", + "Patient Notes-Benchmark Harness", + "X", + "Neil", + "Search", + "first", + "last", + "qop", + "N!", + "Sorenson", + "1984-03-12", + "Clinical note:", + "Patient: Neil Sorenson (1984-03-12)", + "Loaded 1 patients", + "Save Note", + "ActivateWindows", + "Go to Settings to activate Windows.", + "8", + "O Type here to search", + "3:55 AM", + "10/29/2026" + ], + "completed_intents": [ + "click recorded visual target", + "type 'Neil'", + "click 'Search'", + "click 'eil'", + "click recorded visual target", + "type " + ] + }, + "render_scale": null, + "db_outcome": "no_write", + "record_detail": { + "note_holders": [], + "record_check": { + "effect_verdict": "REFUTED", + "outcome": "reconciliation_required", + "verification_tier": "independent_readback", + "checked_how": "system_of_record" + } + }, + "class": "false_stop", + "must_save": true, + "elapsed_s": 20.34, + "first_frame_sha256": "dbc363f9e8fdc476591bcbc34177bc540644f5e1de6ba3558b39e181b4d6aa9c", + "frame_sha256": [ + "640f0dd684fcc99423771553e161185a0fd890b5d517e6f1ed8df3b5bd42ace7", + "dbc363f9e8fdc476591bcbc34177bc540644f5e1de6ba3558b39e181b4d6aa9c", + "74454937fee0d4668a64919485a87def0290243e611283df5a38260ddc0eb472", + "640f0dd684fcc99423771553e161185a0fd890b5d517e6f1ed8df3b5bd42ace7", + "7ad4051206d1301c9627e58adce1da61db80e8fc34c96efc2d903a7d5c4c1036", + "74454937fee0d4668a64919485a87def0290243e611283df5a38260ddc0eb472", + "646e23c0d91708484e186c81b57cf4c8be0c5278a7994500a65bc5270bd9afd2", + "fe653f24d8fae3838b419aefe40d41cb3a61a2159e34c708e0310f737556ef82", + "1405c0b4dadca4d9863a6f794091c2502193ae20fe240ca74172f1ceb418e989", + "646e23c0d91708484e186c81b57cf4c8be0c5278a7994500a65bc5270bd9afd2", + "004453c317ea1dd310db64627983e02eaba50298f789cf07c71abd351beb0cd7", + "1405c0b4dadca4d9863a6f794091c2502193ae20fe240ca74172f1ceb418e989", + "cb20c3def35b281bb6cc89413adabc8d68c0aadd313be6c59a7b8eb31a28b6c1", + "cb20c3def35b281bb6cc89413adabc8d68c0aadd313be6c59a7b8eb31a28b6c1" + ] + } +] \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/environment.json b/packages/flow/benchmark/rdp_avc_real/results/environment.json new file mode 100644 index 0000000000..49476ffb62 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/environment.json @@ -0,0 +1,40 @@ +{ + "schema": "openadapt.rdp-avc444-environment.v1", + "captured_at": "2026-10-29T04:35:00Z", + "host": { + "os": "Microsoft Windows Server 2022 Datacenter Azure Edition 10.0.20348", + "azure_size": "Standard_D4ds_v4 (no GPU)", + "role": "remote desktop host, remote administration mode (no RD Session Host role, no CAL)", + "graphics_policy": { + "AVC444ModePreferred": 1, + "AVCHardwareEncodePreferred": 1, + "bEnumerateHWBeforeSW": 1, + "fEnableWddmDriver": 1, + "policy_names": [ + "Prioritize H.264/AVC 444 graphics mode for Remote Desktop Connections: Enabled", + "Configure H.264/AVC hardware encoding for Remote Desktop Connections: Enabled (no GPU present, so the encoder is software)", + "Use hardware graphics adapters for all Remote Desktop Services sessions: Enabled", + "Use WDDM graphics display driver for Remote Desktop Connections: Enabled" + ] + } + }, + "client": { + "os": "Microsoft Windows Server 2022 Datacenter Azure Edition 10.0.20348", + "azure_size": "Standard_D2ds_v4", + "remote_desktop_client": "Remote Desktop Connection (mstsc.exe) 10.0.20348.5386, mstscax.dll 10.0.20348.5386", + "h264_decode": "Server-Media-Foundation feature installed", + "session_settings": "1280x800 windowed, 32-bit colour unless the condition says 16-bit or 1440x900; network autodetect on; clipboard and audio redirection off", + "transport": "same Azure virtual network (TCP 3389, UDP transport accepted per RdpCoreTS event 131)" + }, + "negotiated_codec": { + "source": "host log Microsoft-Windows-RemoteDesktopServices-RdpCoreTS/Operational, event 162", + "connections_logged": 24, + "window_utc": ["2026-10-29T00:46:54Z", "2026-10-29T04:29:53Z"], + "message_on_every_connection": "The client supports version 0xA0600 of the RDP graphics protocol, client mode: 0, AVC available: 1, Initial profile: 2048. Server: vm-rdhost", + "interpretation": "AVC available: 1 with Initial profile: 2048 is the documented signal that AVC444 is in use (Microsoft, RDP 10 AVC/H.264 improvements, 2016). No event 170 (hardware encoder) was logged: encoding is software." + }, + "apps": { + "patient_notes": "scripts/desktop/patient_notes.ps1 (WinForms) with its SQLite record C:\\oa\\patients.db on the host", + "openemr": "openemr/openemr:7.0.3 (patch 4) with mariadb:11.4 on a separate Linux VM, shown in Microsoft Edge app mode on the host; GPL, run as an external app, nothing vendored" + } +} diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/ambiguity_refusal_pn_siblings_step_003.png b/packages/flow/benchmark/rdp_avc_real/results/frames/ambiguity_refusal_pn_siblings_step_003.png new file mode 100644 index 0000000000..4dac25ad0c Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/ambiguity_refusal_pn_siblings_step_003.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/emr_session_avc444.png b/packages/flow/benchmark/rdp_avc_real/results/frames/emr_session_avc444.png new file mode 100644 index 0000000000..acedd3ba79 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/emr_session_avc444.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/hover_reresolve_pn_desktop1440_step_006.png b/packages/flow/benchmark/rdp_avc_real/results/frames/hover_reresolve_pn_desktop1440_step_006.png new file mode 100644 index 0000000000..9e1609aa3f Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/hover_reresolve_pn_desktop1440_step_006.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/identity_precheck_emr_healthy_step_011.png b/packages/flow/benchmark/rdp_avc_real/results/frames/identity_precheck_emr_healthy_step_011.png new file mode 100644 index 0000000000..8b69db78c3 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/identity_precheck_emr_healthy_step_011.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/layout_reflow_emr_desktop1440_step_000.png b/packages/flow/benchmark/rdp_avc_real/results/frames/layout_reflow_emr_desktop1440_step_000.png new file mode 100644 index 0000000000..c8400d0887 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/layout_reflow_emr_desktop1440_step_000.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/pn_session_avc444.png b/packages/flow/benchmark/rdp_avc_real/results/frames/pn_session_avc444.png new file mode 100644 index 0000000000..58696f2e68 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/pn_session_avc444.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/readback_session_case1.png b/packages/flow/benchmark/rdp_avc_real/results/frames/readback_session_case1.png new file mode 100644 index 0000000000..de80e27365 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/readback_session_case1.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/state_check_pn_decoy_step_002.png b/packages/flow/benchmark/rdp_avc_real/results/frames/state_check_pn_decoy_step_002.png new file mode 100644 index 0000000000..9f0e812bb6 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/state_check_pn_decoy_step_002.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/frames/timing_codec_emr_healthy_step_014.png b/packages/flow/benchmark/rdp_avc_real/results/frames/timing_codec_emr_healthy_step_014.png new file mode 100644 index 0000000000..4d5636d093 Binary files /dev/null and b/packages/flow/benchmark/rdp_avc_real/results/frames/timing_codec_emr_healthy_step_014.png differ diff --git a/packages/flow/benchmark/rdp_avc_real/results/raw_files.sha256 b/packages/flow/benchmark/rdp_avc_real/results/raw_files.sha256 new file mode 100644 index 0000000000..7e51b5bcd6 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/raw_files.sha256 @@ -0,0 +1,188 @@ +083bf6dca3d0a2f3579c92ffb83a36c174e643e006d5eb1820c4596ea54133d0 emr-run-1/emr_bpp16_first.png +d27939f88bb9aff7794911bfe5303993b9f72c4ac5be3a925177940551615b48 emr-run-1/emr_bundle.zip +7ced4c6627d1dc4837aa582dacb938e53e7218f9843e98d2b95fb761c23a8220 emr-run-1/emr_desktop1440_first.png +36c576e719f09090473b3e08e566d00b4015cffa7b4aca67ce144c2ae36d09e0 emr-run-1/emr_healthy_first.png +dd723cef06bc48877bf586c5f3b302f34477d6b1cd20dfca2f0c31363b202a08 emr-run-1/emr_record_start.png +f765f472f9fa3b7a1b94b9902973fba32b53cc2f5deee39d2bc5a6ec425e011f emr-run-1/emr_run_bpp16.zip +f9e9df140d78a119f197328e87e5258f37abd9bc9851726cdcaa7c48753ed9b7 emr-run-1/emr_run_desktop1440.zip +b0f6094bead34cfeff5119f9c02e54d41e74390417ea808736bd3e3d838e5d8d emr-run-1/emr_run_healthy.zip +6d569c6b46472f6f9e8f302bd5e7cf767115823c38f52d2ec9a7f1c25fbbe123 emr-run-1/report_000.json +88f6dc92b7e139f25c540c20c770d5b92d250ba43b49f0d6ff4c3a098ed05f3e emr-run-1/report_001.json +b9761cb2e540a78b458492a01aaee3d0d9f6f897dce2702169992b83eebc9632 emr-run-1/report_002.json +43ddad236808897bc7a5c5465ffc11fb06dd0d484ee81d895061e188aa6f0518 emr-run-1/report_003.json +a39df137b638193d08f30be8258c454af994077b2a0d605c403d1cdb1c4c875c emr-run-1/report_004.json +d4eaf410ee14ef4896cf9c9244c8c29f7064657e6da940e21b16b3b89c40db9d emr-run-1/report_005.json +73e863cd04552dbda91d4bb372620d2df10c37bad6710b9456f741584c11d1a0 emr-run-1/report_006.json +aa830a3aa4bed4039258c6bf9daf9e58d25835a6331c6adf77166a12c3ebc643 emr-run-1/report_007.json +743d50ae61e3aff099352b918908ff37cae709ae19f0a993b05faf66abf4a81d emr-run-1/report_008.json +8b3967d9e9fc44ff4bc4df0224920027637b48904d6e713cf68fb7248e098a99 emr-run-1/report_009.json +739b04d9704e3f9d00c174a8cd353ed3704877528284b0907d595fabb085b2b1 emr-run-1/report_010.json +cdfb04f969ffb88f9f9a05b19ac81912a2cd3253b097fc88bf93d71e2ea59d5b emr-run-1/report_011.json +d1a9a12250418d2df41cf7a0801b7ac74ee1e0f7405bc5a6719cd758d821e167 emr-run-1/report_012.json +7c5c46dc00c39866e09be01ee08c0af1bec54012cda79356398ea91f54ad629b emr-run-1/report_013.json +4fc69226531835d22224eea47ddeddafc8fed31f8cfb648b9fbfac4c1477df8a emr-run-1/report_014.json +878ee54a3e9328c0daaaca93a1c6e8e0b4b6c1b15b3824f8a74433348555b110 emr-run-1/report_015.json +38e14cb0b7b8c10ddd6ccafe366e249627810d4a6f20f0f042dbdefbbd116970 emr-run-1/report_016.json +a7809c55faae0e9aa5cfe20093acb0816d5da4eebf4d414ab5b0639d748eb25b emr-run-1/report_017.json +66fbbd772dec0ed9b8e4e29939ac67b82f97797292a3c085c9ef4a0f237c44e4 emr-run-1/report_018.json +75096a86fdfe88a919ddc918676e792d0e9d076e7c3ba5be9b2485645d25449a emr-run-1/report_019.json +7d2224bdaa3c58949e167f00f588500f3ab82d2ba2ddf4d68e04a4b8c43eb031 emr-run-1/result.json +42d620be39133de2b3531155f043cc5f8c2e0734cdf90e98913534df623d209a emr-run-1/runner.log +7ee3108c86f3e7f58db1d5df092406cf0b59b26cd1f896deac32a77e3b3932e3 emr-run-1/trials.json +818de0dd7fe12070db472daf791ed626267d3e32bbcdb62cddf6514304db7487 emr-run-2/emr_bpp16_first.png +c79965aa4367cf0a781453a742ecdc798504215ab0a75aa684c8b6a728de4ae8 emr-run-2/emr_bundle.zip +a01da85833ef6e781feb2c5bec8e276b6e882acf7bbae4a7be524fd7a3a2d1cd emr-run-2/emr_healthy_first.png +e3ed2cfb5e8aecc6b1218453f7f3b3f5fd89ebea73bc31bcbe9ffc7e1906e887 emr-run-2/emr_record_start.png +7e317540523c7e5b76635e385b288d0734bb057b1aa49b27e300e68cadb90688 emr-run-2/emr_run_bpp16.zip +2e4c2d32de00faf82122820fa7dda4a57ffa6a9d98fb84aa1e2d9e2420ba4815 emr-run-2/emr_run_bpp16_false_stop.zip +058955958dc82741e6148913e2b75ab96adba966dd94c733a4bd3813e625cf9b emr-run-2/emr_run_healthy.zip +3288abe6931c42e9a4dea874cb969e542c67a18ccfc236022147689324c812d9 emr-run-2/report_000.json +24fcb42a9b8faad294b9b7197ba4d19801070c6489f7bbfae4baf28f17a4320d emr-run-2/report_001.json +f91b015b3bfff216b4bcee6ec314073df69fb45400160cf03d59005fd83f5fa4 emr-run-2/report_002.json +9c36606da0b5998bcd02373d9a09726cd9589cc06b4728f94f4fac4947da582e emr-run-2/report_003.json +6fb9f0d1e3217b08fbe23f54a3376d4ac0569046d436ebbe5e765231b18788f2 emr-run-2/report_004.json +8840cd377b9cbed1f78296a3fc7b88b04112a90d8e5a087b4fc9e45aefb2d3e1 emr-run-2/report_005.json +ab6e0c8c2d10d33af8089f3473dc9f8696bef159bcc487920c98b12b4dafc6f0 emr-run-2/report_006.json +e250cb5d83402777b50158d9e62cae813b7b4cde744dfbb2f213962a080511e8 emr-run-2/report_007.json +a4227518ac2caad473e631dd80b41120ce79333fe6301541c8bd285ae52772c6 emr-run-2/report_008.json +de345cc62e99044f277dd0180393c04d3ecb0c11c54d23493607af8d3697628b emr-run-2/report_009.json +dd260a4dce0bfd7df9afd3bd710c64bb98ee234b81ee5c8d418485ec52c78b8c emr-run-2/report_010.json +b2d0ece64399f2916422a0cf5be4a34f3d1f756a84d103a1ccac97cffc07cb53 emr-run-2/report_011.json +a9cebd461ab57f584d09db7558cf0e5c94b524cf3fab27b91cdd723a0f3d9dda emr-run-2/report_012.json +e7e79de63404bd12ca87ea3c063f2f5e11f76eb59c1d8c2b33127b6adebc1979 emr-run-2/report_013.json +0c23701683c4512793984f9d1dee270415699b8266b097040ad4d766791592f9 emr-run-2/report_014.json +4dee851826afc778d655b835ef678f1372a66bce81973801ca9217e169cf19c9 emr-run-2/result.json +42d620be39133de2b3531155f043cc5f8c2e0734cdf90e98913534df623d209a emr-run-2/runner.log +a13b687739e60c8722d1b7ee47cd5053e332c9d122fd197fa7417ab1d83c5382 emr-run-2/trials.json +2ffdfe3b20e89465f73786b17fff77def1b119237cecd429e072675c8ecda04c emr-run-3/emr_bpp16_first.png +4dde92566d213a6219bfd824b3fe7793634ab0cceee2ac29bee7d92c52994ba1 emr-run-3/emr_bundle.zip +52f4c1d7f58a8a83cf28ca457d198d0e29132eeadb4f74f2d6c3a40708e5a7ec emr-run-3/emr_healthy_first.png +8fbae9c95df04e26298767734a8d0a9805a63786a381b03cbdec920ee7e0e3fe emr-run-3/emr_record_start.png +71d2839eeff08d96f813d2386b7111f2b80bc405897c450c87334e71ae0dc87d emr-run-3/emr_run_bpp16.zip +a408708767d26a7a595074e5f3b4f30bfadb936bbe11817ba1abe81b95393833 emr-run-3/emr_run_healthy.zip +99a0a384ce3a717c9fedf604e60712aa6bd6d8bd3affdb5ff96d27839514ae2e emr-run-3/emr_run_healthy_false_stop.zip +a88f06c26bbfee7574c7b73fdc32674b6d2af95a6cd591e67aeabed9c95f7dad emr-run-3/report_000.json +9709da58cf33804172786892a963f7bbd6a464dc178f5430e45ea6ac3d423e0b emr-run-3/report_001.json +4329ff09f10e4f2c5a1b4bfab27ec808f469edefca9b0ae122fc0161d1f31ba7 emr-run-3/report_002.json +6ca064215e0701ea7712bba09258578c4c67596f024537440e72dfc4f6786a81 emr-run-3/report_003.json +3caae3365f242d601e80deedeb523449b44bc68f6ac53c7ede5546b93f40a6ea emr-run-3/report_004.json +54af96c3d998edb5e9bca8b8bd6f8bace3cd8b660438d34bf60f559a943469e3 emr-run-3/report_005.json +17bba37cb018e467075413ff139e848a5f45551373dd84f4bf9f626244060a5e emr-run-3/report_006.json +022f9007db89b6b6d036776757026d28d6999edc6f8b2166ff7c37994a5b7a88 emr-run-3/report_007.json +df0f0d5a2d0fcef359062fc61765f77dd47822ddbf1bd1c829fec2824408edf0 emr-run-3/report_008.json +d697425f2794960101b19445ea5f5ee198d2448b6945312f851790d1795d6f26 emr-run-3/report_009.json +8a7893a6463c5daac82a2ff4e8d30eced4ded023a287dcefd1e36c5b1aab5684 emr-run-3/report_010.json +2d9ffb8f665a10feb470756ad9b27174a6fdc05465de4da19ae58b3d3171f7f3 emr-run-3/report_011.json +f03c26e5cb7adad8f614edd9f36a9c03d547f64e0a48b1da701b7cb12db2fa28 emr-run-3/report_012.json +7f3901f011e0ae3be3858f00c69a8c951f37fc4e15d9a46d598d3e0fc0c020d4 emr-run-3/report_013.json +ed14097e129a0b91768c442f1c37258a7ec81df6913b58acdc693d6e8bfdfa28 emr-run-3/report_014.json +56944eaed119d561bb1f94a422dcf56c7e7f4189ee473040849aaa0836aa19ed emr-run-3/result.json +2a039b7026923eb52d73ebaaaece376d14d9b4242b0cdef9a2b6b26a077288fa emr-run-3/runner.log +fa785792a2da09a60697074c8a3ec4cbf374492a33abc4235a5b98241fd43722 emr-run-3/trials.json +142805e2f7cbff059782161169427198ac90a659a91d3248379a8678984caa7c pn-readback-1/pnread_case1.png +9e353e6b399d3de178082e31d7f7ef877456633eec74fa7b097dc179ad475ca7 pn-readback-1/pnread_case2.png +9b61595e6fe7791f0b153d6d440d93a08e015a37e84b80d33d77b080fd831ef4 pn-readback-1/readback.json +216bb07c9d1cd6321b45d5e793c4e7328e853e7e37dacde01f5b71c515a083e0 pn-readback-1/result.json +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 pn-readback-1/runner.log +b23db933136ee910ea1264ca2bf2705d371e2f676a5a48b389baeffa76d1eb1c pn-run-1/pn_bpp16_first.png +3accf4a0df5d4d0877fc66adf0c62db72f79339b11efc32d2039192732b352a2 pn-run-1/pn_decoy_first.png +e6e02a305e7cd6453f328226ebc99b55fe9bd32db59dc3e332f7797df906eb1e pn-run-1/pn_desktop1440_first.png +9c0ac4b1d38cafdeb380be043c31fd3ea2832cae9db24ae2d5d618ae0980e66a pn-run-1/pn_healthy_first.png +7d81032971b04eba44b7b7838957e2be0c2d1861afe20c960f54e4dc7784179c pn-run-1/pn_record_start.png +bccc9e8e162a07708b14af7f8070b28627a10842661dc3584958c424ac33ff26 pn-run-1/pn_reorder_first.png +00ab40490413286d9855d72731e835bc0daf21aae73c3ea537a57aec3ed5192b pn-run-1/pn_siblings_first.png +0dc9e7229cfb0e09bdbce59f35338be3656d305671ea479cd8f37ee1b3c6c53a pn-run-1/report_000.json +1259f8b782c725e41ea487384ccc430263dd69137533ab0294823933b8479790 pn-run-1/report_001.json +46a4440ba45c1ba23dcee5101289608615d8f43e67d3923eb426f0bd58fd25f0 pn-run-1/report_002.json +199f5020df0bef5db7784ce1cb886586be24235d4fb1ef0e4df514f76adf8572 pn-run-1/report_003.json +8132674844369923f3775e7a48a252230b11a38ab4db20fe605bb59d3be3c087 pn-run-1/report_004.json +00e974e1d27a8688395db281e6a38de25bdd66c0886cfab4703146c73068b3d5 pn-run-1/report_005.json +15b8b7bd859d1c168d99b178e37c8ddbf8cce282cd5521083d135329c2cf6cd1 pn-run-1/report_006.json +007ab7b2edcfb1320c7c2dc6aebd5652e17eac30854aec4ae585ba6e6bc51f04 pn-run-1/report_007.json +3dd4719fcbf3d4dd4d6d953eb56619a4529accbe543d1c688a15ed144149c88c pn-run-1/report_008.json +c78be3a76c336a715ae677d7cd18896ccc7537214a783b01e2191c4905669fb1 pn-run-1/report_009.json +ab8f1af6c0c58ee5efca0cd24c8fd653977656e5e349679d64b09064dcef9b62 pn-run-1/report_010.json +8c1ebceaf9ef7feba5372ac7b8dc2a767d0523dbef528f3cdd66e3fed59fb51a pn-run-1/report_011.json +8c66ef8beaef51072897f5e2f04996f1f03783afb128a39b6e41d970a8b209af pn-run-1/report_012.json +b4247478d2233e01d9ff3185e12b41513b8ad6019c796ebc1394ee81b5898288 pn-run-1/report_013.json +81a105d13e58aac3237fff8ef30b7502ddf3bb56a11006f6426378cf544b47a0 pn-run-1/report_014.json +922f31cdec4970f3f21e7d25b29ab8fae895bd4ac75a1a27c98238a5ba7abea3 pn-run-1/report_015.json +d06763cc56439697faf17da2958b2ecddcd4dc869ad7bae250a4afff48fcb2dc pn-run-1/report_016.json +7cf4085bcea438b56e76e2dba4bfe3055e314676e9786c664ccbcac252fdb66a pn-run-1/report_017.json +365e6ded91cf5f98322af7c596db77f71b5db033b651ee578c760f1a957929b5 pn-run-1/report_018.json +4e8e5a52f209b52c81ed35c8514cb0d3c0ba8accd0ec1dc9fc2a6d2163b0169d pn-run-1/report_019.json +f9ea74ccc42a83db0498ade2d5e42431aeaff6c095cdc4425f586050d23a82a3 pn-run-1/report_020.json +283cfa94d91db3f0bd6f610fab884300327e4ff4f2ed6517f28cb7577c26c7da pn-run-1/report_021.json +84f3596bb1ec20ffdc00668b6fc8600ace9ae677f85d338dbf1ffaa84a9b7a2d pn-run-1/report_022.json +9934219372741a9ea4e89acd225bd0d2a234a21b997b45c29e5170c06be023b6 pn-run-1/report_023.json +de6224c3a522c5f5fc1f10b71c4f39d5d307d8a54e0301cd0703cff483335d31 pn-run-1/report_024.json +606aac65f5864cf6862c242b271abe1433fcf96c1c7c36f023e9b73de629b5bf pn-run-1/report_025.json +e6bd3fc008d48f6b7c621f18777bbcb77a990882c70b5e5f840599fa78a8b77b pn-run-1/report_026.json +1d54a78b3ba76fc6734abc90ab15709ca690e7d8e50493d1e8682d835febc2fa pn-run-1/report_027.json +1ca78622a62bbafd572d78884a8fdb9ad8f930f1154133abbd7edfa9db13deb7 pn-run-1/report_028.json +1293e030d8e8bf4c70d0fd17ab397a328a26336a41a46b1b3a122f01c0ee3495 pn-run-1/report_029.json +1bcb324e009c3f02b03261afb1d9903988dbb44cfb11636b5eb257f753ce4e7e pn-run-1/report_030.json +b419976bedb11bf947bf960f97e98fa3e7cb0a4ff1b23abd33e0a9479275d7e0 pn-run-1/report_031.json +e67a57403b8321495454f461b10e65c3ef0e8398efb7f84400393d4447c4c334 pn-run-1/report_032.json +1069042fba1b8e2f551e636bc119e2cb7b3ebeabe20a5316105c72e3c69f5cdb pn-run-1/report_033.json +9e4de828ff7e713c35214c08fe4a37ead85615d4212d41d74d7cadd442227ae7 pn-run-1/report_034.json +c7389861c71c73a83769bf43755d1008fff67dd63345605ce0fecc66bf47effb pn-run-1/result.json +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 pn-run-1/runner.log +af6f97f6da4ba451480902a7e7f1670f1fc35fae7a20ebe619d08df5c66a7a7b pn-run-1/trials.json +7cbfbd57904b947f3ff915ec4463dc9f064fe04788afbe260c365a9e0b2b389f pn-run-3/pn_bpp16_first.png +fce6d3f19465a004944a4b0cca72a8fa356467c7aba32bba528eeebc7e0e1171 pn-run-3/pn_bundle.zip +0a1e9bbaf4278c6aec136213aee26267321ba3349481464e3dbe3d29c57fc2b0 pn-run-3/pn_decoy_first.png +4179509b305ce041d84e1c97bde962d3ac862fe782aa9b0ca8c151c2bd5e8cdd pn-run-3/pn_desktop1440_first.png +af672ce15e7db963d2caff403e9342ae4dc89907c3c85a4b257457dceccc11bc pn-run-3/pn_healthy_first.png +64d591be8fd143bce95e09c666d76572c1b7f18625c529b8f581c6a02092fe2b pn-run-3/pn_record_start.png +dfaf1553df01474439d5e2fef5a996085784aafa1452914d4661f736032efcd4 pn-run-3/pn_reorder_first.png +990c39e9280d5b344e44ab0052f49fd42dddcb3b535024f0fefd6c7ba668a68d pn-run-3/pn_run_bpp16.zip +1e9243cb2d498639d1fcbf8ae1a068ec5c0427bd6e98aadc9e69454b3d7a1ab4 pn-run-3/pn_run_decoy.zip +ddf7f2ceff04dc202c519964c28e26e07d65d61c574c1c798a1dd181aa99647e pn-run-3/pn_run_desktop1440.zip +12a5f83e6b9778d641b4daa0f369262383a187a12f940eb494b58c042c1ee341 pn-run-3/pn_run_healthy.zip +2c3a79094296b735f61a654c38ab71e8c2e62f486f379534b4b69819c8d8608d pn-run-3/pn_run_reorder.zip +27488de4ff9a3616aa95c68d22464c583b60680c69574bbe347a7ee8700ed18b pn-run-3/pn_run_siblings.zip +7593772fbbf016e5fff1f0c2a3e77ee72a80966672aaecf40c8999e92c358395 pn-run-3/pn_siblings_first.png +09f3ca785571ac9860a49374b34e6ce3af2ff85dd09cf3db74532597baefb78c pn-run-3/report_000.json +e5fb6d712360628ddbe03c630622a5e48a042daabc8feb2b375a051519cec230 pn-run-3/report_001.json +3f9940212a9e439ab6f68c68bb192abcca22eb708fd77dcbb25da0b33a924d99 pn-run-3/report_002.json +e66efd129e205e272f097056bd37c6731e42c6e488045206a1624aa47f28860c pn-run-3/report_003.json +8305b49da25087e6486ddc6f89c3ffe300c48bea24d32b27e14ca9a437bd79ba pn-run-3/report_004.json +1ecbeb22cd87e658ca2d64a76243231df42651d6a8596b1d6971e61a5a256893 pn-run-3/report_005.json +5543a28469c37316772c4ee2ce6acc652c080f885c48a1aeab21768e1b5a80ec pn-run-3/report_006.json +2704fd15cd010acd276d5f009de79062d69be4553f8b76870f65d90b5aad08a2 pn-run-3/report_007.json +a58d8b2e417d216ead9c002b04c5d82d2f2164b8925339211760ff206641afc1 pn-run-3/report_008.json +85613efd0f6e349d7cbf73cf793104d4c76ac8a864e6b2040f7aae02a3d58f7b pn-run-3/report_009.json +43db71d1a581671a01ff07c0154116f0940fbe3050699f349ef0e42e7c3db266 pn-run-3/report_010.json +2f362951f31e089182fad1258910471adb1b7ffde935082b5f6bb8108131ace4 pn-run-3/report_011.json +e3d42d845608f63fa58bc401e59f8dd2e1e09a845594055d2daa6e98f3644334 pn-run-3/report_012.json +edbce07180739e05159a0523dfa944053d835c2de9632159dd6ec9425385fcbd pn-run-3/report_013.json +5e8d1def3331f873362bff1224170ddd96d7780bdc649af134598d519f9e1f46 pn-run-3/report_014.json +847f2e0b8f2e7238b25b0005b4ab10610ab3a7ea3a4a2651281614a6be4527cc pn-run-3/report_015.json +20f3212e6c5ace550183d267690c2b401dfcd11396ee562da0aadc88d8d6f0c2 pn-run-3/report_016.json +c8ec6c5fdcc801effd9cfa1ca898e4ecc604df9fe2c700e9866cc04d37a249f0 pn-run-3/report_017.json +470df844047e9350cc04a445081490b81278c8ffcbb1d3169d2f3bf433480d41 pn-run-3/report_018.json +bfec58d3cf8115de031b0f4bc172c11eebb039e518d92ee001842eedac72642e pn-run-3/report_019.json +be59ebf27dae4e58cdaac190ebb89ee5a9aab73630c710b25cd09eba4beaa8a9 pn-run-3/report_020.json +fed9cabd8e905dda02f66a5bf550aa531644d937f527d93d87039cc99c63fc95 pn-run-3/report_021.json +fb443372051eb134cd93c30d6e35407c74a3c4575b9e377d744489a7d24e5245 pn-run-3/report_022.json +fcc67d6ce439f8ebcefb8a7003ad1e5663429f8cc42566bc106f232f6c8b9d6f pn-run-3/report_023.json +190e7b01a2d506b7b8e8abdef84fcbaaf3bd837e23d4637072ae78b2d3c31fa6 pn-run-3/report_024.json +a68b278411384ce218748908a9155d7465cceadec016a42ca3d5191410377323 pn-run-3/report_025.json +7cce7a3abf9b2715bce55b9032c8b8434a64b2536a4188a28974565d51ce0ce9 pn-run-3/report_026.json +b9124a792815edf19ee819b2a89c62ff16180ec265d50ca44d6e912cebd35e02 pn-run-3/report_027.json +66fbfb3e0bfdd2eb895d61cf7280d852c1739464ff4bf9ea50a5615227eecda1 pn-run-3/report_028.json +965276fbb591e70c80de33d086f8aa01fcfea2cb2435e5bd326cd8d2b7702eeb pn-run-3/report_029.json +44c3e25f4b489069363fc165f34a014e9cc094639fa71f162812278fdbca4ae7 pn-run-3/report_030.json +f19f96f66f1a5eba94146d741c0caab2c55fa736a0338ad2159e4737011ac9cb pn-run-3/report_031.json +085122dd9f4705022e94724b727043739a303390457298c54a8e2af041416221 pn-run-3/report_032.json +b9986ff751ab9d56cf5a53780a3e9978c08b0513c6d7e5834b0f07d0d97f4fe0 pn-run-3/report_033.json +89cab8c871429086b32f3c16a3e181ca79cbcd0f8bb13526daf95f14f56b5554 pn-run-3/report_034.json +a6e14771d8433f84bee0cdb16296e1974ca0955deb4af2475f2d39943cf81923 pn-run-3/result.json +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 pn-run-3/runner.log +b1c7bc5876b9d06fe18d3c68db1a9d71ff151b32fdbe62ce659e8fdad2688b40 pn-run-3/trials.json +cbe022ac1d9cd55436ad71f7b20f96130328f7c3086a143f167c8917d6e1135b pn-readback-2/pnread_case1.png +16715cb0ea250b3d5aeff879cdb99905c4651782229efce73b69f09100fd5fcf pn-readback-2/pnread_case2.png +143ea288c7758e32df2ebee671af35e2b7b2ee65a7729bb9a42f3c784b8ab039 pn-readback-2/readback.json +0a6373a1f79e3b29dcc28e6cc5b75e1a6ce87200b8428bd0d733eadef938fd69 pn-readback-2/result.json +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 pn-readback-2/runner.log diff --git a/packages/flow/benchmark/rdp_avc_real/results/results.json b/packages/flow/benchmark/rdp_avc_real/results/results.json new file mode 100644 index 0000000000..a56400faee --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/results.json @@ -0,0 +1,965 @@ +{ + "schema": "openadapt.rdp-avc444-campaigns.v1", + "conditions": { + "emr/bpp16": { + "trials": 10, + "verified_saves": { + "count": 7, + "per_100": 70.0, + "ci95": [ + 39.7, + 89.2 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "false_stops": { + "count": 3, + "per_100": 30.0, + "ci95": [ + 10.8, + 60.3 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "model_calls": 0, + "median_elapsed_s": 90.36 + }, + "emr/desktop1440": { + "trials": 5, + "verified_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 43.4 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 43.4 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 43.4 + ] + }, + "false_stops": { + "count": 5, + "per_100": 100.0, + "ci95": [ + 56.6, + 100.0 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 43.4 + ] + }, + "model_calls": 0, + "median_elapsed_s": 11.3 + }, + "emr/healthy": { + "trials": 20, + "verified_saves": { + "count": 8, + "per_100": 40.0, + "ci95": [ + 21.9, + 61.3 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "false_stops": { + "count": 12, + "per_100": 60.0, + "ci95": [ + 38.7, + 78.1 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "model_calls": 0, + "median_elapsed_s": 81.88 + }, + "pn/bpp16": { + "trials": 10, + "verified_saves": { + "count": 10, + "per_100": 100.0, + "ci95": [ + 72.2, + 100.0 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "false_stops": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "model_calls": 0, + "median_elapsed_s": 18.72 + }, + "pn/decoy": { + "trials": 10, + "verified_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "false_stops": { + "count": 10, + "per_100": 100.0, + "ci95": [ + 72.2, + 100.0 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "model_calls": 0, + "median_elapsed_s": 11.38 + }, + "pn/desktop1440": { + "trials": 10, + "verified_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "false_stops": { + "count": 10, + "per_100": 100.0, + "ci95": [ + 72.2, + 100.0 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "model_calls": 0, + "median_elapsed_s": 18.58 + }, + "pn/healthy": { + "trials": 20, + "verified_saves": { + "count": 20, + "per_100": 100.0, + "ci95": [ + 83.9, + 100.0 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "false_stops": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 16.1 + ] + }, + "model_calls": 0, + "median_elapsed_s": 18.28 + }, + "pn/reorder": { + "trials": 10, + "verified_saves": { + "count": 10, + "per_100": 100.0, + "ci95": [ + 72.2, + 100.0 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "false_stops": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "stops_on_lookalike": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "model_calls": 0, + "median_elapsed_s": 18.06 + }, + "pn/siblings": { + "trials": 10, + "verified_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "wrong_target": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "silent_wrong_saves": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "false_stops": { + "count": 0, + "per_100": 0.0, + "ci95": [ + 0.0, + 27.8 + ] + }, + "stops_on_lookalike": { + "count": 10, + "per_100": 100.0, + "ci95": [ + 72.2, + 100.0 + ] + }, + "model_calls": 0, + "median_elapsed_s": 6.59 + } + }, + "stops": [ + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 16, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 17, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 18, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 19, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 20, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 21, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "identity_precheck", + "step_id": "step_003", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 22, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "identity_precheck", + "step_id": "step_003", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed '1983-01-19 Sorensen' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 23, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "identity_precheck", + "step_id": "step_003", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 24, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "identity_precheck", + "step_id": "step_003", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 25, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "identity_precheck", + "step_id": "step_003", + "detail": "Identity check failed for step 'step_003' (click 'eil'): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Sorensen 1983-01-19' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 31, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 32, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 33, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 34, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-1", + "app": "pn", + "trial": 35, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 16, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 17, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 18, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 19, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 20, + "condition": "decoy", + "class": "false_stop", + "cause": "state_check", + "step_id": "step_002", + "detail": "Postconditions failed for step 'step_002' (click 'Search'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'Patient: Neil Sorenson (1984-03-12)' \u2014 run aborted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 21, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "ambiguity_refusal", + "step_id": "step_003", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 22, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "ambiguity_refusal", + "step_id": "step_003", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 23, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "ambiguity_refusal", + "step_id": "step_003", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 24, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "ambiguity_refusal", + "step_id": "step_003", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 25, + "condition": "siblings", + "class": "stop_on_lookalike", + "cause": "ambiguity_refusal", + "step_id": "step_003", + "detail": "OCR safety refusal for step 'step_003' (click 'eil'): OCR landmark evidence did not uniquely establish a target \u2014 no action was admitted" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 31, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 32, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 33, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 34, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "pn-run-3", + "app": "pn", + "trial": 35, + "condition": "desktop1440", + "class": "false_stop", + "cause": "hover_reresolve", + "step_id": "step_006", + "detail": "Step 'step_006' (click 'Save Note') resolved to (561, 490) after remote pointer hover, not the prepared point (560, 489); refusing a click whose target moved before actuation" + }, + { + "campaign": "emr-run-1", + "app": "emr", + "trial": 16, + "condition": "desktop1440", + "class": "false_stop", + "cause": "layout_reflow", + "step_id": "step_000", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + }, + { + "campaign": "emr-run-1", + "app": "emr", + "trial": 17, + "condition": "desktop1440", + "class": "false_stop", + "cause": "layout_reflow", + "step_id": "step_000", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + }, + { + "campaign": "emr-run-1", + "app": "emr", + "trial": 18, + "condition": "desktop1440", + "class": "false_stop", + "cause": "layout_reflow", + "step_id": "step_000", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + }, + { + "campaign": "emr-run-1", + "app": "emr", + "trial": 19, + "condition": "desktop1440", + "class": "false_stop", + "cause": "layout_reflow", + "step_id": "step_000", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + }, + { + "campaign": "emr-run-1", + "app": "emr", + "trial": 20, + "condition": "desktop1440", + "class": "false_stop", + "cause": "layout_reflow", + "step_id": "step_000", + "detail": "Postconditions failed for step 'step_000' (click 'ername'): expected screen state not reached (semantic drift) \u2014 failed: region_stable region=(554, 342, 303, 93) \u2014 run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 1, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 2, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 4, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 6, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_009", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 8, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 9, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 10, + "condition": "healthy", + "class": "false_stop", + "cause": "identity_precheck", + "step_id": "step_011", + "detail": "Identity check failed for step 'step_011' (click recorded visual target): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Preferred Name: Patient preferred name or name patier' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 12, + "condition": "bpp16", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 13, + "condition": "bpp16", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-2", + "app": "emr", + "trial": 14, + "condition": "bpp16", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_014", + "detail": "Step 'step_014' (click 'Unassigned'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-3", + "app": "emr", + "trial": 3, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_009", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-3", + "app": "emr", + "trial": 4, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_009", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-3", + "app": "emr", + "trial": 5, + "condition": "healthy", + "class": "false_stop", + "cause": "identity_precheck", + "step_id": "step_011", + "detail": "Identity check failed for step 'step_011' (click recorded visual target): a target was found positionally (rung 'geometry', confidence 0.90) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Preferred Name: Patient preferred name or namepatier' (coverage 0.00) \u2014 refusing to act; run aborted" + }, + { + "campaign": "emr-run-3", + "app": "emr", + "trial": 6, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_009", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + }, + { + "campaign": "emr-run-3", + "app": "emr", + "trial": 10, + "condition": "healthy", + "class": "false_stop", + "cause": "timing_codec", + "step_id": "step_009", + "detail": "Step 'step_009' (click 'First Name'): the screen kept changing at the identity region between the check and the click \u2014 click not delivered; run aborted" + } + ], + "ranked_causes": [ + { + "cause": "timing_codec", + "false_stops": 13, + "fix": "Wait for the region to settle (bounded) before the identity and stability checks instead of failing on the first changing frame; mask a blinking caret; record the codec's refinement interval.", + "where": [ + "emr/bpp16:step_014", + "emr/healthy:step_009", + "emr/healthy:step_014" + ] + }, + { + "cause": "state_check", + "false_stops": 10, + "fix": "Do not compile an auto-selected row's header text into the next step's postcondition when the row is chosen by a later click.", + "where": [ + "pn/decoy:step_002" + ] + }, + { + "cause": "hover_reresolve", + "false_stops": 10, + "fix": "Accept a re-resolve inside the target's own box (or within the codec's measured jitter, 1-2 px) instead of exact pixel equality.", + "where": [ + "pn/desktop1440:step_006" + ] + }, + { + "cause": "layout_reflow", + "false_stops": 5, + "fix": "Bind postcondition regions to the resolved target (offsets from it), not the recorded frame's absolute pixels; or halt at the first step with an explicit 'layout changed' reason.", + "where": [ + "emr/desktop1440:step_000" + ] + }, + { + "cause": "identity_precheck", + "false_stops": 2, + "fix": "Exclude run parameters and transient dialog content (duplicate lists) from the recorded identity band.", + "where": [ + "emr/healthy:step_011", + "pn/siblings:step_003" + ] + } + ], + "cause_frames": { + "state_check": "state_check_pn_decoy_step_002.png", + "ambiguity_refusal": "ambiguity_refusal_pn_siblings_step_003.png", + "hover_reresolve": "hover_reresolve_pn_desktop1440_step_006.png", + "layout_reflow": "layout_reflow_emr_desktop1440_step_000.png", + "timing_codec": "timing_codec_emr_healthy_step_014.png", + "identity_precheck": "identity_precheck_emr_healthy_step_011.png" + } +} \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/results/results_grounding_heldout.json b/packages/flow/benchmark/rdp_avc_real/results/results_grounding_heldout.json new file mode 100644 index 0000000000..cf3c37eca0 --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/results/results_grounding_heldout.json @@ -0,0 +1,1912 @@ +{ + "schema": "openadapt.rdp-avc444-grounding-heldout.v1", + "generated_at": "2026-10-29T04:30:23+00:00", + "host": { + "machine": "arm64", + "system": "Darwin", + "python": "3.12.13" + }, + "thresholds": { + "accept_score": 0.78, + "lookalike_margin": 0.15, + "template_threshold": 0.985, + "tolerance_px": [ + 24, + 12 + ] + }, + "summary": { + "frames_steps": 86, + "scored": 81, + "rung_outcomes": { + "abstain": 75, + "target": 6 + }, + "wrong_target": 0, + "abstain": 75, + "abstain_text_label": 56, + "template_found": 74, + "template_wrong": 3, + "median_ms": 118.7, + "max_ms": 158.8 + }, + "rows": [ + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "bpp16", + "archive": "pn_run_bpp16.zip", + "step_id": "step_000", + "text_label": null, + "truth": [ + 260, + 55 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.912, + "best_other_score": 0.911, + "ms": 119.8, + "template_found": true, + "template_wrong": false, + "template_ms": 92.8 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "bpp16", + "archive": "pn_run_bpp16.zip", + "step_id": "step_002", + "text_label": "Search", + "truth": [ + 559, + 55 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.929, + "best_other_score": 0.725, + "ms": 114.5, + "template_found": true, + "template_wrong": false, + "template_ms": 84.7 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "bpp16", + "archive": "pn_run_bpp16.zip", + "step_id": "step_003", + "text_label": "eil", + "truth": [ + 115, + 145 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.95, + "best_other_score": 0.902, + "ms": 115.7, + "template_found": true, + "template_wrong": false, + "template_ms": 84.0 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "bpp16", + "archive": "pn_run_bpp16.zip", + "step_id": "step_004", + "text_label": null, + "truth": [ + 311, + 400 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": false, + "true_score": null, + "best_other_score": 0.852, + "ms": 91.3, + "template_found": false, + "template_wrong": true, + "template_ms": 74.5 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "bpp16", + "archive": "pn_run_bpp16.zip", + "step_id": "step_006", + "text_label": "Save Note", + "truth": [ + 560, + 489 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.889, + "best_other_score": 0.757, + "ms": 117.9, + "template_found": false, + "template_wrong": false, + "template_ms": 86.1 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "decoy", + "archive": "pn_run_decoy.zip", + "step_id": "step_000", + "text_label": null, + "truth": [ + 260, + 55 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.912, + "best_other_score": 0.911, + "ms": 116.6, + "template_found": true, + "template_wrong": false, + "template_ms": 85.7 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "decoy", + "archive": "pn_run_decoy.zip", + "step_id": "step_002", + "text_label": "Search", + "truth": [ + 559, + 55 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.911, + "best_other_score": 0.723, + "ms": 117.5, + "template_found": true, + "template_wrong": false, + "template_ms": 84.5 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "desktop1440", + "archive": "pn_run_desktop1440.zip", + "step_id": "step_000", + "text_label": null, + "truth": null, + "truth_source": null, + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": false, + "true_score": null, + "best_other_score": 0.91, + "ms": 131.5, + "template_found": false, + "template_wrong": false, + "template_ms": 98.5 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "desktop1440", + "archive": "pn_run_desktop1440.zip", + "step_id": "step_002", + "text_label": "Search", + "truth": null, + "truth_source": null, + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": false, + "true_score": null, + "best_other_score": 0.921, + "ms": 133.1, + "template_found": false, + "template_wrong": false, + "template_ms": 98.5 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "desktop1440", + "archive": "pn_run_desktop1440.zip", + "step_id": "step_003", + "text_label": "eil", + "truth": null, + "truth_source": null, + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": false, + "true_score": null, + "best_other_score": 0.93, + "ms": 131.1, + "template_found": false, + "template_wrong": false, + "template_ms": 98.3 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "desktop1440", + "archive": "pn_run_desktop1440.zip", + "step_id": "step_004", + "text_label": null, + "truth": null, + "truth_source": null, + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": false, + "true_score": null, + "best_other_score": 0.813, + "ms": 98.0, + "template_found": false, + "template_wrong": false, + "template_ms": 91.7 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "desktop1440", + "archive": "pn_run_desktop1440.zip", + "step_id": "step_006", + "text_label": "Save Note", + "truth": null, + "truth_source": null, + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": false, + "true_score": null, + "best_other_score": 0.903, + "ms": 132.3, + "template_found": false, + "template_wrong": false, + "template_ms": 96.7 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "healthy", + "archive": "pn_run_healthy.zip", + "step_id": "step_000", + "text_label": null, + "truth": [ + 260, + 55 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.912, + "best_other_score": 0.911, + "ms": 122.5, + "template_found": true, + "template_wrong": false, + "template_ms": 86.3 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "healthy", + "archive": "pn_run_healthy.zip", + "step_id": "step_002", + "text_label": "Search", + "truth": [ + 559, + 55 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.933, + "best_other_score": 0.724, + "ms": 120.1, + "template_found": true, + "template_wrong": false, + "template_ms": 85.4 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "healthy", + "archive": "pn_run_healthy.zip", + "step_id": "step_003", + "text_label": "eil", + "truth": [ + 115, + 145 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.951, + "best_other_score": 0.91, + "ms": 116.8, + "template_found": true, + "template_wrong": false, + "template_ms": 84.4 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "healthy", + "archive": "pn_run_healthy.zip", + "step_id": "step_004", + "text_label": null, + "truth": [ + 311, + 400 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": false, + "true_score": null, + "best_other_score": 0.863, + "ms": 92.9, + "template_found": false, + "template_wrong": true, + "template_ms": 78.5 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "healthy", + "archive": "pn_run_healthy.zip", + "step_id": "step_006", + "text_label": "Save Note", + "truth": [ + 560, + 489 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.898, + "best_other_score": 0.754, + "ms": 118.7, + "template_found": false, + "template_wrong": false, + "template_ms": 82.6 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "reorder", + "archive": "pn_run_reorder.zip", + "step_id": "step_000", + "text_label": null, + "truth": [ + 260, + 55 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.912, + "best_other_score": 0.911, + "ms": 116.8, + "template_found": true, + "template_wrong": false, + "template_ms": 86.4 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "reorder", + "archive": "pn_run_reorder.zip", + "step_id": "step_002", + "text_label": "Search", + "truth": [ + 559, + 55 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.934, + "best_other_score": 0.725, + "ms": 114.8, + "template_found": true, + "template_wrong": false, + "template_ms": 85.2 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "reorder", + "archive": "pn_run_reorder.zip", + "step_id": "step_003", + "text_label": "eil", + "truth": [ + 115, + 145 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.943, + "best_other_score": 0.925, + "ms": 117.0, + "template_found": true, + "template_wrong": false, + "template_ms": 82.1 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "reorder", + "archive": "pn_run_reorder.zip", + "step_id": "step_004", + "text_label": null, + "truth": [ + 311, + 400 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": false, + "true_score": null, + "best_other_score": 0.857, + "ms": 94.4, + "template_found": false, + "template_wrong": true, + "template_ms": 76.2 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "reorder", + "archive": "pn_run_reorder.zip", + "step_id": "step_006", + "text_label": "Save Note", + "truth": [ + 560, + 489 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.895, + "best_other_score": 0.75, + "ms": 117.8, + "template_found": false, + "template_wrong": false, + "template_ms": 85.7 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "siblings", + "archive": "pn_run_siblings.zip", + "step_id": "step_000", + "text_label": null, + "truth": [ + 260, + 55 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.904, + "best_other_score": 0.902, + "ms": 117.6, + "template_found": true, + "template_wrong": false, + "template_ms": 84.7 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "siblings", + "archive": "pn_run_siblings.zip", + "step_id": "step_002", + "text_label": "Search", + "truth": [ + 559, + 55 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.915, + "best_other_score": 0.716, + "ms": 117.6, + "template_found": true, + "template_wrong": false, + "template_ms": 84.5 + }, + { + "app": "pn", + "campaign": "pn-run-3", + "condition": "siblings", + "archive": "pn_run_siblings.zip", + "step_id": "step_003", + "text_label": "eil", + "truth": [ + 115, + 189 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 10, + "true_found": true, + "true_score": 0.781, + "best_other_score": 0.767, + "ms": 120.4, + "template_found": false, + "template_wrong": false, + "template_ms": 85.5 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_000", + "text_label": "ername", + "truth": [ + 705, + 396 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.937, + "best_other_score": 0.946, + "ms": 113.3, + "template_found": true, + "template_wrong": false, + "template_ms": 86.6 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_002", + "text_label": null, + "truth": [ + 684, + 450 + ], + "truth_source": "verified_trial", + "rung_outcome": "target", + "lookalikes": 2, + "true_found": true, + "true_score": 0.943, + "best_other_score": 0.784, + "ms": 119.1, + "template_found": true, + "template_wrong": false, + "template_ms": 86.5 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_004", + "text_label": "Login", + "truth": [ + 640, + 558 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.943, + "best_other_score": 0.773, + "ms": 117.4, + "template_found": true, + "template_wrong": false, + "template_ms": 84.9 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_005", + "text_label": null, + "truth": [ + 361, + 43 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.959, + "best_other_score": 0.947, + "ms": 116.1, + "template_found": true, + "template_wrong": false, + "template_ms": 86.7 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_006", + "text_label": "New/Search", + "truth": [ + 378, + 71 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.978, + "best_other_score": 0.67, + "ms": 144.2, + "template_found": true, + "template_wrong": false, + "template_ms": 79.2 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_007", + "text_label": "Last Name", + "truth": [ + 706, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 7, + "true_found": true, + "true_score": 0.933, + "best_other_score": 0.873, + "ms": 119.5, + "template_found": true, + "template_wrong": false, + "template_ms": 88.8 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_009", + "text_label": "First Name", + "truth": [ + 445, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 10, + "true_found": true, + "true_score": 0.749, + "best_other_score": 0.841, + "ms": 128.1, + "template_found": true, + "template_wrong": false, + "template_ms": 82.9 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_011", + "text_label": null, + "truth": [ + 429, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": false, + "true_score": null, + "best_other_score": 0.852, + "ms": 117.0, + "template_found": true, + "template_wrong": false, + "template_ms": 85.9 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_014", + "text_label": "Unassigned", + "truth": [ + 965, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.916, + "best_other_score": 0.814, + "ms": 116.2, + "template_found": true, + "template_wrong": false, + "template_ms": 88.4 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_017", + "text_label": "Billing Note:", + "truth": [ + 146, + 599 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 11, + "true_found": true, + "true_score": 0.862, + "best_other_score": 0.835, + "ms": 117.2, + "template_found": true, + "template_wrong": false, + "template_ms": 87.5 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_019", + "text_label": "Create New Patient", + "truth": [ + 272, + 733 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.977, + "best_other_score": 0.486, + "ms": 119.1, + "template_found": true, + "template_wrong": false, + "template_ms": 88.9 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_020", + "text_label": "Confirm Create New Patient", + "truth": [ + 600, + 260 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 3, + "true_found": true, + "true_score": 0.955, + "best_other_score": 0.787, + "ms": 118.3, + "template_found": true, + "template_wrong": false, + "template_ms": 85.8 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_000", + "text_label": "ername", + "truth": [ + 705, + 396 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.945, + "best_other_score": 0.945, + "ms": 111.7, + "template_found": true, + "template_wrong": false, + "template_ms": 86.8 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_002", + "text_label": null, + "truth": [ + 684, + 450 + ], + "truth_source": "verified_trial", + "rung_outcome": "target", + "lookalikes": 2, + "true_found": true, + "true_score": 0.949, + "best_other_score": 0.781, + "ms": 119.0, + "template_found": true, + "template_wrong": false, + "template_ms": 87.3 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_004", + "text_label": "Login", + "truth": [ + 640, + 558 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.941, + "best_other_score": 0.791, + "ms": 118.7, + "template_found": true, + "template_wrong": false, + "template_ms": 84.7 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_005", + "text_label": null, + "truth": [ + 360, + 43 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 7, + "true_found": true, + "true_score": 0.966, + "best_other_score": 0.951, + "ms": 117.4, + "template_found": true, + "template_wrong": false, + "template_ms": 84.5 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_006", + "text_label": "New/Search", + "truth": [ + 378, + 71 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.978, + "best_other_score": 0.675, + "ms": 146.0, + "template_found": true, + "template_wrong": false, + "template_ms": 82.1 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_007", + "text_label": "Last Name", + "truth": [ + 706, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 7, + "true_found": true, + "true_score": 0.92, + "best_other_score": 0.871, + "ms": 116.3, + "template_found": true, + "template_wrong": false, + "template_ms": 85.2 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_009", + "text_label": "First Name", + "truth": [ + 445, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 3, + "true_found": true, + "true_score": 0.877, + "best_other_score": 0.834, + "ms": 128.7, + "template_found": true, + "template_wrong": false, + "template_ms": 81.3 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_011", + "text_label": null, + "truth": [ + 429, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": false, + "true_score": null, + "best_other_score": 0.854, + "ms": 123.3, + "template_found": true, + "template_wrong": false, + "template_ms": 85.7 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_014", + "text_label": "Unassigned", + "truth": [ + 965, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.912, + "best_other_score": 0.799, + "ms": 140.1, + "template_found": true, + "template_wrong": false, + "template_ms": 96.2 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_017", + "text_label": "Billing Note:", + "truth": [ + 146, + 599 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 10, + "true_found": true, + "true_score": 0.867, + "best_other_score": 0.826, + "ms": 148.3, + "template_found": true, + "template_wrong": false, + "template_ms": 85.6 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_019", + "text_label": "Create New Patient", + "truth": [ + 272, + 733 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.976, + "best_other_score": 0.522, + "ms": 119.9, + "template_found": true, + "template_wrong": false, + "template_ms": 86.7 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_020", + "text_label": "Confirm Create New Patient", + "truth": [ + 600, + 261 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.957, + "best_other_score": 0.791, + "ms": 120.7, + "template_found": true, + "template_wrong": false, + "template_ms": 88.5 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_000", + "text_label": "ername", + "truth": [ + 705, + 396 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.945, + "best_other_score": 0.945, + "ms": 119.5, + "template_found": true, + "template_wrong": false, + "template_ms": 85.6 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_002", + "text_label": null, + "truth": [ + 684, + 450 + ], + "truth_source": "verified_trial", + "rung_outcome": "target", + "lookalikes": 2, + "true_found": true, + "true_score": 0.949, + "best_other_score": 0.781, + "ms": 122.7, + "template_found": true, + "template_wrong": false, + "template_ms": 84.4 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_004", + "text_label": "Login", + "truth": [ + 640, + 558 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.941, + "best_other_score": 0.791, + "ms": 117.3, + "template_found": true, + "template_wrong": false, + "template_ms": 84.6 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_005", + "text_label": null, + "truth": [ + 360, + 43 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 7, + "true_found": true, + "true_score": 0.967, + "best_other_score": 0.949, + "ms": 115.5, + "template_found": true, + "template_wrong": false, + "template_ms": 84.5 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_006", + "text_label": "New/Search", + "truth": [ + 378, + 71 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.979, + "best_other_score": 0.669, + "ms": 140.3, + "template_found": true, + "template_wrong": false, + "template_ms": 81.8 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_007", + "text_label": "Last Name", + "truth": [ + 706, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.934, + "best_other_score": 0.868, + "ms": 120.1, + "template_found": true, + "template_wrong": false, + "template_ms": 86.1 + }, + { + "app": "emr", + "campaign": "emr-run-3", + "condition": "healthy", + "archive": "emr_run_healthy_false_stop.zip", + "step_id": "step_009", + "text_label": "First Name", + "truth": [ + 445, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.854, + "best_other_score": 0.836, + "ms": 133.2, + "template_found": true, + "template_wrong": false, + "template_ms": 83.7 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_000", + "text_label": "ername", + "truth": [ + 705, + 396 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.906, + "best_other_score": 0.946, + "ms": 119.4, + "template_found": true, + "template_wrong": false, + "template_ms": 86.4 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_002", + "text_label": null, + "truth": [ + 684, + 450 + ], + "truth_source": "verified_trial", + "rung_outcome": "target", + "lookalikes": 2, + "true_found": true, + "true_score": 0.944, + "best_other_score": 0.793, + "ms": 110.5, + "template_found": true, + "template_wrong": false, + "template_ms": 85.7 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_004", + "text_label": "Login", + "truth": [ + 640, + 558 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.939, + "best_other_score": 0.782, + "ms": 115.8, + "template_found": true, + "template_wrong": false, + "template_ms": 84.2 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_005", + "text_label": null, + "truth": [ + 361, + 43 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.964, + "best_other_score": 0.946, + "ms": 117.4, + "template_found": true, + "template_wrong": false, + "template_ms": 86.0 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_006", + "text_label": "New/Search", + "truth": [ + 378, + 71 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.964, + "best_other_score": 0.689, + "ms": 144.2, + "template_found": true, + "template_wrong": false, + "template_ms": 81.6 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_007", + "text_label": "Last Name", + "truth": [ + 706, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.929, + "best_other_score": 0.866, + "ms": 120.8, + "template_found": true, + "template_wrong": false, + "template_ms": 87.6 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_009", + "text_label": "First Name", + "truth": [ + 445, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": true, + "true_score": 0.896, + "best_other_score": 0.841, + "ms": 129.2, + "template_found": true, + "template_wrong": false, + "template_ms": 83.0 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_011", + "text_label": null, + "truth": [ + 429, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 8, + "true_found": true, + "true_score": 0.811, + "best_other_score": 0.825, + "ms": 120.3, + "template_found": true, + "template_wrong": false, + "template_ms": 84.4 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_014", + "text_label": "Unassigned", + "truth": [ + 965, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.913, + "best_other_score": 0.804, + "ms": 125.4, + "template_found": true, + "template_wrong": false, + "template_ms": 86.3 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_017", + "text_label": "Billing Note:", + "truth": [ + 146, + 599 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 11, + "true_found": true, + "true_score": 0.862, + "best_other_score": 0.84, + "ms": 115.7, + "template_found": true, + "template_wrong": false, + "template_ms": 87.6 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_019", + "text_label": "Create New Patient", + "truth": [ + 272, + 733 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.981, + "best_other_score": 0.486, + "ms": 117.9, + "template_found": true, + "template_wrong": false, + "template_ms": 84.1 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16.zip", + "step_id": "step_020", + "text_label": "Confirm Create New Patient", + "truth": [ + 600, + 261 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.951, + "best_other_score": 0.789, + "ms": 117.9, + "template_found": true, + "template_wrong": false, + "template_ms": 87.3 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_000", + "text_label": "ername", + "truth": [ + 705, + 396 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.906, + "best_other_score": 0.946, + "ms": 111.6, + "template_found": true, + "template_wrong": false, + "template_ms": 86.3 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_002", + "text_label": null, + "truth": [ + 684, + 450 + ], + "truth_source": "verified_trial", + "rung_outcome": "target", + "lookalikes": 2, + "true_found": true, + "true_score": 0.944, + "best_other_score": 0.793, + "ms": 112.8, + "template_found": true, + "template_wrong": false, + "template_ms": 84.5 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_004", + "text_label": "Login", + "truth": [ + 640, + 558 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.937, + "best_other_score": 0.787, + "ms": 115.5, + "template_found": true, + "template_wrong": false, + "template_ms": 86.5 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_005", + "text_label": null, + "truth": [ + 361, + 43 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.964, + "best_other_score": 0.946, + "ms": 121.9, + "template_found": true, + "template_wrong": false, + "template_ms": 91.4 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_006", + "text_label": "New/Search", + "truth": [ + 378, + 71 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.965, + "best_other_score": 0.68, + "ms": 150.2, + "template_found": true, + "template_wrong": false, + "template_ms": 78.9 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_007", + "text_label": "Last Name", + "truth": [ + 706, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.93, + "best_other_score": 0.865, + "ms": 115.8, + "template_found": true, + "template_wrong": false, + "template_ms": 86.3 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_009", + "text_label": "First Name", + "truth": [ + 445, + 311 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 9, + "true_found": true, + "true_score": 0.852, + "best_other_score": 0.839, + "ms": 126.9, + "template_found": true, + "template_wrong": false, + "template_ms": 81.2 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_011", + "text_label": null, + "truth": [ + 429, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 8, + "true_found": true, + "true_score": 0.818, + "best_other_score": 0.814, + "ms": 127.5, + "template_found": true, + "template_wrong": false, + "template_ms": 86.9 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "bpp16", + "archive": "emr_run_bpp16_false_stop.zip", + "step_id": "step_014", + "text_label": "Unassigned", + "truth": [ + 965, + 423 + ], + "truth_source": "verified_trial", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.916, + "best_other_score": 0.806, + "ms": 121.9, + "template_found": true, + "template_wrong": false, + "template_ms": 92.5 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_000", + "text_label": "ername", + "truth": [ + 705, + 396 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.906, + "best_other_score": 0.946, + "ms": 123.5, + "template_found": true, + "template_wrong": false, + "template_ms": 85.8 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_002", + "text_label": null, + "truth": [ + 684, + 450 + ], + "truth_source": "recorded_layout", + "rung_outcome": "target", + "lookalikes": 2, + "true_found": true, + "true_score": 0.944, + "best_other_score": 0.793, + "ms": 115.8, + "template_found": true, + "template_wrong": false, + "template_ms": 93.1 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_004", + "text_label": "Login", + "truth": [ + 640, + 558 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 2, + "true_found": true, + "true_score": 0.937, + "best_other_score": 0.787, + "ms": 158.8, + "template_found": true, + "template_wrong": false, + "template_ms": 90.6 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_005", + "text_label": null, + "truth": [ + 361, + 43 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.966, + "best_other_score": 0.948, + "ms": 120.0, + "template_found": true, + "template_wrong": false, + "template_ms": 87.2 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_006", + "text_label": "New/Search", + "truth": [ + 378, + 71 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 1, + "true_found": true, + "true_score": 0.966, + "best_other_score": 0.693, + "ms": 143.9, + "template_found": true, + "template_wrong": false, + "template_ms": 81.5 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_007", + "text_label": "Last Name", + "truth": [ + 706, + 311 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 6, + "true_found": true, + "true_score": 0.929, + "best_other_score": 0.866, + "ms": 116.9, + "template_found": true, + "template_wrong": false, + "template_ms": 86.8 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_009", + "text_label": "First Name", + "truth": [ + 445, + 311 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 4, + "true_found": true, + "true_score": 0.896, + "best_other_score": 0.841, + "ms": 131.7, + "template_found": true, + "template_wrong": false, + "template_ms": 84.2 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_011", + "text_label": null, + "truth": [ + 429, + 423 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 8, + "true_found": true, + "true_score": 0.811, + "best_other_score": 0.825, + "ms": 116.0, + "template_found": true, + "template_wrong": false, + "template_ms": 88.6 + }, + { + "app": "emr", + "campaign": "emr-run-2", + "condition": "healthy", + "archive": "emr_run_healthy.zip", + "step_id": "step_014", + "text_label": "Unassigned", + "truth": [ + 965, + 423 + ], + "truth_source": "recorded_layout", + "rung_outcome": "abstain", + "lookalikes": 5, + "true_found": true, + "true_score": 0.913, + "best_other_score": 0.804, + "ms": 116.6, + "template_found": true, + "template_wrong": false, + "template_ms": 85.4 + } + ] +} \ No newline at end of file diff --git a/packages/flow/benchmark/rdp_avc_real/runner.py b/packages/flow/benchmark/rdp_avc_real/runner.py new file mode 100644 index 0000000000..48489d19af --- /dev/null +++ b/packages/flow/benchmark/rdp_avc_real/runner.py @@ -0,0 +1,146 @@ +"""Job runner for the real H.264/AVC444 remote-desktop lab (vm-client). + +Runs at logon in the ``harness`` user's interactive desktop, which is where +mstsc paints and where SendInput and PrintWindow work. The operator drops a +job file into ``C:\\oa\\jobs`` through an Azure run command; the runner +executes it with ``harness.py`` and writes results to +``C:\\oa\\results\\``. Run commands themselves execute in session 0, +which has no desktop, so they never drive the client directly. + +On first start it stores the two session credentials with cmdkey and +deletes the file they arrived in. +""" + +from __future__ import annotations + +import ctypes +import json +import subprocess +import sys +import time +import traceback +import winreg +from pathlib import Path + +ROOT = Path(r"C:\oa") +JOBS = ROOT / "jobs" +RESULTS = ROOT / "results" +CREDS = ROOT / "secret" / "creds.json" +HERE = Path(__file__).resolve().parent + +# One cmdkey target per session user, so no two users share a credential: +# OpenEMR sessions connect by IP, Patient Notes by host name, the read-back +# session by the fully qualified host name. +READER_TARGET = "vm-rdhost.x3hkkcl3scte1kfhx4qhkg0tta.bx.internal.cloudapp.net" +TARGETS = { + "emr": ("10.30.1.7", "emruser"), + "pn": ("vm-rdhost", "pnuser"), + # Read-back: its own logon, under the host's fully qualified name. + "pnread": (READER_TARGET, "pnreader"), +} + + +def _set_resolution(width: int = 1920, height: int = 1080) -> None: + class DEVMODE(ctypes.Structure): + _fields_ = [ + ("dmDeviceName", ctypes.c_wchar * 32), + ("dmSpecVersion", ctypes.c_ushort), + ("dmDriverVersion", ctypes.c_ushort), + ("dmSize", ctypes.c_ushort), + ("dmDriverExtra", ctypes.c_ushort), + ("dmFields", ctypes.c_ulong), + ("dmPosition", ctypes.c_long * 2), + ("dmDisplayOrientation", ctypes.c_ulong), + ("dmDisplayFixedOutput", ctypes.c_ulong), + ("dmColor", ctypes.c_short), + ("dmDuplex", ctypes.c_short), + ("dmYResolution", ctypes.c_short), + ("dmTTOption", ctypes.c_short), + ("dmCollate", ctypes.c_short), + ("dmFormName", ctypes.c_wchar * 32), + ("dmLogPixels", ctypes.c_ushort), + ("dmBitsPerPel", ctypes.c_ulong), + ("dmPelsWidth", ctypes.c_ulong), + ("dmPelsHeight", ctypes.c_ulong), + ("dmDisplayFlags", ctypes.c_ulong), + ("dmDisplayFrequency", ctypes.c_ulong), + ("dmICMMethod", ctypes.c_ulong), + ("dmICMIntent", ctypes.c_ulong), + ("dmMediaType", ctypes.c_ulong), + ("dmDitherType", ctypes.c_ulong), + ("dmReserved1", ctypes.c_ulong), + ("dmReserved2", ctypes.c_ulong), + ("dmPanningWidth", ctypes.c_ulong), + ("dmPanningHeight", ctypes.c_ulong), + ] + + dm = DEVMODE() + dm.dmSize = ctypes.sizeof(DEVMODE) + dm.dmPelsWidth, dm.dmPelsHeight = width, height + dm.dmFields = 0x00080000 | 0x00100000 # DM_PELSWIDTH | DM_PELSHEIGHT + ctypes.windll.user32.ChangeDisplaySettingsW(ctypes.byref(dm), 0) + + +def store_credentials() -> None: + """Store any session passwords left in the secret file, then delete it.""" + if CREDS.exists(): + creds = json.loads(CREDS.read_text(encoding="utf-8-sig")) + for target, user in TARGETS.values(): + if user not in creds: + continue + subprocess.run( + [ + "cmdkey", + f"/generic:TERMSRV/{target}", + f"/user:vm-rdhost\\{user}", + f"/pass:{creds[user]}", + ], + check=True, + capture_output=True, + ) + CREDS.unlink() + + +def _prepare_client() -> None: + # Self-signed host certificate: connect without the warning dialog. + key = winreg.CreateKey( + winreg.HKEY_CURRENT_USER, r"Software\Microsoft\Terminal Server Client" + ) + winreg.SetValueEx(key, "AuthenticationLevelOverride", 0, winreg.REG_DWORD, 0) + store_credentials() + + +def main() -> int: + JOBS.mkdir(parents=True, exist_ok=True) + RESULTS.mkdir(parents=True, exist_ok=True) + _set_resolution() + _prepare_client() + (RESULTS / "runner_ready.json").write_text( + json.dumps({"ready_at": time.time(), "python": sys.version}), encoding="utf-8" + ) + while True: + for job_path in sorted(JOBS.glob("*.json")): + job = json.loads(job_path.read_text(encoding="utf-8-sig")) + out = RESULTS / job["id"] + out.mkdir(parents=True, exist_ok=True) + running = job_path.with_suffix(".running") + job_path.rename(running) + log = (out / "runner.log").open("w", encoding="utf-8") + try: + proc = subprocess.run( + [sys.executable, str(HERE / "harness.py"), str(running), str(out)], + stdout=log, + stderr=subprocess.STDOUT, + timeout=job.get("timeout_s", 7200), + ) + status = {"exit": proc.returncode} + except Exception: # noqa: BLE001 -- the runner must outlive any job + status = {"exit": -1, "error": traceback.format_exc()} + log.close() + (out / "status.json").write_text(json.dumps(status), encoding="utf-8") + running.rename(job_path.with_suffix(".done")) + time.sleep(2) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index da9a52d607..3cdcffc007 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -23,6 +23,7 @@ Every published result used synthetic patients, claims and records. | OpenEMR 8.0.0.3, local test copies (open-source EHR; its 7.0 release is ONC-certified) | Referral entry by agent, screen program and API step; injected database faults; repair of a relabeled save button; duplicate check | | openIMIS, local test copy (open-source insurance and claims system) | Claim entry over a worklist; a relabeled Save sent to repair with no wrong save | | Windows 11 over RDP, real network | 3 of 3 effect-confirmed trials, plus a FreeRDP batch | +| Windows Server 2022 over RDP with H.264/AVC444, stock Remote Desktop client | 105 runs on two apps: 55 database-confirmed saves, 0 wrong-record saves, 40 false stops ([benchmark/rdp_avc_real](../benchmark/rdp_avc_real/README.md)) | | Citrix stand-in, no DOM | 3 healthy runs and 3 drift stops through the Citrix backend | | WinForms, TextEdit, a GTK3 app | 3 of 3 effects on each native backend | | MockMed, MockLoan and fixtures (OpenAdapt's own test apps) | Fault suites, approvals, repair and the agent comparisons | @@ -43,7 +44,7 @@ signed; the Python packages are published from the reviewed release workflow. | Windows UI Automation | **Counted task acceptance** | The native UIA backend completed 3/3 exact WinForms effects and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. | | Native macOS automation | **Counted task acceptance** | The AX/AppKit backend completed 3/3 exact TextEdit file effects and refused an ambiguous two-window target; structural record/locate/refuse behavior is covered separately. Known AX-only ` - Google Chrome` and ` - Google Chrome for Testing` window-title suffixes are accepted only for the corresponding exact CoreGraphics application owner; PID, unique-candidate, focused/main, and topmost-window checks remain mandatory. | Acceptance is bound to the tested host and application. Accessibility exposure and supported native actions vary by application. | | Native Linux automation | **Counted task acceptance plus required CI** | Required current-main CI exercises AT-SPI against GTK3 with 3 exact effects, 3 ambiguity refusals, and 3 stale-target refusals. | The built-in path is X11. Wayland needs an operator-approved portal session, and each application retains its own qualification. | -| RDP automation | **Counted task acceptance plus required CI** | A real-network Aardwolf/Windows batch passed 3/3 with an independent guest-tools oracle; a separate real-FreeRDP batch covers record, compile, governed replay, and drift refusal. | The two records do not by themselves cover arbitrary applications, WAN conditions, display policy, or Citrix ICA/HDX. | +| RDP automation | **Counted task acceptance plus required CI** | A real-network Aardwolf/Windows batch passed 3/3 with an independent guest-tools oracle; a separate real-FreeRDP batch covers record, compile, governed replay, and drift refusal. | The records do not cover arbitrary applications, WAN conditions, or Citrix ICA/HDX. A real AVC444 session (same virtual network, software encoding) stops on 12 of 20 healthy OpenEMR runs; the causes are ranked in benchmark/rdp_avc_real. | | Citrix / VDI pixel-window automation | **Required CI plus counted no-DOM stand-in** | The dedicated exact-Workspace-window backend, readiness gate, governed run, and durable resume passed 3 healthy effect-confirmed no-DOM trials plus 3 drift safe-halts. | The retained record explicitly has `ica_hdx_accepted=false`; live ICA/HDX acceptance stays bound to the exact deployment instead of being inferred from a stand-in or RDP. | | Managed browser execution | **Separate product target and workflow admissions** | The hosted lane admits attested browser bundles; production mode requires a configured real runner and refuses silent mock fallback. | It does not extend the supported claim to Windows, RDP, Citrix, PHI-bearing shared-cloud execution, an SLA, or a regulated certification. | | On-premises / customer-managed deployment | **Release and clean-machine evidence; site acceptance required** | Deployment configuration, local run gates, egress checks, and audit primitives are supplied. | OpenAdapt does not configure the customer's firewall, KMS, storage, identity provider, backups, retention, incident response, or legal compliance program. | diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index f50cc8f8e8..3412ea0eca 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3283,13 +3283,16 @@ def save( _seal_template_assets(bundle, key, self._decrypted_templates) path = encrypted_path else: - plaintext_path.write_text(serialized) + # Bytes, not write_text: the platform default codec (cp1252 on + # Windows) cannot encode every character OCR records, and text + # mode would turn newlines into CRLF and change the bundle digest. + plaintext_path.write_bytes(serialized.encode("utf-8")) if encrypted_path.exists(): encrypted_path.unlink() path = plaintext_path if self.manifest is not None: - (bundle / "manifest.json").write_text( - self.manifest.model_dump_json(indent=2) + (bundle / "manifest.json").write_bytes( + self.manifest.model_dump_json(indent=2).encode("utf-8") ) return path @@ -3396,7 +3399,7 @@ def load( ) raw = json.loads(decrypted) else: - raw = json.loads(plaintext_path.read_text()) + raw = json.loads(plaintext_path.read_bytes()) raw = _bv.migrate_bundle_dict(raw) # A manifest may be embedded in workflow.json OR sit in a sidecar; the # embedded one wins, else the sidecar, else it is computed fresh. diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index c332497a5b..2cda722e07 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -272,6 +272,32 @@ "benchmark/rdp/results.json", "benchmark/rdp/results_82a658a_20260718.sanitized.json", "benchmark/rdp/results_e2c7acf_20260718.json", + "benchmark/rdp_avc_real/results/campaigns/emr-run-1/reports.tar.gz", + "benchmark/rdp_avc_real/results/campaigns/emr-run-1/trials.json", + "benchmark/rdp_avc_real/results/campaigns/emr-run-2/reports.tar.gz", + "benchmark/rdp_avc_real/results/campaigns/emr-run-2/trials.json", + "benchmark/rdp_avc_real/results/campaigns/emr-run-3/reports.tar.gz", + "benchmark/rdp_avc_real/results/campaigns/emr-run-3/trials.json", + "benchmark/rdp_avc_real/results/campaigns/pn-readback-1/readback.json", + "benchmark/rdp_avc_real/results/campaigns/pn-readback-1/result.json", + "benchmark/rdp_avc_real/results/campaigns/pn-readback-2/readback.json", + "benchmark/rdp_avc_real/results/campaigns/pn-readback-2/result.json", + "benchmark/rdp_avc_real/results/campaigns/pn-run-1/reports.tar.gz", + "benchmark/rdp_avc_real/results/campaigns/pn-run-1/trials.json", + "benchmark/rdp_avc_real/results/campaigns/pn-run-3/reports.tar.gz", + "benchmark/rdp_avc_real/results/campaigns/pn-run-3/trials.json", + "benchmark/rdp_avc_real/results/environment.json", + "benchmark/rdp_avc_real/results/frames/ambiguity_refusal_pn_siblings_step_003.png", + "benchmark/rdp_avc_real/results/frames/emr_session_avc444.png", + "benchmark/rdp_avc_real/results/frames/hover_reresolve_pn_desktop1440_step_006.png", + "benchmark/rdp_avc_real/results/frames/identity_precheck_emr_healthy_step_011.png", + "benchmark/rdp_avc_real/results/frames/layout_reflow_emr_desktop1440_step_000.png", + "benchmark/rdp_avc_real/results/frames/pn_session_avc444.png", + "benchmark/rdp_avc_real/results/frames/readback_session_case1.png", + "benchmark/rdp_avc_real/results/frames/state_check_pn_decoy_step_002.png", + "benchmark/rdp_avc_real/results/frames/timing_codec_emr_healthy_step_014.png", + "benchmark/rdp_avc_real/results/results.json", + "benchmark/rdp_avc_real/results/results_grounding_heldout.json", "benchmark/rdp_ladder/policy.yaml", "benchmark/rdp_ladder/results.json", "benchmark/rdp_multiapp/campaign.json", diff --git a/packages/flow/scripts/avd_lab/provision_avd_lab.sh b/packages/flow/scripts/avd_lab/provision_avd_lab.sh new file mode 100755 index 0000000000..75f9e5fb6b --- /dev/null +++ b/packages/flow/scripts/avd_lab/provision_avd_lab.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Provision a one-host Azure Virtual Desktop lab: a pooled host pool with one +# Entra-joined Windows 11 multi-session host, a Windows Server client VM that +# runs Windows App (the harness drives that window), and a Linux VM serving +# OpenEMR 7.0.3 on the private network. +# +# COST running: session host D4s_v4 ($0.376/hr) + client D2s_v4 ($0.188/hr) +# + OpenEMR D2s_v4 Linux ($0.096/hr) = ~$0.66/hr. DEALLOCATE WHEN IDLE +# (bash lifecycle.sh stop). One resource group, so teardown is one command. +# +# Prereqs: az logged in; `az extension add -n desktopvirtualization`; the +# admin password in the macOS keychain item `openadapt-avd-lab` (account +# azureadmin) or in $ADMIN_PW. Session-host registration runs afterwards with +# register_session_host.sh. +set -euo pipefail + +RG="${RG:-openadapt-avd-lab}" +LOC="${LOC:-eastus}" +MYIP="${MYIP:-$(curl -s https://api.ipify.org)}" +ADMIN_USER="${ADMIN_USER:-azureadmin}" +ADMIN_PW="${ADMIN_PW:-$(security find-generic-password -s openadapt-avd-lab -a azureadmin -w)}" +HOST_IMG="MicrosoftWindowsDesktop:windows-11:win11-25h2-avd:latest" +CLIENT_IMG="MicrosoftWindowsServer:WindowsServer:2022-datacenter-azure-edition:latest" +here="$(cd "$(dirname "$0")" && pwd)" +echo "Resource group: $RG Region: $LOC Allowed IP: $MYIP" + +az group create -n "$RG" -l "$LOC" -o none \ + --tags project=openadapt purpose=avd-lab autostop=deallocate-when-idle +az network vnet create -g "$RG" -n avdlab-vnet -o none \ + --address-prefix 10.30.0.0/16 --subnet-name lab --subnet-prefix 10.30.1.0/24 +az network nsg create -g "$RG" -n avdlab-nsg -o none +# RDP to the client VM only, and only from the operator. The session host +# needs no inbound rule: AVD connections are reverse-connect through the +# AVD gateway. +az network nsg rule create -g "$RG" --nsg-name avdlab-nsg -n allow-rdp-from-me -o none \ + --priority 100 --source-address-prefixes "${MYIP}/32" --destination-port-ranges 3389 \ + --access Allow --protocol Tcp --direction Inbound + +common=(--vnet-name avdlab-vnet --subnet lab --nsg avdlab-nsg --public-ip-sku Standard + --storage-sku StandardSSD_LRS --nic-delete-option Delete --os-disk-delete-option Delete -o none) + +# Session host: Entra-joined, so it needs a system identity and the +# AADLoginForWindows extension. +az vm create -g "$RG" -n vm-avdhost --image "$HOST_IMG" --size Standard_D4s_v4 \ + --admin-username "$ADMIN_USER" --admin-password "$ADMIN_PW" --license-type Windows_Client \ + --private-ip-address 10.30.1.4 --assign-identity --os-disk-size-gb 128 "${common[@]}" +az vm extension set -g "$RG" --vm-name vm-avdhost -n AADLoginForWindows \ + --publisher Microsoft.Azure.ActiveDirectory -o none + +az vm create -g "$RG" -n vm-client --image "$CLIENT_IMG" --size Standard_D2s_v4 \ + --admin-username "$ADMIN_USER" --admin-password "$ADMIN_PW" \ + --private-ip-address 10.30.1.5 --os-disk-size-gb 128 "${common[@]}" + +# OpenEMR: the same compose file as benchmark/openemr_live, run by cloud-init. +# OpenEMR is GPL and runs as an external app; nothing from it is vendored. +init="$(mktemp)" +cat > "$init" </dev/null || date -u -d '+4 hours' '+%Y-%m-%dT%H:%M:%SZ') +token=$(az desktopvirtualization hostpool update -g "$RG" -n "$POOL" \ + --registration-info expiration-time="$expiry" registration-token-operation=Update \ + --query registrationInfo.token -o tsv) + +script=$(mktemp) +cat > "$script" <<'PS' +param([Parameter(Mandatory = $true)][string]$Token) +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" +if (Get-Service RDAgentBootLoader -ErrorAction SilentlyContinue) { + Write-Output "Agent already installed."; exit 0 +} +$agent = "C:\RDAgent.msi"; $boot = "C:\RDAgentBootLoader.msi" +Invoke-WebRequest "https://go.microsoft.com/fwlink/?linkid=2310011" -OutFile $agent -UseBasicParsing +Invoke-WebRequest "https://go.microsoft.com/fwlink/?linkid=2311028" -OutFile $boot -UseBasicParsing +$p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList "/i $agent /quiet /norestart REGISTRATIONTOKEN=$Token" +if ($p.ExitCode -ne 0) { throw "RDAgent install exit $($p.ExitCode)" } +$p = Start-Process msiexec.exe -Wait -PassThru -ArgumentList "/i $boot /quiet /norestart" +if ($p.ExitCode -ne 0) { throw "BootLoader install exit $($p.ExitCode)" } +Write-Output "Agent and boot loader installed." +PS + +name="oa-register-$(date +%s)" +az vm run-command create -g "$RG" --vm-name "$VM" --name "$name" --script "@$script" \ + --protected-parameters "Token=$token" --timeout-in-seconds 1800 --async-execution false -o none || true +az vm run-command show -g "$RG" --vm-name "$VM" --name "$name" --instance-view \ + --query "instanceView.{state:executionState,exit:exitCode,out:output,err:error}" -o json +az vm run-command delete -g "$RG" --vm-name "$VM" --name "$name" --yes --no-wait +rm -f "$script" diff --git a/packages/flow/scripts/avd_lab/setup_apps.sh b/packages/flow/scripts/avd_lab/setup_apps.sh new file mode 100755 index 0000000000..3c674ee138 --- /dev/null +++ b/packages/flow/scripts/avd_lab/setup_apps.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Install the two published apps on vm-avdhost and publish them as RemoteApps: +# - OpenEMR in Edge app mode, served by vm-emr on the private network; +# - Patient Notes, the WinForms app from scripts/desktop (SQLite record). +# Python is installed for pn_db.py, which owns the Patient Notes record. +set -euo pipefail +RG="${RG:-openadapt-avd-lab}" +VM="${VM:-vm-avdhost}" +EMR_URL="${EMR_URL:-http://10.30.1.6:8390/interface/login/login.php?site=default}" +here="$(cd "$(dirname "$0")" && pwd)" +desk="$here/../desktop" +PY="C:\\Program Files\\Python312\\python.exe" + +script=$(mktemp) +cat > "$script" <