diff --git a/.gitignore b/.gitignore index b5fb8cbf..ea179353 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ package-lock.json # end-to-end suite output (#429) test-results/ playwright-report/ + +# tsc incremental build info (bun run typecheck) +.cache/ diff --git a/CLAUDE.md b/CLAUDE.md index 26edf03b..69d15d9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,6 +150,7 @@ Sessions live in the daemon. Clients are stateless. - **Scrollback buffer**: circular ring (5000 entries / 20MB) replayed on attach - **Transcript persistence**: JSONL per session, user prompts written before API call - **Session resume**: on daemon restart, meta files scanned, sessions rebuilt, scrollback restored +- **Lazy loading** (`session.lazyLoad`, default on): a resumed session holds its metadata only (`Session.prepareResume`: the log's next seq and a mark of each of its files, its newest canonical turn, the turn index) and loads its scrollback and canonical history the first time it is used, exactly as a resume loaded them: the load reads the files as they stood at resume (`LoadTranscriptOptions.asOf`: by identity and size, so whatever was appended since is left out, whatever its seqs; an old log's seqs can go back down), and `#seq` then continues past the highest seq loaded. A transcript that can't be read at start fails the session's resume (logged, left on disk), as before. A transcript, canonical log or turn index that can't be read at a later load fails that use (the session left unloaded, read again on the next). A canonical log that can't be read when the history loads at resume (`lazyLoad: false`, or a session loaded then) leaves the session running without it, as before, but never written over (`Session#historyUnread`): going back or forking from it is refused (`RewindError` `history_unavailable`), a whole rewrite of it is never written (a rotation, which empties it anyway, excepted), turns are still appended, and each use reads it again (`#rereadHistory`, not mid-turn). Entry points await `Session.ensureHydrated()` (one shared load): `SessionManager.handle` for any message naming a session (less `NO_HISTORY_MESSAGES`, less a caller without the scopes its handler needs, `HISTORY_SCOPES`, and less what a loop worker's guard refuses, `#refusedBeforeHistory`; while one waits, every later message naming that session waits behind it, `#historyGate`, so they run in arrival order; a prompt waiting is a preparing send, `Session.holdSend`, which the drain stops and waits out; a client gone meanwhile is detached after), `send`, and the async history readers; `#scrollback` / `#accumulator` are getters that load synchronously, logged, if anything reaches them first (`Session.syncHydrations`). A notice while unloaded is appended at the next seq and kept for the load (`#buffer`); a re-seed owed by a teardown waits for it (`#reseedAfterTeardown`). Sessions from before the canonical log or turn index, or with a fresh backing, load at resume. Detach from terminal -> attach from Telegram -> same session, same context, scrollback replayed. @@ -161,8 +162,25 @@ retries the turn on that model instead of failing it. There is no daemon-side retry manager. **Graceful shutdown**: cleanup registry runs in LIFO order on SIGTERM/SIGINT/SIGHUP. -Sessions drained (in-flight work interrupted), store flushed, WebSockets closed, -frontends stopped. 30-second grace period before forced exit. +Sessions drained: from its start no turn starts (a send is refused, a turn +the backend starts on its own is interrupted as it starts, no wake, recovery +or seed retry runs: `Session.beginDrain`); in-flight work is interrupted once +per turn, and every turn not yet recorded (one its owner, a routine or a loop +stopped just before included) is waited for until its end, spend included, is +recorded — a backend acknowledges an interrupt before the turn's result +arrives — but at most its backend's settle bound (`interruptSettleMs`, +`DRAIN_SETTLE_MS` 2 s; opencode 3 s), all sessions at once, then named in the +log and left; then, at once and at most `TURN_END_SETTLE_MS` (5 s), what +each turn's end started and doesn't await (`Session.turnEndWork`: the #448 +git/settings-commit check's audit row and warning line, the end snapshot, +`after_turn` hooks) and the routine runs' final rows, spend included +(`RoutineService.settled`: a run waits as long as drain waits for its turn, +then records it, cost unknown if drain gave up on it); then the transcript +writes are flushed; polled at 10 ms backing off to 50 ms, bounded by drain's +10 s, never a fixed wait. Drain is one-way: the manager refuses sends after it. +A dispatch task whose start a stopping daemon refused goes back to the queue, +no attempt spent, and a stopped dispatcher claims nothing more. Store flushed, +WebSockets closed, frontends stopped. 30-second grace period before forced exit. **One daemon per data dir**: `codeoid start` locks each dir the daemon owns (`daemonOwnedDirs`: the database's, transcripts, opencode's database and data diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ba63a23d..fefbc5f4 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -87,6 +87,9 @@ CODEOID_RESUME_MAX_SESSIONS=200 # sessions restored from disk at startu # Resume is also time-boxed, so raising this costs # startup time; the remainder stays on disk and loads # on a later restart. +CODEOID_SESSION_LAZY_LOAD=true # a resumed session loads its scrollback and history the + # first time it is used (opened, sent to, forked, …), + # not at start; false loads every session's at start CODEOID_IDLE_RELEASE_MINUTES=30 # minutes idle before a session's backend process and # everything it started (MCP servers, commands still # running) are stopped; the next message resumes it diff --git a/docs/FEATURES.md b/docs/FEATURES.md index d8dcc7a7..8492cd7d 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -552,7 +552,7 @@ Thinking content and sub-agent tool calls surface as separate italic messages. S | Pattern | What it does | |---|---| | **Retry with fallback model** | Exponential backoff with jitter, 429/529/5xx categorization, falls back to cheaper model after 3 capacity failures. | -| **Graceful shutdown** | LIFO cleanup registry with 30s grace. SIGTERM / SIGINT / SIGHUP handlers drain sessions before closing the store. | +| **Graceful shutdown** | LIFO cleanup registry with 30s grace. SIGTERM / SIGINT / SIGHUP handlers drain sessions before closing the store: no new turn starts once the drain begins, and every turn still ending (interrupted by the drain, or stopped just before it) has its late result (its last reply, its cost), what its end starts (the sandbox's git and settings-commit check, the end snapshot, `after_turn` hooks; at most 5 s), a routine run's final row with its spend, and its transcript writes recorded first. A backend that never ends its turn holds the shutdown only its settle bound (2 s; opencode 3 s), every session at once, and is named in the log. | | **Session resume** | JSONL transcripts per session; user prompts written BEFORE API calls. On daemon restart, sessions rebuild from transcript; `#hasQueried` flag ensures Claude Code's own session store is reused via `resume` instead of re-creating. | | **Session limits** | **None by default** — running many sessions in parallel is the point. `rateLimit.maxSessionsPerUser` / `rateLimit.maxCreationsPerHour` default to `0` (unlimited); set either for a shared multi-user daemon. Automated spawning is bounded at its source by `dispatch.maxConcurrentWorkers`. | | **Tool approval correlation** | Each approval request has a unique `approvalId`; first response wins; multiple concurrent approvals supported. | diff --git a/docs/hooks.md b/docs/hooks.md index b9f6c5f0..d32e1ddb 100644 --- a/docs/hooks.md +++ b/docs/hooks.md @@ -59,7 +59,7 @@ Hooks are declared in `~/.codeoid/config.json`. No plugin loading, no JS API — | `tool_call` | Before a tool executes, **before** the approval gate | block; mutate input | | `tool_result` | After a tool completes | patch the recorded output | | `before_turn` | A fresh turn is starting (not mid-turn injections) | append to the system prompt | -| `after_turn` | Turn finished, carries the normalized result | observe | +| `after_turn` | Turn finished, carries the normalized result. A shutdown waits for it (at most 5 s, with the rest of the turn's end) | observe | | `session_start` | Session created (`source`: `"new"` \| `"resume"`) | observe | | `session_end` | Session destroyed | observe | | `provider_switched` | Backend switched mid-session (`from`, `to`, `seeded`) | observe | diff --git a/docs/routines.md b/docs/routines.md index 55cf162a..2591f48d 100644 --- a/docs/routines.md +++ b/docs/routines.md @@ -142,7 +142,7 @@ So a routine that asks "is anything wrong?" stays quiet on the days when nothing | `over_budget` | It crossed `--max-cost` and was stopped. Counts toward the failure pause. | | `skipped` | It didn't run: the last run was still going (`--overlap skip`), a run was already queued behind it (`--overlap queue`), a persistent routine's session was busy, waiting on you, or had your background work going, the daily spend cap was reached, the routine was paused before it started, or the fire was missed during downtime. | | `cancelled` | You cancelled it (`codeoid routine cancel `). | -| `interrupted` | The daemon stopped while it was in flight. | +| `interrupted` | The daemon stopped while it was in flight. Its spend up to the stop is recorded before the daemon exits (unless its backend didn't end the turn within its settle bound, 2 s, opencode 3 s: then the run is recorded with its cost unknown). | A `needs_attention` run keeps tracking its session after it ends. When you approve and the agent finishes, the run's cost and summary are updated, so the spend counts toward the daily cap and the history shows what was actually done. diff --git a/docs/sandbox.md b/docs/sandbox.md index eeeb6a58..92508a17 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -1352,7 +1352,8 @@ session's first turn) and that changes one of `.qwen/settings.json`, `.codex/`, `.pi/` (at any depth: a session may start in a subdirectory; against each parent, a root commit against nothing) is named in a warning line, with its files (event `sandbox.settings_committed`, -audited `session.settings_committed`). **Review those commits +audited `session.settings_committed`; a turn interrupted by a shutdown +gets its look before the daemon exits, at most 5 s). **Review those commits (`git show `) before a session runs where they are checked out.** The look runs the daemon's hardened git (nothing the repository's config names runs; only names are compared) and reads at most 500 new commits per diff --git a/e2e/specs/lazy-load.spec.ts b/e2e/specs/lazy-load.spec.ts new file mode 100644 index 00000000..e925c261 --- /dev/null +++ b/e2e/specs/lazy-load.spec.ts @@ -0,0 +1,147 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { Page } from "@playwright/test"; +import { type Codeoid, createSessionInUi, expect, focusSession, messages, openUi, send, test, waitIdle } from "../harness"; + +// A resumed session holds its metadata only, and loads its scrollback and +// history the first time it is opened (`session.lazyLoad`, on by default). + +/** Long enough that a few turns overflow the attach tail set below. */ +const FILLER = "lorem ipsum dolor sit amet ".repeat(5).trim(); +const TURNS = 8; + +/** The sidebar row's text, less its relative "created …" line. */ +async function rowText(page: Page, name: string): Promise { + const row = page.getByRole("listitem").filter({ has: page.getByText(name, { exact: true }) }); + return (await row.innerText()).replace(/created .*$/m, "").trim(); +} + +/** Sessions whose history the daemon has loaded since `from` in its log. */ +function loadsSince(codeoid: Codeoid, from: number): string[] { + return codeoid.log + .slice(from) + .split("\n") + .filter((l) => l.includes("history loaded:")); +} + +/** `codeoid ls`, one block per session, sorted (a resume lists newest first), less what attaching changes. */ +async function ls(codeoid: Codeoid): Promise { + return (await codeoid.cliOk("ls")) + .replace(/clients: \d+/g, "clients: _") + .split(/\n\s*\n/) + .map((b) => b.trim()) + .filter((b) => b && !b.startsWith("Sessions:")) + .sort(); +} + +/** Scroll the transcript (virtualized: only what is in view is rendered). */ +async function scrollTranscript(page: Page, to: "top" | "bottom"): Promise { + await page.getByRole("main").evaluate((el, end) => { + type Box = { scrollHeight: number; clientHeight: number; scrollTop: number }; + for (const n of [el, ...el.querySelectorAll("*")] as unknown as Box[]) { + if (n.scrollHeight > n.clientHeight) n.scrollTop = end === "top" ? 0 : n.scrollHeight; + } + }, to); +} + +/** Page back to the session's first message, then show it. */ +async function pageToTop(page: Page): Promise { + const sentinel = page.getByTestId("older-history-sentinel"); + for (let i = 0; i < 40 && (await sentinel.count()) > 0; i++) { + const more = sentinel.getByRole("button"); + // Scrolling it into view may load the page first: the button goes away. + if ((await more.count()) > 0) await more.click({ timeout: 1_000 }).catch(() => {}); + await page.waitForTimeout(100); + } + await expect(sentinel).toHaveCount(0); + await scrollTranscript(page, "top"); +} + +test("resumed sessions list before they load, and load their whole history when opened, across restarts", async ({ page, browser, codeoid }) => { + test.setTimeout(180_000); + // Memory on (embeddings off: nothing to download), so each turn's usage is + // recorded and restored with the session, as on a default install; and a + // small attach tail, so opening the session pages its history. + const configPath = join(codeoid.configDir, "codeoid", "config.json"); + const config = JSON.parse(readFileSync(configPath, "utf8")) as { session: Record }; + writeFileSync( + configPath, + JSON.stringify({ ...config, memory: { enabled: true, embeddings: { enabled: false } }, session: { ...config.session, attachTailBytes: 2048 } }), + ); + await codeoid.restart(); + await openUi(page, codeoid); + await createSessionInUi(page, "long", codeoid.workdir); + for (let i = 1; i <= TURNS; i++) { + await send(page, `turn ${i} ${FILLER}`); + await expect(messages(page).filter({ hasText: `scripted: turn ${i} ` })).toHaveCount(1); + await waitIdle(page); + } + await codeoid.cliOk("new", "other", codeoid.workdir); + // The sidebar lists another client's new session on its next list. + await page.reload(); + await expect(page.getByText("live", { exact: true })).toBeVisible(); + await focusSession(page, "other"); + await send(page, "hello other"); + await expect(messages(page).filter({ hasText: "scripted: hello other" })).toHaveCount(1); + await waitIdle(page); + const rows = { long: await rowText(page, "long"), other: await rowText(page, "other") }; + expect(rows.long).toContain(`${TURNS}t`); + const listed = await ls(codeoid); + const idOf = (name: string) => /id: ([0-9a-f-]{36})/.exec(listed.find((b) => b.startsWith(`${name} `))!)![1]!; + const ids: Record = { long: idOf("long"), other: idOf("other") }; + const longId = ids.long!; + const loadsOf = (name: string, from: number) => loadsSince(codeoid, from).filter((l) => l.includes(ids[name]!.slice(0, 8))); + + + // This page re-attaches its focused session when it reconnects: park it. + await page.goto("about:blank"); + + for (const round of [1, 2]) { + const from = codeoid.log.length; + await codeoid.restart(); + + // A fresh browser: nothing focused, so nothing is opened. + const ctx = await browser.newContext(); + const fresh = await ctx.newPage(); + const sent: string[] = []; + fresh.on("websocket", (ws) => ws.on("framesent", (f) => sent.push(String(f.payload)))); + await openUi(fresh, codeoid); + await expect(fresh.getByRole("listitem").filter({ has: fresh.getByText("long", { exact: true }) })).toBeVisible(); + // The list says what it said before the restart. The UI opens the newest + // session by itself; the other one, listed but not opened, isn't loaded. + expect(await rowText(fresh, "long")).toBe(rows.long); + expect(await rowText(fresh, "other")).toBe(rows.other); + expect(await ls(codeoid)).toEqual(listed); + const opened = (await fresh.getByRole("heading", { level: 2 }).innerText()).trim(); + const notOpened = opened === "long" ? "other" : "long"; + expect(loadsOf(notOpened, from)).toEqual([]); + + // Opening one loads it (and only it): the newest turns at once, the rest paged. + await focusSession(fresh, "long"); + await expect(messages(fresh).filter({ hasText: `scripted: turn ${TURNS} ` })).toHaveCount(1); + await pageToTop(fresh); + await expect(messages(fresh).filter({ hasText: "scripted: turn 1 " })).toBeVisible(); + // The first turns came by paging: the attach sent only the newest. + expect(sent.some((f) => f.includes('"scrollback.page"') && f.includes(longId))).toBe(true); + // Loaded once, when opened. + expect(loadsOf("long", from)).toHaveLength(1); + + // It takes a new prompt and answers it. + await scrollTranscript(fresh, "bottom"); + await send(fresh, `after restart ${round}`); + await scrollTranscript(fresh, "bottom"); + await expect(messages(fresh).filter({ hasText: `scripted: after restart ${round}` })).toHaveCount(1); + await waitIdle(fresh); + rows.long = await rowText(fresh, "long"); + expect(rows.long).toContain(`${TURNS + round}t`); + await ctx.close(); + } + + // The first browser signs in again and finds the whole history. + await openUi(page, codeoid); + await focusSession(page, "long"); + await expect(messages(page).filter({ hasText: "scripted: after restart 2" })).toHaveCount(1); + await expect(messages(page).filter({ hasText: "scripted: after restart 1" })).toHaveCount(1); + await pageToTop(page); + await expect(messages(page).filter({ hasText: "scripted: turn 1 " })).toBeVisible(); +}); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json index 6edbf2e5..6a25fc22 100644 --- a/e2e/tsconfig.json +++ b/e2e/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "../tsconfig.json", "compilerOptions": { + "tsBuildInfoFile": "../.cache/tsc/e2e.tsbuildinfo", "rootDir": "..", "declaration": false, "sourceMap": false diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 81066e52..0c53cb30 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { + "tsBuildInfoFile": "../../.cache/tsc/core.tsbuildinfo", "rootDir": "src", "noEmit": true }, diff --git a/packages/protocol/tsconfig.json b/packages/protocol/tsconfig.json index 81066e52..1f31054e 100644 --- a/packages/protocol/tsconfig.json +++ b/packages/protocol/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { + "tsBuildInfoFile": "../../.cache/tsc/protocol.tsbuildinfo", "rootDir": "src", "noEmit": true }, diff --git a/src/config.ts b/src/config.ts index 6cf02c6f..ec97dcfd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -549,6 +549,15 @@ const SessionSchema = z * restart. */ resumeMaxSessions: z.number().int().min(1).default(200), + /** + * Resume sessions without loading their history: a resumed session holds + * its metadata only, and loads its scrollback and canonical history the + * first time something uses them (attach, a prompt, a fork, …). Most + * resumed sessions are never opened before the next restart, and their + * history was most of the daemon's memory. Off = load every session's at + * start, as before. + */ + lazyLoad: z.boolean().default(true), /** * Per-turn workspace snapshots (#354): at the start of every turn the * session's working directory (minus .gitignore'd files, secrets and @@ -604,6 +613,7 @@ const SessionSchema = z mcpToolTimeoutMs: 120_000, attachTailBytes: 512 * 1024, resumeMaxSessions: 200, + lazyLoad: true, seedBudgetChars: 0, seedCharsPerToken: 2, seedWindowFraction: 0.7, @@ -1554,6 +1564,8 @@ export interface CodeoidConfig { idleReleaseMinutes?: number; /** Sessions restored from disk at daemon start, newest-first by last activity. Resume is also time-boxed, so this is the coarse guard; the remainder stays on disk. Defaults to 200 when omitted. */ resumeMaxSessions?: number; + /** Load a resumed session's history the first time it is used, not at start. Defaults to true when omitted. */ + lazyLoad?: boolean; /** Per-turn workspace snapshots (#354). Enabled unless `enabled: false`. */ checkpoints?: { enabled?: boolean; @@ -1868,6 +1880,7 @@ const ENV_OVERRIDES: readonly EnvOverride[] = [ { env: "CODEOID_TURN_STALL_TIMEOUT_MS", path: "session.turnStallTimeoutMs", kind: "int" }, { env: "CODEOID_MCP_TOOL_TIMEOUT_MS", path: "session.mcpToolTimeoutMs", kind: "int" }, { env: "CODEOID_RESUME_MAX_SESSIONS", path: "session.resumeMaxSessions", kind: "int" }, + { env: "CODEOID_SESSION_LAZY_LOAD", path: "session.lazyLoad", kind: "boolean" }, { env: "CODEOID_SEED_BUDGET_CHARS", path: "session.seedBudgetChars", kind: "int" }, { env: "CODEOID_SEED_CHARS_PER_TOKEN", path: "session.seedCharsPerToken", kind: "float" }, { env: "CODEOID_SEED_WINDOW_FRACTION", path: "session.seedWindowFraction", kind: "float" }, diff --git a/src/daemon/dispatch.ts b/src/daemon/dispatch.ts index 9a991ac9..ecb6f535 100644 --- a/src/daemon/dispatch.ts +++ b/src/daemon/dispatch.ts @@ -95,6 +95,14 @@ export const DEFAULT_DISPATCH_CONFIG: DispatchConfig = { holdMaxMs: 2 * 60 * 60_000, }; +/** + * A send refused because the daemon is shutting down (session.ts's + * DaemonStoppingError, by name: this module stays free of the session's). + */ +function isDaemonStopping(err: unknown): boolean { + return err instanceof Error && err.name === "DaemonStoppingError"; +} + /** Thrown by host methods when retrying can never succeed (target gone, bad input). */ export class NonRetryableDispatchError extends Error {} @@ -138,6 +146,8 @@ export interface DispatcherHost { workerStatus(sessionId: string): SessionStatus | null; /** Compressed completion digest for a finished worker — never raw transcript. */ buildWorkerDigest(task: DispatchTaskRow): string; + /** Load what `buildWorkerDigest` reads (a resumed worker's history), before it runs. */ + prepareWorkerDigest?(task: DispatchTaskRow): Promise; /** Tear down a finished worker session (best-effort). */ destroyWorker(sessionId: string, reason: string): Promise; /** @@ -304,6 +314,7 @@ export class Dispatcher { start(): void { if (!this.#config.enabled || this.#timer) return; + this.#stopped = false; this.#timer = setInterval(() => { void this.tick(); }, this.#config.tickMs); @@ -312,7 +323,11 @@ export class Dispatcher { void this.tick(); // first pass immediately: boot-time reclaim + resume } + /** Set by stop(): a tick still in flight claims nothing more. */ + #stopped = false; + stop(): void { + this.#stopped = true; if (this.#timer) { clearInterval(this.#timer); this.#timer = null; @@ -740,6 +755,8 @@ export class Dispatcher { // starve the rest of the queue). const touched: string[] = []; for (;;) { + // The daemon is stopping: what is still queued waits for the next start. + if (this.#stopped) return; const task = this.#store.dispatchClaimNext( this.bootId, Date.now(), @@ -814,6 +831,18 @@ export class Dispatcher { this.#host.audit("dispatch.spawned", `task=${task.id} worker=${sessionId}`); } catch (err) { const message = err instanceof Error ? err.message : String(err); + // The daemon began shutting down before the work started (its send was + // refused, the worker torn down): nothing ran, so no attempt is spent. + // Back to the queue untouched, for the next start. Only a claim: a + // grouped send is marked running with its target before delivery (a + // retry takes it as delivered), and an attempt is its stopped loop's. + const current = isDaemonStopping(err) && task.kind !== "attempt" ? this.#store.dispatchGet(task.id) : null; + if (current?.status === "claimed" && current.generation === task.generation) { + this.#unwatchTask(task.id); + this.#store.dispatchRelease(task.id, Date.now()); + this.#host.audit("dispatch.released", `task=${task.id} reason=daemon_stopping`); + return; + } const retryable = !(err instanceof NonRetryableDispatchError); const now = Date.now(); const status = this.#store.dispatchFail(task.id, task.generation, message, now, { @@ -1048,6 +1077,7 @@ export class Dispatcher { try { // Digest BEFORE teardown — it reads the live session + memory. + await this.#host.prepareWorkerDigest?.(task); const digest = this.#host.buildWorkerDigest(task); if (status === "idle") { // The read above and this write are not one transaction; the store's diff --git a/src/daemon/hooks/bus.ts b/src/daemon/hooks/bus.ts index 697f75c2..c0d9bbec 100644 --- a/src/daemon/hooks/bus.ts +++ b/src/daemon/hooks/bus.ts @@ -187,16 +187,18 @@ export class HookBus { * Fire-and-forget dispatch for observe-only events (`after_turn` + * lifecycle). Outcomes are ignored; failures are logged by #run. Never * blocks the caller — lifecycle seams (constructor, destroy) are not - * places to await user scripts. + * places to await user scripts. The returned promise settles once every + * hook ran (never rejects), for a caller that must not cut them off: a + * turn's end, waited for at shutdown (`Session.turnEndWork`). */ emit( event: HookEvent, ctx: HookSessionContext, payload: Record, - ): void { + ): Promise { const entries = this.#matching(event); - if (entries.length === 0) return; - void (async () => { + if (entries.length === 0) return Promise.resolve(); + return (async () => { for (const entry of entries) { await this.#run(entry, ctx, { event, ...payload }); } diff --git a/src/daemon/providers/interface.ts b/src/daemon/providers/interface.ts index 81d21c99..4a9bc6ed 100644 --- a/src/daemon/providers/interface.ts +++ b/src/daemon/providers/interface.ts @@ -816,6 +816,12 @@ export interface SessionProvider extends AgentProvider { * wakes itself only as a fallback if no turn begins. */ readonly continuesAfterBackgroundWork?: boolean; + /** + * How long, at most, the backend takes to end a turn (its result, spend + * included) after an interrupt. A daemon shutting down waits that long for + * an interrupted turn's end, then gives up on it (`DRAIN_SETTLE_MS` when unset). + */ + readonly interruptSettleMs?: number; /** * Stop background tasks by id (the ids of the `background_tasks` level). * What Stop does when no turn is in flight: without it, nobody holding only diff --git a/src/daemon/providers/opencode/index.ts b/src/daemon/providers/opencode/index.ts index 5484339d..ee8b231d 100644 --- a/src/daemon/providers/opencode/index.ts +++ b/src/daemon/providers/opencode/index.ts @@ -343,6 +343,13 @@ export class OpencodeProvider implements SessionProvider { readonly appliesToolInputRewrite = false; /** A background subagent's result is handed to the model by opencode, which then runs the main agent itself. */ readonly continuesAfterBackgroundWork = true; + /** + * An interrupt before the prompt started ends the turn after a second ask + * (250 ms), the settle timer (300 ms) and the read-back with its total + * (`SPEND_READ_MS` + 250): 1.8 s on 2.2.5, mid-reply a few ms (I71). A feed + * gap's longer read-back (`INTERRUPT_SETTLE_MS`) isn't waited for at shutdown. + */ + readonly interruptSettleMs = 3_000; readonly effortChange = "turn" as const; onRecoveryNeeded: ((content: string) => void) | undefined; diff --git a/src/daemon/providers/pi/index.ts b/src/daemon/providers/pi/index.ts index 7be78236..521c5f01 100644 --- a/src/daemon/providers/pi/index.ts +++ b/src/daemon/providers/pi/index.ts @@ -298,13 +298,17 @@ export class PiProvider implements SessionProvider { this.#gatedToolIds.clear(); this.#hasQueried = true; - void this.#startTurn(opts).catch((err) => { + // An interrupt before the prompt went out (pi still starting, its model + // or level being set) has nothing to abort yet: the prompt mustn't go out after it. + const turn = { aborted: false }; + void this.#startTurn(opts, turn).catch((err) => { this.#failTurn(err instanceof Error ? err.message : String(err)); }); return { events: queue, interrupt: async () => { + turn.aborted = true; this.#proc?.send({ type: "abort" }); }, pushMidTurn: (content, priority) => { @@ -319,7 +323,8 @@ export class PiProvider implements SessionProvider { }; } - async #startTurn(opts: TurnOpts): Promise { + async #startTurn(opts: TurnOpts, turn: { aborted: boolean }): Promise { + const queue = this.#turnQueue; await this.#ensureProcess(opts); if (!this.#bridgeReady) { @@ -338,6 +343,12 @@ export class PiProvider implements SessionProvider { const level = opts.effort ?? this.#defaultThinkingLevel; if (level && level !== this.#thinkingLevel) await this.#setThinkingLevel(level); + if (turn.aborted) { + // Interrupted before pi had a prompt to abort: the turn ends here, its + // history seed kept for the next prompt. + if (this.#turnQueue === queue) this.#endAborted(); + return; + } let message = opts.userMessage; if (this.#pendingHistorySeed) { message = `${this.#pendingHistorySeed}\n\n${message}`; @@ -618,6 +629,30 @@ export class PiProvider implements SessionProvider { this.#canUseTool = null; } + /** End a turn interrupted before its prompt was sent: nothing ran, nothing was spent. */ + #endAborted(): void { + const queue = this.#turnQueue; + if (!queue) return; + this.#turnQueue = null; + this.#canUseTool = null; + queue.push({ + type: "turn_done", + result: { + providerId: this.id, + model: this.#model ?? "pi-default", + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheCreationTokens: 0, + totalCostUsd: 0, + costSource: "backend", + durationMs: Date.now() - this.#turnStartedAt, + stopReason: "interrupted", + }, + }); + queue.close(); + } + /** * Abort the in-flight turn queue with an error event (spawn failure, * process death, teardown). Idempotent and double-close-safe: the process diff --git a/src/daemon/routines/service.ts b/src/daemon/routines/service.ts index 574cb3e9..94aa1683 100644 --- a/src/daemon/routines/service.ts +++ b/src/daemon/routines/service.ts @@ -581,6 +581,11 @@ function commandDigest(command: string | undefined): string { /** Midnight UTC of the day `ms` falls in — the daily spend cap's window. */ /** How long a stop on a held session gets to land before it's tried again, and how often it is. */ +/** + * At shutdown a run's wind-down polls every 10 ms for its turn to end; this + * many polls (about drain's 10 s) back up `settled()`, which ends it sooner. + */ +const STOPPING_POLLS = 1_000; const STOP_RETRY_MS = 10_000; const STOP_TRIES = 3; @@ -662,6 +667,8 @@ export class RoutineService { #timer: ReturnType | null = null; #ticking = false; readonly #live = new Map(); + /** Each live run's execution, until it has recorded its end. */ + readonly #running = new Set>(); /** sessionId → the live run waiting on it. */ readonly #bySession = new Map(); /** @@ -675,6 +682,12 @@ export class RoutineService { /** Routines being deleted: nothing fires or starts for them, and a run that ends meanwhile (its loop cancelled) tells no one. */ readonly #deleting = new Set(); #stopped = false; + /** + * Set by `settled()` at shutdown: the daemon's drain has waited for every + * turn as long as it will (each its backend's settle bound). A run whose + * turn hasn't ended by now records its row with the cost unknown. + */ + #settling = false; constructor( store: RoutineStore, @@ -711,6 +724,7 @@ export class RoutineService { start(): void { if (!this.config.enabled || this.#timer) return; this.#stopped = false; + this.#settling = false; // First: a run whose loop started but which died before it recorded the // loop (or was closed as interrupted for it) follows that loop again. this.#relinkLoops(); @@ -736,6 +750,27 @@ export class RoutineService { } } + /** + * After `stop()`, at shutdown, once the daemon's drain has waited for the + * turns: resolves true once every live run has recorded its end (its row, + * spend included), false after `timeoutMs`. A run whose turn drain gave up + * on records it now, the cost unknown. Drain awaits it before the store closes. + */ + async settled(timeoutMs: number): Promise { + this.#settling = true; + if (this.#running.size === 0) return true; + let timer: ReturnType | undefined; + const ended = await Promise.race([ + Promise.allSettled([...this.#running]).then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + timer.unref?.(); + }), + ]); + clearTimeout(timer); + return ended; + } + /** * A loop is started before the routine run records it, so a daemon that * died in between left a loop run in progress that no routine run follows. @@ -976,11 +1011,13 @@ export class RoutineService { abort: new AbortController(), }; this.#live.set(run.id, live); - void this.#execute(routine, run, live).finally(() => { + const running = this.#execute(routine, run, live).finally(() => { this.#live.delete(run.id); + this.#running.delete(running); if (live.sessionId) this.#bySession.delete(live.sessionId); this.#drainQueue(); }); + this.#running.add(running); } } @@ -1247,9 +1284,14 @@ export class RoutineService { if (force) await this.#host.interrupt(id, { turnOnly: true }).catch(() => {}); let stoppedTasks = false; let quiet = 0; - for (let i = 0; i < 100; i++) { + // Stopping: as long as the daemon's drain waits for the turn (its + // backend's settle bound), not a fixed count; `settled()` ends it. + // STOPPING_POLLS only backs that up, for a stop with no drain after it. + for (let i = 0; i < (this.#stopped ? STOPPING_POLLS : 100); i++) { const info = this.#host.info(id); if (!info) return; + // Drain has given up on the turn: record the run now, before the store closes. + if (this.#settling && info.turnInFlight) break; if (info.turnInFlight && info.lastTurnId !== turnAtStop) { turnAtStop = info.lastTurnId; unsettled = true; @@ -1272,12 +1314,14 @@ export class RoutineService { stoppedTasks = true; quiet = 0; await this.#host.stopBackgroundTasks(id, left).catch(() => {}); - } else if (!(force || stoppedTasks) || ++quiet >= 3) { + } else if (!(force || stoppedTasks) || ++quiet >= 3 || this.#stopped) { // After a stop, a few quiet polls: a wake that slipped in shows up as a turn. + // Not when the daemon is stopping: no wake is delivered then, and the store closes next. return; } } - await this.#sleep(100); + // Stopping: the row is written as soon as the turn has ended, before the store closes. + await this.#sleep(this.#stopped ? 10 : 100); } unsettled = true; }; diff --git a/src/daemon/session-manager.ts b/src/daemon/session-manager.ts index 7dedb143..dde7f573 100644 --- a/src/daemon/session-manager.ts +++ b/src/daemon/session-manager.ts @@ -13,7 +13,8 @@ import { type RequestDirectoryInput, type SandboxMcpMount, SandboxMcpHttp } from import { existsSync, mkdirSync, realpathSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join, resolve, sep } from "node:path"; -import { type ForkPoint, RewindError, SendStoppedError, Session, type AttachedClient, type WindowScope } from "./session.js"; +import { type ForkPoint, RewindError, type SendHold, SendStoppedError, SHUTDOWN_AUTH, Session, type AttachedClient, type WindowScope } from "./session.js"; +import type { CanonicalTurn } from "./providers/canonical.js"; import { sweepCheckpoints } from "./checkpoints.js"; import type { CompareRunRow } from "./store.js"; import { type CompareState, type CompareTargetState, type ErrorCode, LIMITS, backendHasAccounts, DEFAULT_ACCOUNT, type AccountKind, type ClaudeAccountKind } from "../protocol/types.js"; @@ -33,7 +34,6 @@ import { applyPatches, getManifest, getSnapshot, previewPatches, readMcpServerEn import { adoptability, adoptableName, adoptedEntry, adoptionRefusal, claudeMcpServers, fromRepoFile, fullUrlRequired, isSameAdoption, shellArg } from "./mcp/adopt.js"; import { readWorkdirServers } from "./mcp/workdir-servers.js"; import { cleanMcpError, commandSummary, MAX_SHOWN_KEYS, quotedVisible, redactMcpUrl, serverProgram, shownLabel, visibleCompact, visibleVerbatim } from "./mcp/redact.js"; -import { canonicalFromTranscript, turnIndexFromHistory } from "./canonical-restore.js"; import { BackendLoginBroker, BackendLoginError, type LoginOutcome, redact } from "./auth/backend-login.js"; import { ClaudeAccountError, ClaudeAccountStore } from "./claude-accounts.js"; import { AccountError, type AccountStore, LOGIN_KINDS, validateAccountName } from "./accounts.js"; @@ -364,6 +364,15 @@ function toFleetEventWire(row: DispatchEventRow): FleetEventWire { * configured value is `session.resumeMaxSessions`. */ const RESUME_MAX_SESSIONS_DEFAULT = 200; const RESUME_DEADLINE_MS = 20_000; +/** + * At shutdown, how long drain waits, after the turns ended, for what their + * ends started (Session.turnEndWork: the #448 git check, the end snapshot, + * hooks) and for routine runs to record theirs. Measured: the settings-commit + * look over a 3000-commit repository, and the end snapshot of a 20 000-file + * workspace, each well under a second; the margin is for a large repository + * on a loaded machine. + */ +export const TURN_END_SETTLE_MS = 5_000; /** Per-session transcript read budget on resume. Scrollback keeps at most * 20 MiB / 5000 messages — parsing history past that would be evicted on * arrival, so cap the read slightly above the scrollback byte cap. */ @@ -406,6 +415,71 @@ export function cutRowsAfterTurn( */ const RESUME_CANONICAL_MAX_BYTES = 4 * 1024 * 1024; +/** + * Messages that name a session but never read its scrollback or canonical + * history: they don't load a resumed session's (lazy session loading). A + * notice one of them adds is kept for the load (`Session#buffer`); anything + * that does read the history loads it there (`Session#hydrateNow`). + */ +const NO_HISTORY_MESSAGES: ReadonlySet = new Set([ + "session.detach", + "session.destroy", + "session.interrupt", + "session.ui_response", + "session.commands", + "session.set_mode", + "session.set_sandbox", + "session.pin", + "session.unpin", + "session.rename", + "session.turns", + "session.approvals", + "session.set_model", + "session.set_effort", + "session.set_account", + "session.set_claude_account", + "session.mcp_reconnect", + "session.mcp_refresh", + "session.export", + "claude.config", + "mcp.approve", + "mcp.approve.preview", + "fs.list", + "fs.read", + "fs.browse_dir", + "blackboard.index", + "blackboard.read", + "collaboration.panels", + "compare.get", + "compare.list", + "compare.keep", + "mcp.adopt", +]); + +/** + * The scopes each history-reading message's handler requires (every group, + * any scope within one): a caller without them is refused there, so its + * message loads nothing. Types not listed load for any caller of the tenant. + */ +const HISTORY_SCOPES: Readonly>> = { + "session.attach": [[SCOPES.SESSION_ATTACH, SCOPES.SESSION_WATCH]], + "scrollback.page": [[SCOPES.SESSION_ATTACH, SCOPES.SESSION_WATCH]], + "session.send": [[SCOPES.SESSION_SEND]], + "session.part_action": [[SCOPES.SESSION_SEND]], + "session.rotate": [[SCOPES.SESSION_SEND]], + "session.rewind": [[SCOPES.SESSION_SEND]], + "session.approve": [[SCOPES.SESSION_APPROVE]], + "session.set_provider": [[SCOPES.SESSION_APPROVE]], + "session.fork": [[SCOPES.SESSION_CREATE]], + "session.compare": [[SCOPES.SESSION_CREATE], [SCOPES.SESSION_SEND]], +}; + +/** Whether `auth` holds the scopes `type`'s handler requires before it reads a session's history. */ +function mayReadHistory(type: string, auth: AuthContext): boolean { + const groups = HISTORY_SCOPES[type]; + return !groups || groups.every((any) => any.some((scope) => hasScope(auth.scopes as string[], scope))); +} + /** Sort key for resume ordering: most-recently-active first. Falls back to * createdAt, then 0, so a malformed timestamp never throws. */ function resumeSortKey(m: { lastActivityAt?: string; createdAt?: string }): number { @@ -422,6 +496,12 @@ const LOOP_WORKER_PROVIDERS: ReadonlySet = new Set(["claude"]); export class SessionManager { #sessions = new Map(); + /** Per session, the newest message waiting on its history's load (`#historyGate`): the next one waits behind it. */ + #historyGates = new Map>(); + /** Clients with messages waiting on a load, and how many. */ + #clientsWaiting = new Map(); + /** Of those, the ones that disconnected meanwhile (`disconnectClient`). */ + #clientsGone = new Set(); #store: Store; #transcriptStore: TranscriptStore; #identityManager?: AgentIdentityManager; @@ -958,6 +1038,7 @@ export class SessionManager { if (m.collaboration) goalConfigs.set(m.sessionId, m.collaboration); } const deadline = Date.now() + RESUME_DEADLINE_MS; + const lazyLoad = this.#config?.session?.lazyLoad ?? true; let resumed = 0; let skippedDeadline = 0; let resumedChildren = 0; @@ -1084,54 +1165,22 @@ mcpHub: this.#mcpHub, session.adoptReportedBackgroundTasks(meta.reportedBackgroundTasks); session.adoptKnownBackgroundTasks(meta.knownBackgroundTasks); - // Restore scrollback from transcript, seeding the seq counter past - // the persisted tail so new appends continue the monotonic sequence. - // Byte-budgeted + deadline-aware: one huge transcript can neither - // OOM the daemon nor eat the whole resume window by itself. - const loadStats: { truncated?: boolean } = {}; - const entries = await this.#transcriptStore.loadTranscript(meta.sessionId, { - maxBytes: RESUME_TRANSCRIPT_MAX_BYTES, - deadlineAt: deadline, - stats: loadStats, - }); - const messages = entries.map((e) => e.message); - const maxSeq = entries.reduce((max, e) => Math.max(max, e.seq), -1); - session.restoreScrollback(messages, maxSeq + 1, entries.map((e) => e.bytes), { - partialHistory: loadStats.truncated === true, - }); - // The backend-neutral conversation (#354). Without it a restored - // session forked / switched backend with no history, and stateless - // backends answered the next prompt with no memory of the session. - // Sessions from before the log existed are rebuilt from the - // transcript once, then read from the log. - try { - const canonical = await this.#transcriptStore.loadCanonical(meta.sessionId, { - maxBytes: RESUME_CANONICAL_MAX_BYTES, - }); - const index = await this.#transcriptStore.loadTurnIndex(meta.sessionId); - // No log: a session from before it existed. Rebuild once from the - // WHOLE transcript (the scrollback window above may be a tail) and - // persist it — a long session's history must not shrink to the - // window, or to just the next turn on the following restart. - let rebuildFrom = messages; - if (!canonical && loadStats.truncated === true) { - const all = await this.#transcriptStore.loadTranscript(meta.sessionId, { deadlineAt: deadline }); - rebuildFrom = all.map((e) => e.message); - } - const history = - canonical?.turns ?? (rebuildFrom.length > 0 ? canonicalFromTranscript(rebuildFrom, providerId) : []); - session.restoreTurns(history, index ?? turnIndexFromHistory(history), { - persistHistory: !canonical && history.length > 0, - persistIndex: !index && history.some((t) => t.turnId), - partial: canonical?.partial ?? false, - }); - if (meta.redactedBackings) session.adoptRedactedBackings(meta.redactedBackings); - if (meta.backingFresh) await session.reseedFreshBacking(); - } catch (err) { - console.error( - `[codeoid/resume] canonical history restore failed for ${meta.sessionId} (session resumes without it): ${err instanceof Error ? err.message : String(err)}`, - ); - } + // The history (scrollback + canonical conversation) loads the first + // time something uses the session (`session.lazyLoad`, default on): + // only what it reports and appends with before then is read here. + // Sessions from before the canonical log or the turn index load now, + // since the load rebuilds and persists them. + if (meta.redactedBackings) session.adoptRedactedBackings(meta.redactedBackings); + const loadNow = !(await session.prepareResume({ + transcriptMaxBytes: RESUME_TRANSCRIPT_MAX_BYTES, + canonicalMaxBytes: RESUME_CANONICAL_MAX_BYTES, + deadlineMs: RESUME_DEADLINE_MS, + providerId, + backingFresh: meta.backingFresh === true, + })); + // A fresh backing is re-seeded at resume, as before: it is rare (a + // rewind not yet followed by a turn) and its backend is owed it now. + if (loadNow || !lazyLoad || meta.backingFresh) await session.ensureHydrated({ deadlineAt: deadline, atResume: true }); this.#sessions.set(session.id, session); // Track a resumed child's mount so teardown revokes it. Without this a @@ -1152,8 +1201,12 @@ mcpHub: this.#mcpHub, // "Concurrent session limit reached" until the user // /destroys some. resumed++; - } catch { - // Skip sessions that fail to resume + } catch (err) { + // Skip sessions that fail to resume (a transcript that can't be + // read): left on disk as they are, tried again at the next start. + console.error( + `[codeoid/resume] could not resume ${meta.sessionId}: ${err instanceof Error ? err.message : String(err)}`, + ); } } @@ -1333,6 +1386,49 @@ mcpHub: this.#mcpHub, */ rawToken?: string; }, + ): Promise { + // A resumed session loads its history the first time it is used. Only + // then is there an await here: handlers that claim state synchronously + // (a rewind's stop point) keep doing so before the caller's next call. + const gate = this.#historyGate(msg, auth); + if (!gate) return this.#route(msg, auth, client, opts); + return this.#routeAfterLoad(gate, msg, auth, client, opts); + } + + /** `handle`, once the session `msg` names has loaded its history (`#historyGate`). */ + async #routeAfterLoad( + gate: { session: Session; wait: Promise }, + msg: ClientMessage, + auth: AuthContext, + client: AttachedClient, + opts?: { rawToken?: string }, + ): Promise { + // A prompt waiting on the load is a send preparing its turn: the drain + // sees it, and a Stop meanwhile stops it, as a queued send. + const hold = msg.type === "session.send" ? gate.session.holdSend() : undefined; + this.#clientsWaiting.set(client.id, (this.#clientsWaiting.get(client.id) ?? 0) + 1); + try { + const unloadable = await gate.wait; + if (unloadable) return unloadable; + return await this.#route(msg, auth, client, opts, hold); + } finally { + hold?.release(); + const waiting = (this.#clientsWaiting.get(client.id) ?? 1) - 1; + if (waiting > 0) this.#clientsWaiting.set(client.id, waiting); + else { + this.#clientsWaiting.delete(client.id); + // Gone while its messages waited: what they attached it to lets go of it. + if (this.#clientsGone.delete(client.id)) this.disconnectClient(client.id); + } + } + } + + async #route( + msg: ClientMessage, + auth: AuthContext, + client: AttachedClient, + opts?: { rawToken?: string }, + sendHold?: SendHold, ): Promise { switch (msg.type) { case "ping": @@ -1366,7 +1462,7 @@ mcpHub: this.#mcpHub, case "session.detach": return this.#detach(msg, client); case "session.send": - return this.#send(msg, auth); + return this.#send(msg, auth, sendHold); case "session.interrupt": return this.#interrupt(msg, auth); case "session.approve": @@ -1585,6 +1681,51 @@ mcpHub: this.#mcpHub, } } + /** + * Load the history of the session `msg` names (lazy session loading), so + * every handler finds it in memory: the one place a client's use of a + * session loads it. Only for a session the caller's tenant owns, with the + * scope its handler needs (`HISTORY_SCOPES`), and not for messages that + * never read it (`NO_HISTORY_MESSAGES`). While a load is awaited here, + * every later message naming that session waits behind it, whether it + * reads the history or not, so they are handled in the order they came + * (an interrupt never overtakes the prompt it stops); other sessions' + * don't. Null when nothing waits; `wait` resolves to an error response + * when the transcript can't be read. + */ + #historyGate(msg: ClientMessage, auth: AuthContext): { session: Session; wait: Promise } | null { + const id = (msg as { sessionId?: unknown }).sessionId; + if (typeof id !== "string") return null; + const session = this.#sessions.get(id); + if (!session || session.accountId !== auth.accountId || session.projectId !== auth.projectId) return null; + const load = (): Promise | null => { + if (NO_HISTORY_MESSAGES.has(msg.type) || !session.historyPending || !mayReadHistory(msg.type, auth)) return null; + if (this.#refusedBeforeHistory(msg, auth)) return null; + return session.ensureHydrated().then( + () => null, + (err): DaemonMessage => { + console.error( + `[codeoid/resume] could not load the history of ${id}: ${err instanceof Error ? err.message : String(err)}`, + ); + return { + type: "response.error", + requestId: (msg as { id?: string }).id ?? "", + error: "The session's history could not be loaded from disk — try again, or see the daemon's log.", + code: "internal", + }; + }, + ); + }; + const behind = this.#historyGates.get(id); + const wait = behind ? behind.then(load) : load(); + if (!wait) return null; + this.#historyGates.set(id, wait); + void wait.finally(() => { + if (this.#historyGates.get(id) === wait) this.#historyGates.delete(id); + }); + return { session, wait }; + } + async #sessionExport( msg: Extract, auth: AuthContext, @@ -1813,6 +1954,10 @@ mcpHub: this.#mcpHub, }, ); + // The imported log's lines are seqs 1..N: the session's own appends + // continue past them, so its seqs only grow along the log. + this.#sessions.get(result.newSessionId)?.restoreScrollback([], result.importedMessages + 1); + return { type: "session.import.result", requestId: msg.id, @@ -3960,6 +4105,9 @@ mcpHub: this.#mcpHub, /** Remove a client from all sessions (e.g. on disconnect). */ disconnectClient(clientId: string): void { + // A message of its own still waiting on a session's load runs after + // this: whatever it attaches the client to is let go of then. + if (this.#clientsWaiting.has(clientId)) this.#clientsGone.add(clientId); for (const session of this.#sessions.values()) { session.detach(clientId); } @@ -3987,16 +4135,27 @@ mcpHub: this.#mcpHub, } /** - * Graceful drain — wait for all in-flight sessions to reach idle. + * Graceful drain: stop every turn and wait for each to record its end. * Used during shutdown. * - * Each working session is interrupted **once**. The previous loop - * re-interrupted on every poll iteration; if the SDK subprocess - * was mid HTTP retry and didn't respond within 500 ms, drain - * piled up duplicate "interrupted by system:shutdown" info rows - * (up to 20 in a 10-s window) and 20× the audit log churn. Track - * a per-session "already interrupted" set, then poll status until - * idle or deadline. + * From its start no turn starts: every session refuses sends, interrupts a + * turn its backend starts on its own and delivers no wake (`beginDrain`). + * Each running turn is interrupted **once** (a re-interrupt per poll piled + * up duplicate "interrupted by system:shutdown" rows when the SDK was mid + * HTTP retry). Then drain waits, polling, for every turn that hasn't + * recorded its end, the ones stopped just before it began included: idle + * isn't ended, since a session goes idle once its backend acknowledges the + * interrupt and the turn's result follows. Each is waited for at most its + * backend's settle bound (`Session.interruptSettleMs`), all at once, so a + * backend that never ends the turn holds shutdown that long, not to the + * deadline; it is named in the log and left. Then, all at once, what the + * turns' ends started and don't await is waited for (`Session.turnEndWork`: + * the #448 git check's audit row and warning line, the end snapshot, hooks) + * and the routine runs record their rows, spend included, at most + * `TURN_END_SETTLE_MS` together; then the transcript writes are flushed. + * The store closes after drain, so nothing a turn's end started is cut off. + * + * One-way: the manager refuses every send from then on. It is shutdown's. */ async drain(timeoutMs = 10_000): Promise { try { @@ -4007,7 +4166,13 @@ mcpHub: this.#mcpHub, } } + /** Set when drain begins: no session.send is taken from then on. */ + #draining = false; + async #drain(timeoutMs: number): Promise { + // Before anything else: nothing starts a turn from here on. + this.#draining = true; + for (const s of this.#sessions.values()) s.beginDrain(); // An abandoned login is a live pty holding an open OAuth attempt. Kill // those first, and unconditionally — they are not work worth draining. this.#backendLogin.dispose(); @@ -4019,31 +4184,102 @@ mcpHub: this.#mcpHub, for (const t of this.#evidenceTimers) clearTimeout(t); this.#evidenceTimers = []; const deadline = Date.now() + timeoutMs; - const systemAuth: AuthContext = { - sub: "system:shutdown", - scopes: [], - delegationDepth: 0, - accountId: "", - projectId: "", - }; - const interrupted = new Set(); + const remaining = () => Math.max(0, deadline - Date.now()); + // Per turn (its run; the session for a send still preparing): when drain + // first found it, so a turn started after an interrupt is a new one. + const since = new Map(); + const givenUp = new Set(); + // Turns usually settle within milliseconds of the interrupt: look again + // soon, backing off only to 50 ms. + let pollMs = 10; while (Date.now() < deadline) { - const working = [...this.#sessions.values()].filter( - (s) => - s.status === "thinking" || - s.status === "tool_running" || - s.status === "waiting_approval" || + let waiting = 0; + for (const session of this.#sessions.values()) { + // A session made since drain began. + session.beginDrain(); + const working = + session.status === "thinking" || + session.status === "tool_running" || + session.status === "waiting_approval" || // Looks idle, but a queued send is about to start a turn: stop it // too, or it starts after shutdown began. - s.preparingTurn, - ); - if (working.length === 0) return; - for (const session of working) { - if (interrupted.has(session.id)) continue; - void session.interrupt(systemAuth); - interrupted.add(session.id); + session.preparingTurn; + if (!working && !session.turnUnsettled) continue; + const key = session.drainKey; + if (givenUp.has(key)) continue; + const first = since.get(key); + if (first === undefined) { + since.set(key, Date.now()); + // Stopped already (by its owner, a routine, or as it started): only waited for. + if (working && !session.interruptRequested) void session.interrupt(SHUTDOWN_AUTH).catch(() => {}); + } else if (Date.now() - first >= session.interruptSettleMs) { + givenUp.add(key); + console.warn( + `[codeoid/session-manager] drain: session ${session.id.slice(0, 8)} (${session.name}) didn't end its turn within ${session.interruptSettleMs} ms of the interrupt; going on without it`, + ); + continue; + } + waiting++; } - await Bun.sleep(500); + // A message waiting on a session's load is handled after it: wait for + // it too (a prompt among them is preparing, and is stopped above). + if (waiting === 0 && this.#historyGates.size === 0) break; + await Bun.sleep(Math.min(pollMs, remaining())); + pollMs = Math.min(pollMs * 2, 50); + } + await this.#settleTurnEnds(Math.min(remaining(), TURN_END_SETTLE_MS)); + await this.#flushTranscripts(deadline); + } + + /** + * Wait, at most `boundMs`, for what every session's turn end started + * (`Session.turnEndWork`) and for the routine runs to record their final + * rows, spend included (they do once their turn has ended, or once drain + * gave up on it): all at once. The store closes next. + */ + async #settleTurnEnds(boundMs: number): Promise { + const pending = new Set(); + const work = [...this.#sessions.values()].map((s) => { + pending.add(s); + return s.turnEndWork().then(() => void pending.delete(s)); + }); + let timer: ReturnType | undefined; + const [routinesEnded] = await Promise.all([ + this.#routines ? this.#routines.settled(boundMs) : Promise.resolve(true), + Promise.race([ + Promise.all(work), + new Promise((resolve) => { + timer = setTimeout(resolve, boundMs); + timer.unref?.(); + }), + ]), + ]); + clearTimeout(timer); + for (const s of pending) { + console.warn(`[codeoid/session-manager] drain: session ${s.id.slice(0, 8)} (${s.name}) didn't finish its turn's end within ${boundMs} ms (its git check, snapshot or hooks); going on without it`); + } + if (!routinesEnded) console.warn("[codeoid/session-manager] drain: a routine run didn't record its end in time"); + } + + /** + * Await the transcript store's writes until none is in flight, within the + * deadline. A settled write's handler may start another (the turn index + * written as a turn ends), so look again after each round. + */ + async #flushTranscripts(deadline: number): Promise { + for (let round = 0; round < 10 && Date.now() < deadline; round++) { + // One macrotask first: a handler a settled write woke may start the next write there. + await new Promise((resolve) => setImmediate(resolve)); + if (this.#transcriptStore.pendingWrites === 0) return; + let timer: ReturnType | undefined; + await Promise.race([ + this.#transcriptStore.flush(), + new Promise((resolve) => { + timer = setTimeout(resolve, Math.max(0, deadline - Date.now())); + timer.unref?.(); + }), + ]); + clearTimeout(timer); } } @@ -5028,26 +5264,26 @@ mcpHub: this.#mcpHub, code: "invalid_request", }; } - if (msg.afterTurnId) { - try { - point = await parent.forkPoint(msg.afterTurnId); - } catch (err) { - return { - type: "response.error", - requestId: msg.id, - error: err instanceof Error ? err.message : String(err), - code: - err instanceof RewindError - ? err.code === "not_found" - ? "not_found" - : err.code === "restore_failed" - ? "internal" - : "invalid_request" // before_rotation, busy: pick another point / wait - : "internal", - }; - } + let parentHistory: readonly CanonicalTurn[]; + try { + if (msg.afterTurnId) point = await parent.forkPoint(msg.afterTurnId); + parentHistory = point ? point.history : await parent.fullCanonicalHistory(); + } catch (err) { + return { + type: "response.error", + requestId: msg.id, + error: err instanceof Error ? err.message : String(err), + code: + err instanceof RewindError + ? err.code === "not_found" + ? "not_found" + : err.code === "restore_failed" || err.code === "history_unavailable" + ? "internal" + : "invalid_request" // before_rotation, busy: pick another point / wait + : "internal", + }; } - const history = (point ? point.history : await parent.fullCanonicalHistory()).map((t) => ({ ...t })); + const history = parentHistory.map((t) => ({ ...t })); // From an earlier turn: the parent's background launches after that turn // are its own business, not news for the fork's model (they stay foreign). // Only the parent's unfinished work matters, and it is looked for in the @@ -5666,7 +5902,9 @@ mcpHub: this.#mcpHub, // COMPLETE (or ask for input) on GENUINELY NEW final text — never on the // PRIOR phase's ⟦PHASE-COMPLETE⟧ that still sits in lastAssistantText (e.g. // behind a tools-only turn whose own text is empty). Without this, phase N - // "completes" instantly by reading phase N-1's marker. + // "completes" instantly by reading phase N-1's marker. A pipeline resumed + // after a restart drives a session whose history may not be loaded yet. + if (session.historyPending) await session.ensureHydrated(); let textAtSend = session.lastAssistantText ?? ""; // Every NEW assistant text this phase produced, in order. The phase's // summary is the whole of what the model said across its turns, not only @@ -6704,6 +6942,27 @@ mcpHub: this.#mcpHub, }; } + /** + * A history-reading message its handler refuses before it reads the + * history, for a reason other than scope (`#loopWorkerGuard`, a delegated + * approval of a loop worker's): it loads nothing. A handler that goes on + * after all loads the history itself. + */ + #refusedBeforeHistory(msg: ClientMessage, auth: AuthContext): boolean { + const id = (msg as { sessionId?: unknown }).sessionId; + if (typeof id !== "string") return false; + switch (msg.type) { + case "session.rotate": + case "session.rewind": + case "session.set_provider": + return this.#isLiveLoopWorker(id); + case "session.approve": + return auth.delegationDepth > 0 && this.#isLiveLoopWorker(id); + default: + return false; + } + } + /** The session works in an active loop run's worktree: its current worker, * or an earlier one (replaced after a restart) still holding the same tree. */ #isLiveLoopWorker(sessionId: string): boolean { @@ -7273,6 +7532,11 @@ mcpHub: this.#mcpHub, workerStatus: (sessionId: string) => this.#sessions.get(sessionId)?.status ?? null, + prepareWorkerDigest: async (task: DispatchTaskRow): Promise => { + const worker = this.#sessionForTask(task.workerSessionId, task); + if (worker?.historyPending) await worker.ensureHydrated().catch(() => {}); + }, + buildWorkerDigest: (task: DispatchTaskRow): string => { const worker = this.#sessionForTask(task.workerSessionId, task); const header = `task ${task.id.slice(0, 8)} (${task.kind}/${task.shape}) in ${task.workdir ?? task.targetSession ?? "?"}`; @@ -8214,6 +8478,8 @@ mcpHub: this.#mcpHub, async #freezeBranch(t: CompareRunRow["targets"][number], s: Session): Promise { const turnId = t.baseTurnId!; + // A branch settled after a restart may not have loaded its history yet. + if (s.historyPending) await s.ensureHydrated().catch(() => {}); const status: "idle" | "error" = s.status === "error" ? "error" : "idle"; // Nothing after the compared turn yet: the branch's totals are that turn's // (a fork starts with none). Read before awaiting — the user may go on. @@ -8460,7 +8726,7 @@ mcpHub: this.#mcpHub, type: "response.error", requestId: msg.id, error: err.message, - code: err.code === "not_found" ? "not_found" : "invalid_request", + code: err.code === "not_found" ? "not_found" : err.code === "history_unavailable" ? "internal" : "invalid_request", }; } throw err; @@ -8506,6 +8772,7 @@ mcpHub: this.#mcpHub, #send( msg: Extract, auth: AuthContext, + hold?: SendHold, ): DaemonMessage | Promise { if (!hasScope(auth.scopes as string[], SCOPES.SESSION_SEND)) { return { type: "response.error", requestId: msg.id, error: "Missing scope: session:send", code: "forbidden" }; @@ -8522,6 +8789,12 @@ mcpHub: this.#mcpHub, const loopRun = this.#liveLoopRunOf(session.id); if (loopRun) return this.#steerFromSend(msg, auth, loopRun); + // Shutting down: no turn starts. Said here, not as a line in a stream the client may not get. + // Before the duplicate check: a refused send doesn't mark its clientMsgId seen. + if (this.#draining) { + return { type: "response.error", requestId: msg.id, error: "The daemon is shutting down: the message wasn't sent", code: "internal" }; + } + // Duplicate-send suppression (`send.idempotency`): a client that // couldn't observe whether its send survived a dropped socket resends // with the SAME clientMsgId — acknowledging instead of dispatching @@ -8548,7 +8821,7 @@ mcpHub: this.#mcpHub, // returned a false "ok" and dropped the error — silent data loss). // priority controls mid-turn insertion semantics (default "later" = FIFO). session - .send(msg.text, auth, msg.attachments, msg.priority) + .send(msg.text, auth, msg.attachments, msg.priority, hold) .catch((err) => { // Stopped before its turn started: the user already got a notice. if (err instanceof SendStoppedError) return; diff --git a/src/daemon/session.ts b/src/daemon/session.ts index 17e1cf98..58baa4b1 100644 --- a/src/daemon/session.ts +++ b/src/daemon/session.ts @@ -131,7 +131,8 @@ import type { Store } from "./store.js"; import type { AgentIdentityManager } from "./agent-identity.js"; import { ScrollbackBuffer } from "./scrollback.js"; import { reconcileResumedMessage } from "./resume-reconcile.js"; -import type { TranscriptStore } from "./transcript.js"; +import type { TranscriptEntry, TranscriptFileMark, TranscriptStore } from "./transcript.js"; +import { canonicalFromTranscript, turnIndexFromHistory } from "./canonical-restore.js"; import { backgroundLaunchIds, FOREIGN_AGENT_DENY, @@ -283,6 +284,21 @@ const SUBAGENT_REGISTRATION_FENCE_MS = 5_000; const BACKGROUND_WAKE_FALLBACK_MS = 20_000; /** How long a send waits for an ended run's last events before its own turn replaces it. */ const ENDED_RUN_DRAIN_MS = 5_000; +/** + * How long a daemon shutting down waits for an interrupted turn to end, when + * its backend declares no bound of its own (`SessionProvider.interruptSettleMs`): + * Claude's result follows the interrupt receipt, Codex's turn/completed its + * reply, within well under a second. + */ +export const DRAIN_SETTLE_MS = 2_000; +/** The principal a daemon shutting down interrupts turns as. */ +export const SHUTDOWN_AUTH: AuthContext = { + sub: "system:shutdown", + scopes: [], + delegationDepth: 0, + accountId: "", + projectId: "", +}; /** How long a destroy waits for its look at the session's commits (#checkSettingsCommits) before it goes on without it. */ const DESTROY_COMMIT_CHECK_MS = 15_000; @@ -776,7 +792,7 @@ export interface ForkPoint { /** A rewind that can't proceed (unknown turn, a turn that wouldn't stop). */ export class RewindError extends Error { constructor( - readonly code: "not_found" | "busy" | "restore_failed" | "before_rotation", + readonly code: "not_found" | "busy" | "restore_failed" | "before_rotation" | "history_unavailable", message: string, ) { super(message); @@ -857,12 +873,20 @@ const TURN_INDEX_KEEP = 1000; const TURN_INDEX_SLACK = 200; export class SendStoppedError extends Error { - constructor() { - super("Stopped before the agent started on this message"); + constructor(message = "Stopped before the agent started on this message") { + super(message); this.name = "SendStoppedError"; } } +/** A send refused because the daemon is shutting down: no turn starts once drain began. */ +export class DaemonStoppingError extends SendStoppedError { + constructor() { + super("The daemon is shutting down: no new turn starts"); + this.name = "DaemonStoppingError"; + } +} + /** Claude's file tools that write. */ const CLAUDE_FILE_WRITE_TOOLS = new Set(["Write", "Edit", "MultiEdit", "NotebookEdit"]); @@ -887,6 +911,90 @@ export function writesClaudeSettingsFile(target: string, workdir: string): boole * counts as spending. */ /** What one turn's event consumer counted (#430): model work, requests sent, usage reports received. */ +/** + * How a resumed session loads its history (`Session.prepareResume`): the + * resume read budgets, which SessionManager owns, and what its meta says. + */ +export interface ResumeHistoryPlan { + /** Newest transcript bytes read into the scrollback. */ + transcriptMaxBytes: number; + /** Newest canonical-log bytes read into the history. */ + canonicalMaxBytes: number; + /** Time a load may spend parsing the transcript before it keeps what it has. */ + deadlineMs: number; + /** The backend a history rebuilt from the transcript is for (a session from before the log). */ + providerId: string; + /** The meta's `backingFresh`: the backend is re-seeded from the history once it is loaded. */ + backingFresh: boolean; +} + +/** + * A resumed session whose history isn't loaded yet (lazy session loading): + * what it needs meanwhile, and what the load must add. + */ +interface Dehydrated { + plan: ResumeHistoryPlan; + /** + * The log's files as they were at resume: the load reads them up to there, + * exactly what an eager resume read, whatever this process appended since + * and whatever their seqs. + */ + files: TranscriptFileMark[]; + /** Messages appended since (on disk too), pushed onto the scrollback after the load, as if live. */ + appended: SessionMessage[]; + /** The session had a turn index on disk (else the load rebuilds it from the history). */ + hadIndex: boolean; + /** The turn index or canonical log couldn't be read at resume: the load reads the index again, and fails as a resume did. */ + indexUnread?: boolean; + /** The newest canonical turn, for `lastTurnId` / `lastTurnRole`. */ + lastTurn: Pick | null; + /** A teardown owed the backend a re-seed from the history (`#reseedAfterTeardown`): paid on load. */ + reseed: boolean; + /** The load in flight. */ + loading?: Promise; +} + +/** + * The canonical log (or turn index) couldn't be read when a resume loaded + * the history (`Session#historyUnread`): what the session holds in memory is + * only what came since. + */ +interface HistoryUnread { + plan: ResumeHistoryPlan; + /** The turn index wasn't read either: it is read again with the log. */ + indexUnread: boolean; + /** The last read's error, logged once per distinct one. */ + error: string; + /** The read in flight. */ + retry?: Promise; +} + +/** Why going back or forking is refused while the log couldn't be read. */ +const HISTORY_UNREAD_REFUSAL = + "This session's earlier conversation couldn't be read from disk, so going back or forking now would lose it. Nothing was changed — try again, or see the daemon's log."; + +/** A send counted as preparing before it reaches `Session.send` (`Session.holdSend`). */ +export interface SendHold { + /** The Stop generation when it arrived: a Stop since stops it. */ + readonly stopGen: number; + /** Take the count over (true once; false after `take` or `release`). */ + take(): boolean; + /** Give the count back, unless taken. */ + release(): void; +} + +/** What a history load read, for `Session#installHistory`. */ +interface LoadedHistory { + entries: TranscriptEntry[]; + truncated: boolean; + canonical: { turns: CanonicalTurn[]; partial: boolean } | null; + /** The whole transcript, when there's no log to read and the scrollback read was a tail. */ + rebuildFrom?: DaemonMessage[]; + /** The turn index, read by the load (`Dehydrated.indexUnread`). */ + index?: TurnIndexEntry[] | null; + canonicalError?: unknown; +} + interface TurnCounts { activity: boolean; requests: number; @@ -1050,7 +1158,8 @@ export class Session { #transcriptStore: TranscriptStore; #identityManager?: AgentIdentityManager; #agentIdentity: MessageIdentity; - #scrollback = new ScrollbackBuffer(); + /** The replay buffer. Read it as `#scrollback`, which loads a resumed session's history first. */ + #scrollbackBuffer = new ScrollbackBuffer(); /** * Identity of this Session instance's replay buffer (`replay.resume`). * A client cursor (`sinceSeq`) is only valid against the buffer that @@ -1106,7 +1215,8 @@ export class Session { // this to detect a turn whose event stream has gone silent (hung tool / dead // subprocess) so the session can self-recover instead of wedging forever. #lastEventAt = 0; - #accumulator = new CanonicalHistoryAccumulator(); + /** The canonical history. Read it as `#accumulator`, which loads a resumed session's history first. */ + #historyAccumulator = new CanonicalHistoryAccumulator(); /** * The turn messages are currently attributed to (#354): minted when a * prompt (or a background continuation) starts a turn, stamped on every @@ -1115,6 +1225,8 @@ export class Session { #currentTurnId: string | null = null; /** In-flight workspace snapshot per turn, so destroy can wait for them. */ #pendingCheckpoints = new Set>(); + /** Work a turn's end started and doesn't await (the #448 git check, `after_turn` hooks): see turnEndWork. */ + readonly #turnEndJobs = new Set>(); /** The session's turns, oldest first (#354) — see #indexTurn. */ #turnIndex: TurnIndexEntry[] = []; /** @@ -1122,7 +1234,7 @@ export class Session { * bounded resume read). Features that need the whole conversation (going * back to, or forking from, an old turn) load the full log on demand. */ - #canonicalPartial = false; + #canonicalPartialFlag = false; /** Pluggable seed policy for switch/fork. Default `transcript` (no change); * `CODEOID_CONTEXT_STRATEGY=vws` opts into the compact session map. */ #contextStrategy: ContextStrategy = selectContextStrategy(); @@ -1236,8 +1348,8 @@ export class Session { } | null = null; /** Set by a turn_done whose seed overflowed; the consumer's exit retries it. */ #seedOverflowRetry = false; - /** The last turn failed with "prompt too long" — /rotate then skips its min-turns guard. */ - #promptTooLong = false; + /** The last turn failed with "prompt too long" — /rotate then skips its min-turns guard. Derived from the scrollback on resume. */ + #promptTooLongFlag = false; // Execution mode + turn budget (autonomous mode only). // Default `guarded` (≈ Claude Code's default): read-only tools (Read/Grep/Glob // + memory) auto-approve, while Write/Edit/Bash and other mutations prompt. @@ -1658,8 +1770,14 @@ export class Session { // Durable canonical history (#354): every change is appended to the // session's log so forks, backend switches and rewinds after a restart // still carry the conversation. - this.#accumulator.onChange = (change) => { + this.#historyAccumulator.onChange = (change) => { if (this.#destroyed) return; + // Never write a history that couldn't be read over the log it lost; + // what is appended still is (`#historyUnread`). + if (change.op === "replace" && this.#historyUnread) { + console.error(`[codeoid/session ${this.id}] canonical history not rewritten: the log couldn't be read, and would lose its turns`); + return; + } void this.#transcriptStore.recordCanonical(this.id, change); }; this.#identityManager = opts.identityManager; @@ -2124,9 +2242,8 @@ export class Session { // A backend that can't resume its own conversation (the Gemini CLI // starts a new ACP session) is handed the history, as after an MCP // rebuild — an account switch must never start the conversation over. - if (RESEED_AFTER_TEARDOWN.has(this.#provider.id) && this.#accumulator.history.length > 0) { - await this.#seedProviderFromHistory(); - } + // Deferred to the load for a session whose history isn't loaded yet. + await this.#reseedAfterTeardown(); } /** @@ -2431,6 +2548,7 @@ export class Session { // Rejecting here means we never tear down an actively-running provider. const busy = this.#switchBusyReason(); if (busy) return busy; + if (this.#loadPending) await this.ensureHydrated(); const previous = this.#provider.id; @@ -2574,6 +2692,8 @@ export class Session { } /** turnId of the newest turn in the canonical history (null before any, or when unstamped). */ get lastTurnId(): string | null { + // Before the history loads: the newest turn of its log. + if (this.#dehydrated) return this.#dehydrated.lastTurn?.turnId ?? null; return this.#accumulator.history.at(-1)?.turnId ?? null; } /** Number of turns (user + assistant) in the canonical history. */ @@ -2609,6 +2729,7 @@ export class Session { * length watermark was not). */ get lastTurnRole(): "user" | "assistant" | undefined { + if (this.#dehydrated) return this.#dehydrated.lastTurn?.role; const h = this.#accumulator.history; return h[h.length - 1]?.role; } @@ -2936,9 +3057,7 @@ export class Session { } if (view.fingerprint === this.#mcpBuiltWith) return false; await this.#teardownProvider(); // records what's applied, and lets go of what was removed - if (RESEED_AFTER_TEARDOWN.has(this.#provider.id) && this.#accumulator.history.length > 0) { - await this.#seedProviderFromHistory(); - } + await this.#reseedAfterTeardown(); return true; } @@ -4110,6 +4229,7 @@ export class Session { maxBytes?: number, ): Promise<{ messages: SessionMessage[]; hasMore: boolean; source: "buffer" | "transcript" }> { const budget = Math.min(Math.max(1, maxBytes ?? PAGE_DEFAULT_BYTES), PAGE_MAX_BYTES); + if (this.#loadPending) await this.ensureHydrated(); if (beforeMessageId === undefined) { const tail = this.#scrollback.readTailChunked(budget, budget); @@ -4363,6 +4483,7 @@ export class Session { data: Record | undefined, sender: AuthContext, ): Promise<{ ok: true } | { ok: false; code: "not_found" | "invalid_request"; error: string }> { + if (this.#loadPending) await this.ensureHydrated(); const msg = this.#scrollback.getMessage(messageId) as SessionMessage | undefined; if (!msg) { return { ok: false, code: "not_found", error: "Message not found" }; @@ -4436,7 +4557,11 @@ export class Session { sender: AuthContext, attachments?: readonly Attachment[], priority?: "now" | "next" | "later", + /** Counted as preparing since it arrived (`holdSend`): this send takes it over. */ + hold?: SendHold, ): Promise { + // Shutting down: nothing is recorded, nothing starts. + if (this.#draining) throw new DaemonStoppingError(); // Before anything is recorded: a session whose Claude account was removed // must not quietly run on some other credential. this.assertAccountUsable(); @@ -4447,11 +4572,17 @@ export class Session { // A send is "preparing" from the moment it is queued until its turn has // started (or it failed): drain() must treat that as work, and a Stop in // that window must cancel it — see #stoppedBeforeStart. - const stopGen = this.#stopGen; - this.#preparingSends++; + const held = hold?.take() === true; + const stopGen = held ? hold!.stopGen : this.#stopGen; + if (!held) this.#preparingSends++; const next = this.#sendChain .catch(() => undefined) - .then(() => this.#sendInner(text, sender, attachments, priority, stopGen)) + // A resumed session loads its history before its first turn. + .then(() => + this.#dehydrated || this.#historyUnread + ? this.ensureHydrated().then(() => this.#sendInner(text, sender, attachments, priority, stopGen)) + : this.#sendInner(text, sender, attachments, priority, stopGen), + ) .finally(() => { this.#preparingSends--; }); @@ -4462,6 +4593,30 @@ export class Session { return next; } + /** + * Count a send as preparing before it reaches send(): one the daemon holds + * while the session's history loads (`SessionManager.handle`). The drain + * sees it, and a Stop meanwhile stops it as it stops a queued send. Hand + * it to send(), which takes it over, or release() it. + */ + holdSend(): SendHold { + this.#preparingSends++; + let held = true; + return { + stopGen: this.#stopGen, + take: () => { + const was = held; + held = false; + return was; + }, + release: () => { + if (!held) return; + held = false; + this.#preparingSends--; + }, + }; + } + /** Sends queued or preparing their turn (see send()). */ #preparingSends = 0; /** Bumped by every Stop, so a send queued before it knows it was stopped. */ @@ -4474,6 +4629,62 @@ export class Session { return this.#preparingSends > 0; } + /** Set by the daemon's drain: no turn starts from here on. */ + #draining = false; + + /** + * The daemon is shutting down: a send is refused from here on, a turn the + * backend starts on its own is interrupted as it starts, and no wake is + * delivered. Turns already running are drain's to interrupt and wait for. + */ + beginDrain(): void { + this.#draining = true; + if (this.#backgroundWakeFallback) { + clearTimeout(this.#backgroundWakeFallback); + this.#backgroundWakeFallback = null; + } + } + + /** What drain tells one turn from the next by: the running turn's run, else the session (a send still preparing). */ + get drainKey(): object { + return this.#activeRun ?? this; + } + + /** The running turn was already asked to stop (by its owner, a routine, or as it started during a drain). */ + get interruptRequested(): boolean { + return this.#activeRun !== null && this.#turnInterrupted; + } + + /** Hold `job`, started by a turn's end and not awaited, until it settles: turnEndWork waits for it. */ + #keepTurnEnd(job: Promise): void { + this.#turnEndJobs.add(job); + const done = () => this.#turnEndJobs.delete(job); + void job.then(done, done); + } + + /** + * Settles once the work this session's turn ends started and don't await + * has: the #448 git and settings-commit check (its audit row and warning + * line), the end (and any start) snapshot, `after_turn` hooks. Drain + * awaits it, bounded, before the transcripts flush and the store closes, + * so a shutdown cuts none of it off. A job that starts another (a + * snapshot chained behind one) is waited for too. Never rejects. + */ + async turnEndWork(): Promise { + const waited = new Set>(); + for (;;) { + const jobs = [...this.#turnEndJobs, ...this.#pendingCheckpoints].filter((j) => !waited.has(j)); + if (jobs.length === 0) return; + for (const j of jobs) waited.add(j); + await Promise.allSettled(jobs); + } + } + + /** How long drain waits for this session's interrupted turn to end before it gives up on it. */ + get interruptSettleMs(): number { + return this.#provider.interruptSettleMs ?? DRAIN_SETTLE_MS; + } + /** Per-turn loop controls, reset when a turn starts. */ #resetTurnControls(): void { this.#endAttemptInterruptQueued = false; @@ -4883,6 +5094,8 @@ export class Session { this.#provider.onRecoveryNeeded = (content: string) => { const recoverySender = this.#currentSender ?? sender; this.#sendChain = this.#sendChain.then(async () => { + // The daemon is draining: no turn starts (the prompt is already in the transcript). + if (this.#draining) return; // Reset to a fresh backing session for recovery. const newBackingId = randomUUID(); this.#provider.resetToNewSession(newBackingId); @@ -5602,6 +5815,8 @@ export class Session { this.#setStatus("thinking"); this.#onTurnStarted(); this.#broadcastInfoUpdate(); + // Shutting down: consumed, so its end and spend are recorded, but stopped at once. + if (this.#draining) void this.interrupt(SHUTDOWN_AUTH).catch(() => {}); } /** @@ -5661,7 +5876,7 @@ export class Session { */ #armBackgroundWakeFallback(): void { if (this.#backgroundWakeFallback) return; - if (this.#destroyed) return; + if (this.#destroyed || this.#draining) return; this.#backgroundWakeFallback = setTimeout(() => { this.#backgroundWakeFallback = null; this.#maybeDeliverBackgroundReports({ fallback: true }); @@ -5746,6 +5961,8 @@ export class Session { // interrupt) would inject a duplicate turn beside the backend's own. if (this.#provider.continuesAfterBackgroundWork && !opts.fallback) return; if (this.#destroyed) return; + // Shutting down: no turn starts (its refused send would requeue and retry at once). + if (this.#draining) return; // Mid-rewind: these reports belong to turns being taken back (#355). if (this.#rewinding) return; if (this.#deliveringBackgroundReports) return; @@ -6414,7 +6631,8 @@ export class Session { .catch(() => undefined) .then(async () => { // A send, switch or fork re-seed got here first — it owns the session now. - if (this.#destroyed || this.#pendingSeed !== seed || this.#activeRun) return; + // Nor once the daemon is draining: no turn starts. + if (this.#destroyed || this.#draining || this.#pendingSeed !== seed || this.#activeRun) return; await this.#teardownProvider(); const newBackingId = randomUUID(); this.#provider.resetToNewSession(newBackingId); @@ -6954,11 +7172,383 @@ export class Session { }; } + // ── Lazy session loading ────────────────────────────────────────────── + // + // A resumed session starts with its metadata only (`prepareResume`): its + // scrollback and canonical history stay on disk until something uses them, + // and load exactly as a resume used to load them. Entry points await + // `ensureHydrated()`; the synchronous internals read the two through + // `#scrollback` / `#accumulator`, which load them first if nothing has + // (`#hydrateNow`, logged: a path that should have awaited). + + /** Set while the history isn't loaded; null once it is (and for every session not resumed). */ + #dehydrated: Dehydrated | null = null; + /** Work the load leaves the backend (a re-seed), awaited by `ensureHydrated()`. */ + #afterHydrate: Promise | null = null; + /** + * The canonical log couldn't be read when a resume loaded the history: the + * session runs without it, as a resume always did, but the history in + * memory is only what came since. It is never written over the log (going + * back or forking is refused, `#assertHistoryRead`), turns are still + * appended to it, and every use reads it again (`#rereadHistory`) until it + * can. Null otherwise. + */ + #historyUnread: HistoryUnread | null = null; + /** Loads `#hydrateNow` had to do synchronously, daemon-wide (tests read it). */ + static syncHydrations = 0; + + get #scrollback(): ScrollbackBuffer { + if (this.#dehydrated) this.#hydrateNow("scrollback"); + return this.#scrollbackBuffer; + } + + get #accumulator(): CanonicalHistoryAccumulator { + if (this.#dehydrated) this.#hydrateNow("history"); + return this.#historyAccumulator; + } + + get #canonicalPartial(): boolean { + if (this.#dehydrated) this.#hydrateNow("history"); + return this.#canonicalPartialFlag; + } + + set #canonicalPartial(v: boolean) { + if (this.#dehydrated) this.#hydrateNow("history"); + this.#canonicalPartialFlag = v; + } + + get #promptTooLong(): boolean { + if (this.#dehydrated) this.#hydrateNow("scrollback"); + return this.#promptTooLongFlag; + } + + set #promptTooLong(v: boolean) { + if (this.#dehydrated) this.#hydrateNow("scrollback"); + this.#promptTooLongFlag = v; + } + + /** The history isn't in place yet (or couldn't be read), or a load's re-seed is still running: await `ensureHydrated()`. */ + get #loadPending(): boolean { + return this.#dehydrated !== null || this.#afterHydrate !== null || this.#historyUnread !== null; + } + + /** `ensureHydrated()` has work to do (a load, or a load's re-seed still running). */ + get historyPending(): boolean { + return this.#loadPending; + } + + /** Whether the scrollback and canonical history are in memory. */ + get hydrated(): boolean { + return this.#dehydrated === null; + } + + /** + * Resume this session from disk without loading its history: only what it + * reports and appends with before the load — the seq its log continues + * from, the newest canonical turn, the turn index (small) — is read, from + * the files' ends. `#seq` continues past the log, the backend is told it + * has a conversation to resume, and a fresh backing's flags are set, as a + * full resume sets them. Returns false when the history must be loaded now: + * a session from before the canonical log or the turn index (the load + * rebuilds and persists them), or one whose log or index couldn't be read + * (the load fails as a resume did: the history is resumed without). + * Rejects when the transcript is there but can't be read: the session + * can't resume, as when a resume couldn't load it — never as one with no + * history, whose seq starts over and whose backend starts a new conversation. + */ + async prepareResume(plan: ResumeHistoryPlan): Promise { + const tail = await this.#transcriptStore.transcriptTail(this.id); + let canonical: Awaited> = null; + let index: TurnIndexEntry[] | null = null; + let indexUnread = false; + try { + [canonical, index] = await Promise.all([ + this.#transcriptStore.canonicalTail(this.id), + this.#transcriptStore.loadTurnIndex(this.id), + ]); + } catch { + indexUnread = true; + } + if (tail.nextSeq > this.#seq) this.#seq = tail.nextSeq; + // A session with prior scrollback already exists in its backend's own + // store — the next send() must resume it, not re-create it. + if (tail.hasEntries) this.#provider.setHasQueried(true); + this.#dehydrated = { + plan, + files: tail.files, + appended: [], + hadIndex: index !== null, + ...(indexUnread ? { indexUnread } : {}), + lastTurn: canonical?.last ?? null, + reseed: false, + }; + if (index) { + this.#turnIndex = index; + this.#currentTurnId = index.at(-1)?.turnId ?? null; + this.#endSnapshotTurn = this.#currentTurnId; + } + if (plan.backingFresh) { + // reseedFreshBacking's flags; its seed waits for the history. + this.#backingFresh = true; + this.#provider.setHasQueried(false); + } + // An empty index over a non-empty history names its current turn from the history. + return !indexUnread && canonical !== null && index !== null && (index.length > 0 || canonical.last === null); + } + + /** + * Load the history of a session resumed by `prepareResume`, if it isn't + * loaded: idempotent, and concurrent callers share one load. Rejects when + * the transcript, canonical log or turn index can't be read (the session + * stays unloaded; the next call tries again). At resume (`atResume`) a + * canonical log that can't be read is logged and the session goes on + * without it, as a resume always did (`#historyUnread`); each later call + * reads it again. Resolves once a fresh backing has been re-seeded too. + */ + ensureHydrated(opts: { deadlineAt?: number; atResume?: boolean } = {}): Promise { + const d = this.#dehydrated; + if (!d && this.#historyUnread) return this.#rereadHistory().then(() => this.#afterHydrate ?? undefined); + if (!d) return this.#afterHydrate ?? Promise.resolve(); + if (!d.loading) { + const loading = this.#hydrate(d, opts.deadlineAt, opts.atResume === true).then(() => this.#afterHydrate ?? undefined); + d.loading = loading; + loading.catch(() => { + if (d.loading === loading) d.loading = undefined; + }); + } + return d.loading; + } + + async #hydrate(d: Dehydrated, deadlineAt = Date.now() + d.plan.deadlineMs, atResume = false): Promise { + const started = performance.now(); + // The window an eager resume read: the log as it was at the restart. + // What was appended since is left out (it is in memory). + const stats: { truncated?: boolean } = {}; + const window = { maxBytes: d.plan.transcriptMaxBytes, deadlineAt, stats, asOf: d.files }; + const entries = await this.#transcriptStore.loadTranscript(this.id, window); + const loaded: LoadedHistory = { entries, truncated: stats.truncated === true, canonical: null }; + try { + loaded.canonical = await this.#transcriptStore.loadCanonical(this.id, { maxBytes: d.plan.canonicalMaxBytes }); + if (d.indexUnread) loaded.index = await this.#transcriptStore.loadTurnIndex(this.id); + // No log: a session from before it existed. Rebuild once from the + // WHOLE transcript (the scrollback window may be a tail) and persist it + // — a long session's history must not shrink to the window. + if (!loaded.canonical && loaded.truncated) { + const all = await this.#transcriptStore.loadTranscript(this.id, { deadlineAt, asOf: d.files }); + loaded.rebuildFrom = all.map((e) => e.message); + } + } catch (err) { + // After the resume, the session is used: it must not go on without its + // conversation. The request fails, and the next use loads it again. + if (!atResume) throw err; + loaded.canonicalError = err; + } + // Loaded synchronously meanwhile (#hydrateNow): that load stands. + if (this.#dehydrated !== d) return; + this.#installHistory(d, loaded, atResume ? null : started); + } + + /** + * Load the history now, synchronously: a synchronous path needs it and + * nothing awaited `ensureHydrated()` first. Correct, but it blocks the + * daemon for the read, so it is logged with where it came from. + */ + #hydrateNow(what: string): void { + const d = this.#dehydrated; + if (!d) return; + const started = performance.now(); + Session.syncHydrations++; + const where = new Error().stack?.split("\n").slice(3, 6).map((l) => l.trim()).join(" < ") ?? ""; + console.warn(`[codeoid/session ${this.id}] ${what} needed before it was loaded — loading it synchronously (${where})`); + const stats: { truncated?: boolean } = {}; + const deadlineAt = Date.now() + d.plan.deadlineMs; + const window = { maxBytes: d.plan.transcriptMaxBytes, deadlineAt, stats, asOf: d.files }; + // Any read error throws, the session left unloaded, as in `#hydrate`. + const entries = this.#transcriptStore.loadTranscriptSync(this.id, window); + const loaded: LoadedHistory = { entries, truncated: stats.truncated === true, canonical: null }; + loaded.canonical = this.#transcriptStore.loadCanonicalSync(this.id, { maxBytes: d.plan.canonicalMaxBytes }); + if (d.indexUnread) loaded.index = this.#transcriptStore.loadTurnIndexSync(this.id); + if (!loaded.canonical && loaded.truncated) { + loaded.rebuildFrom = this.#transcriptStore + .loadTranscriptSync(this.id, { deadlineAt, asOf: d.files }) + .map((e) => e.message); + } + this.#installHistory(d, loaded, started); + } + + /** Put a load's history in place, as a resume did: one synchronous step. */ + #installHistory(d: Dehydrated, loaded: LoadedHistory, started: number | null): void { + this.#dehydrated = null; + const { entries } = loaded; + // Restore scrollback from transcript. Byte-budgeted + deadline-aware: one + // huge transcript can neither OOM the daemon nor stall the load. The seq + // counter continues past the log's newest line (prepareResume), and now + // past its highest seq too: a log whose seqs went back down (an older + // daemon restarted them, an import) holds it further up. + const messages = entries.map((e) => e.message); + const maxSeq = entries.reduce((max, e) => Math.max(max, e.seq), -1); + this.restoreScrollback(messages, maxSeq + 1, entries.map((e) => e.bytes), { + partialHistory: loaded.truncated, + markQueried: false, + }); + // The backend-neutral conversation (#354). Without it a restored session + // forked / switched backend with no history, and stateless backends + // answered the next prompt with no memory of the session. + let canonicalError = loaded.canonicalError; + if (canonicalError === undefined) { + try { + const rebuildFrom = loaded.rebuildFrom ?? messages; + const history = + loaded.canonical?.turns ?? + (rebuildFrom.length > 0 ? canonicalFromTranscript(rebuildFrom, d.plan.providerId) : []); + // The index read at resume, or by the load when it couldn't be then. + const index = loaded.index !== undefined ? loaded.index : d.hadIndex ? this.#turnIndex : null; + this.restoreTurns(history, index ?? turnIndexFromHistory(history), { + persistHistory: !loaded.canonical && history.length > 0, + persistIndex: !index && history.some((t) => t.turnId), + partial: loaded.canonical?.partial ?? false, + }); + } catch (err) { + canonicalError = err; + } + } + if (canonicalError !== undefined) { + const error = canonicalError instanceof Error ? canonicalError.message : String(canonicalError); + this.#historyUnread = { plan: d.plan, indexUnread: d.indexUnread === true, error }; + console.error( + `[codeoid/resume] canonical history restore failed for ${this.id} (session resumes without it, and reads it again on its next use): ${error}`, + ); + } + // What this process appended before the load, in order, as it was pushed live. + for (const msg of d.appended) this.#scrollbackBuffer.push(msg); + const reseedFresh = d.plan.backingFresh && canonicalError === undefined; + if (reseedFresh || d.reseed) { + const after: Promise = (async () => { + if (reseedFresh) await this.reseedFreshBacking(); + if (d.reseed) await this.#reseedAfterTeardown(); + })() + .catch((err) => { + console.error( + `[codeoid/resume] canonical history restore failed for ${this.id} (session resumes without it): ${err instanceof Error ? err.message : String(err)}`, + ); + }) + .finally(() => { + if (this.#afterHydrate === after) this.#afterHydrate = null; + }); + this.#afterHydrate = after; + } + if (started === null) return; // part of the resume: not worth a line per session + const ms = Math.round(performance.now() - started); + console.log( + `[codeoid/session ${this.id.slice(0, 8)}] history loaded: ${this.#scrollbackBuffer.length} message(s), ${this.#historyAccumulator.history.length} turn(s) in ${ms}ms`, + ); + } + + /** + * Read the canonical log again, for a session that couldn't at resume + * (`#historyUnread`): never rejects, the session goes on without it until + * a read succeeds. Not while a turn runs (its reply is being built in the + * history), and a read that something changed the history or the turn + * list under is dropped: the next use reads again. + */ + #rereadHistory(): Promise { + const u = this.#historyUnread; + if (!u || this.#turnActive() || this.#destroyed) return Promise.resolve(); + u.retry ??= this.#reread(u).finally(() => { + u.retry = undefined; + }); + return u.retry; + } + + async #reread(u: HistoryUnread): Promise { + const history = this.#historyAccumulator.history; + const indexed = this.#turnIndex; + let canonical: Awaited>; + let index: TurnIndexEntry[] | null = null; + try { + canonical = await this.#transcriptStore.loadCanonical(this.id, { maxBytes: u.plan.canonicalMaxBytes }); + if (u.indexUnread) index = await this.#transcriptStore.loadTurnIndex(this.id); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + if (error !== u.error) { + u.error = error; + console.error(`[codeoid/session ${this.id}] canonical history still can't be read (the session goes on without it): ${error}`); + } + return; + } + if ( + this.#historyUnread !== u || + this.#destroyed || + this.#turnActive() || + this.#historyAccumulator.history !== history || + this.#historyAccumulator.history.length !== history.length || + this.#turnIndex !== indexed || + this.#turnIndex.length !== indexed.length + ) { + return; + } + // What this process appended since is in the log too. No log: nothing was + // written since, and nothing on disk to keep. + const turns = canonical?.turns ?? history; + try { + const turnIndex = u.indexUnread ? (index ?? turnIndexFromHistory(turns)) : indexed; + this.restoreTurns(turns, turnIndex, { + persistHistory: false, + persistIndex: u.indexUnread && !index && turns.some((t) => t.turnId), + partial: canonical?.partial ?? false, + }); + } catch (err) { + console.error(`[codeoid/session ${this.id}] canonical history read again but not restored: ${err instanceof Error ? err.message : String(err)}`); + return; + } + this.#historyUnread = null; + console.log(`[codeoid/session ${this.id.slice(0, 8)}] canonical history read again: ${turns.length} turn(s)`); + // A fresh backing no turn has used yet is owed the history it missed. + if (u.plan.backingFresh && this.#backingFresh && turns.length > 0) { + await this.reseedFreshBacking().catch((err) => { + console.error( + `[codeoid/session ${this.id}] re-seeding the backend from the history read again failed: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + } + } + + /** Refuse what would rewrite the canonical log from a history that couldn't be read (`#historyUnread`). */ + #assertHistoryRead(): void { + if (this.#historyUnread) throw new RewindError("history_unavailable", HISTORY_UNREAD_REFUSAL); + } + + /** + * A backend that can't resume its own conversation (the Gemini CLI starts + * a new ACP session) is handed the history after a teardown. A session + * whose history isn't loaded owes it until the load: nothing reaches the + * backend before then. + */ + async #reseedAfterTeardown(): Promise { + if (!RESEED_AFTER_TEARDOWN.has(this.#provider.id)) return; + if (this.#dehydrated) { + this.#dehydrated.reseed = true; + return; + } + if (this.#accumulator.history.length > 0) await this.#seedProviderFromHistory(); + } + + /** + * Push a message onto the scrollback — or, history not loaded yet, keep it + * for the load, which pushes it after what it restores, as if it had been + * pushed live (the load reads the log as it was before it was appended). + */ + #buffer(msg: SessionMessage): void { + if (this.#dehydrated) this.#dehydrated.appended.push(msg); + else this.#scrollbackBuffer.push(msg); + } + restoreScrollback( messages: DaemonMessage[], nextSeq?: number, sizeHints?: ReadonlyArray, - opts?: { partialHistory?: boolean }, + /** `markQueried: false`: the caller has told the backend already (`prepareResume`). */ + opts?: { partialHistory?: boolean; markQueried?: boolean }, ): void { // Restart-restore with a byte-budgeted transcript window: the buffer // holds only the tail, so history paging must know disk has more. @@ -6993,7 +7583,7 @@ export class Session { } // A session with prior scrollback already exists in Claude Code's own // persistent session store — next send() must use `resume`, not re-create. - if (messages.length > 0) { + if (messages.length > 0 && opts?.markQueried !== false) { this.#provider.setHasQueried(true); } } @@ -7324,9 +7914,10 @@ export class Session { get turnInFlight(): boolean { // A continuation still to come (background results to deliver, a wake // armed) is a turn on its way: its spend belongs with this one's (#430). + // Not once the daemon is draining: none is delivered then. + if (this.#draining) return this.turnUnsettled; return ( - this.#activeRun !== null || - this.#drainingConsumers > 0 || + this.turnUnsettled || this.#backgroundWakeFallback !== null || // Reports that can still be delivered (a wake runs at rest); after an // error they wait for the next turn, which is no turn on its way. @@ -7334,6 +7925,11 @@ export class Session { ); } + /** A turn is running, or its consumer hasn't recorded its end (spend included) yet. */ + get turnUnsettled(): boolean { + return this.#activeRun !== null || this.#drainingConsumers > 0; + } + /** Every live background task, a shell or a watcher included (#430). */ get liveBackgroundTasks(): number { return this.#backgroundTasks.size; @@ -7922,9 +8518,7 @@ export class Session { // Switched since to a backend that takes it per turn: nothing to rebuild. if (this.#provider.effortChange !== "restart") return; await this.#teardownProvider(); - if (RESEED_AFTER_TEARDOWN.has(this.#provider.id) && this.#accumulator.history.length > 0) { - await this.#seedProviderFromHistory(); - } + await this.#reseedAfterTeardown(); } /** @@ -7968,6 +8562,7 @@ export class Session { * then rotating is the only way forward, however few turns there are. */ async manualRotate(sender: AuthContext): Promise { + if (this.#loadPending) await this.ensureHydrated(); const stuck = this.#promptTooLong || this.#status === "error"; if (!stuck && this.#usage.numTurns < (this.#config?.autoRotate.minTurnsBeforeRotate ?? 3)) { const infoMsg = this.#makeMessage( @@ -8048,6 +8643,9 @@ export class Session { // CRITICAL: reset the provider so it creates a fresh session rather // than trying to resume an id that has no persisted history. this.#provider.resetToNewSession(newBackingId); + // A rotation empties the log whatever it held: one that couldn't be read + // (`#historyUnread`) loses nothing by it, and is now what memory holds. + this.#historyUnread = null; this.#accumulator.reset(); // Re-snapshot the frozen workspace index for the fresh context (the rotated // session gets an up-to-date orientation; within the context it stays stable @@ -9556,6 +10154,13 @@ export class Session { this.#commitStreamed(this.#activeAssistantMsg); this.#broadcastRaw(this.#activeAssistantMsg); this.#activeAssistantMsg = null; + } else if (event.content && this.#turnInterrupted) { + // A stopped turn's last reply, shown whole: animating it held the + // turn's end, and so a shutdown's drain, for up to half a second. + const msg = this.#makeMessage("assistant", event.content, this.#agentIdentity, []); + msg.parts = [{ kind: "text", text: event.content, markdown: true }]; + this.#commitStreamed(msg); + this.#broadcastRaw(msg); } else if (event.content) { // No preceding text_delta — the SDK returned the response as a batch // (happens for mid-turn now-priority continuations). Simulate streaming @@ -9941,10 +10546,11 @@ export class Session { // finally reconciles sub-agents alongside tools. Sweeping here too would // just be a redundant pass a few statements earlier. // Hook seam: observe-only (git-checkpoint per turn, usage export). - this.#hookBus?.emit("after_turn", this.#hookContext(), { + const hooks = this.#hookBus?.emit("after_turn", this.#hookContext(), { result: event.result, }); - void this.#checkGitControl(SYSTEM_IDENTITY.sub); + if (hooks) this.#keepTurnEnd(hooks); + this.#keepTurnEnd(this.#checkGitControl(SYSTEM_IDENTITY.sub)); this.#tidyWorktreeFolders(); if (event.result.isError) { const errText = event.result.errorMessage ?? "Turn ended with an error"; @@ -10361,7 +10967,8 @@ export class Session { * transcript. */ #throwIfStoppedBeforeStart(stopGen: number): void { - if (stopGen === this.#stopGen && !this.#destroyed) return; + // A send queued before drain began doesn't start its turn after it. + if (stopGen === this.#stopGen && !this.#destroyed && !this.#draining) return; if (!this.#destroyed) { const note = this.#makeMessage( "info", @@ -10560,6 +11167,8 @@ export class Session { /** The whole canonical history: from the log when memory holds only its tail. */ async fullCanonicalHistory(): Promise { + if (this.#loadPending) await this.ensureHydrated(); + this.#assertHistoryRead(); if (!this.#canonicalPartial) return this.#accumulator.history; const full = await this.#transcriptStore.loadCanonical(this.id); return full && full.turns.length >= this.#accumulator.history.length ? full.turns : this.#accumulator.history; @@ -10610,6 +11219,8 @@ export class Session { * all of it comes from codeoid's own records. */ async forkPoint(turnId: string): Promise { + if (this.#loadPending) await this.ensureHydrated(); + this.#assertHistoryRead(); // One copy of the turn list: a rewind during the awaits below must not // make the turns and the conversation disagree. const index = this.#turnIndex.map((e) => ({ ...e })); @@ -10831,6 +11442,8 @@ export class Session { /** What going back to before `turnId` would remove, restore and not undo. */ async #planRewind(turnId: string, restoreFiles: boolean): Promise { + if (this.#loadPending) await this.ensureHydrated(); + this.#assertHistoryRead(); const pos = this.#turnIndex.findIndex((e) => e.turnId === turnId); if (pos === -1) throw new RewindError("not_found", "That turn isn't in this session (or is older than its turn list)."); const removedIds = this.#turnIndex.slice(pos).map((e) => e.turnId); @@ -10979,6 +11592,9 @@ export class Session { if (this.rewindNeedsStop && opts.canStop !== true) { throw new RewindError("busy", "The session is busy; going back would stop it, which needs session:interrupt."); } + if (this.#loadPending) await this.ensureHydrated(); + // Before anything is stopped: the log couldn't be read, so nothing changes. + this.#assertHistoryRead(); this.#rewinding = true; try { return await this.#rewindStopped(turnId, opts, sender); @@ -11253,7 +11869,9 @@ export class Session { /** Persist to transcript + scrollback buffer + memory chunker */ #persistAndBuffer(msg: SessionMessage): void { - this.#scrollback.push(msg); + // A notice for a session whose history isn't loaded (a mode change, a + // rename) doesn't load it: it is kept for the load. + this.#buffer(msg); // Log instead of swallowing — a silently-dropped transcript write means a // turn that's missing on resume with no trace. (Scrollback above is the // in-memory handoff source and is synchronous, so device-handoff replay diff --git a/src/daemon/settings/manifest.ts b/src/daemon/settings/manifest.ts index 099309d9..3b636e18 100644 --- a/src/daemon/settings/manifest.ts +++ b/src/daemon/settings/manifest.ts @@ -93,6 +93,12 @@ const general: SettingsTab = { default: 200, advanced: true, }), + cfg("session.lazyLoad", "Load history on first use", "A resumed session loads its scrollback and history the first time it is opened or used, not at daemon start. Off loads every session's at start (more memory).", { + kind: "boolean", + envVar: "CODEOID_SESSION_LAZY_LOAD", + default: true, + advanced: true, + }), ], }, { diff --git a/src/daemon/transcript.ts b/src/daemon/transcript.ts index a3f521e8..35e02ed8 100644 --- a/src/daemon/transcript.ts +++ b/src/daemon/transcript.ts @@ -13,7 +13,7 @@ * - Use append-only JSONL — no reads on the hot path */ -import { existsSync, mkdirSync } from "node:fs"; +import { closeSync, existsSync, fstatSync, mkdirSync, openSync, read, readFileSync, readSync, statSync } from "node:fs"; import { appendFile, rename, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { CanonicalHistoryChange, CanonicalToolCall, CanonicalTurn } from "./providers/canonical.js"; @@ -130,6 +130,20 @@ export interface LoadTranscriptOptions { * paging (`scrollback.page`) knows older messages exist on disk. */ stats?: { truncated?: boolean }; + /** + * Read the log as it was when these marks were taken (`transcriptTail`): + * each file up to its size then, wherever rotation has moved it since. What + * a resumed session appended before its scrollback was loaded, which it + * holds itself (`Session#hydrate`), is left out whatever its seqs. + */ + asOf?: readonly TranscriptFileMark[]; +} + +/** One transcript file as it was at a moment: which file (it may be renamed by rotation) and its size then. */ +export interface TranscriptFileMark { + dev: number; + ino: number; + size: number; } /** Tuning knobs, injectable for tests. */ @@ -288,6 +302,13 @@ export class TranscriptStore { ]); } + /** Sessions with a write still in flight (each chain clears itself once its last write settles). */ + get pendingWrites(): number { + return ( + this.#metaWriteChain.size + this.#appendChain.size + this.#canonicalChain.size + this.#turnIndexChain.size + ); + } + // ── Canonical history log (#354) ───────────────────────────────────────── // // The backend-neutral conversation (CanonicalTurn[]) every fork, backend @@ -383,27 +404,56 @@ export class TranscriptStore { async #readCanonicalOnce(sessionId: string, maxBytes?: number): Promise<{ turns: CanonicalTurn[]; partial: boolean }> { const path = this.canonicalPath(sessionId); - const size = Bun.file(path).size; - const offset = maxBytes !== undefined && size > maxBytes ? size - maxBytes : 0; + const offset = canonicalOffset(Bun.file(path).size, maxBytes); const turns: CanonicalTurn[] = []; for await (const line of readLines(path, offset)) { - if (!line.trim()) continue; - try { - const entry = JSON.parse(line) as { op?: string; turn?: CanonicalTurn }; - if (entry.op === "append" && entry.turn && (entry.turn.role === "user" || entry.turn.role === "assistant")) { - turns.push(entry.turn); - } - } catch { - // torn line - } + const turn = canonicalTurnOf(line); + if (turn) turns.push(turn); } - const partial = offset > 0; - if (partial) { - // A tail read may start mid-turn: drop leading assistant turns. - const firstUser = turns.findIndex((t) => t.role === "user"); - turns.splice(0, firstUser === -1 ? turns.length : firstUser); + return canonicalWindow(turns, offset); + } + + /** + * `loadCanonical` read synchronously, for a session whose history a + * synchronous path needs before anything awaited loading it + * (`Session#hydrateNow`). Doesn't wait for pending writes: the caller only + * reads a log nothing has appended to in this process. + */ + loadCanonicalSync(sessionId: string, opts: { maxBytes?: number } = {}): { turns: CanonicalTurn[]; partial: boolean } | null { + const path = this.canonicalPath(sessionId); + const size = fileSize(path); + if (size === null) return null; + let window = opts.maxBytes; + for (;;) { + const offset = canonicalOffset(size, window); + const turns: CanonicalTurn[] = []; + for (const line of readLinesSync(path, offset)) { + const turn = canonicalTurnOf(line); + if (turn) turns.push(turn); + } + const r = canonicalWindow(turns, offset); + if (!r.partial || r.turns.length > 0 || window === undefined) return r; + window *= 2; } - return { turns, partial }; + } + + /** + * The newest turn of a session's canonical log, read from the end of the + * file only: what a resumed session reports (`lastTurnId`, `lastTurnRole`) + * before its history is loaded. Null when the session has no log; + * `last: null` for an empty one. + */ + async canonicalTail(sessionId: string): Promise<{ last: Pick | null } | null> { + await this.#canonicalChain.get(sessionId); + const path = this.canonicalPath(sessionId); + if (fileSize(path) === null) return null; + let last: CanonicalTurn | null = null; + tailLines(path, (line) => { + last = canonicalTurnOf(line); + return last !== null; + }); + const turn = last as CanonicalTurn | null; + return { last: turn ? { role: turn.role, ...(turn.turnId !== undefined ? { turnId: turn.turnId } : {}) } : null }; } /** Append one turn-index entry (#354). Serialized per session; never throws. */ @@ -430,17 +480,23 @@ export class TranscriptStore { await this.#turnIndexChain.get(sessionId); const file = Bun.file(this.turnIndexPath(sessionId)); if (!(await file.exists())) return null; - const out: TurnIndexEntry[] = []; - for (const line of (await file.text()).split("\n")) { - if (!line.trim()) continue; - try { - const e = JSON.parse(line) as TurnIndexEntry; - if (typeof e.turnId === "string" && (e.kind === "prompt" || e.kind === "background")) out.push(e); - } catch { - // torn line - } + return turnIndexOf(await file.text()); + } + + /** + * `loadTurnIndex` read synchronously (`Session#hydrateNow`). Doesn't wait + * for pending writes: the caller only reads an index nothing has written + * to in this process. + */ + loadTurnIndexSync(sessionId: string): TurnIndexEntry[] | null { + let text: string; + try { + text = readFileSync(this.turnIndexPath(sessionId), "utf-8"); + } catch (err) { + if (isMissing(err)) return null; + throw err; } - return out; + return turnIndexOf(text); } async #writeAtomic(path: string, body: string): Promise { @@ -498,13 +554,7 @@ export class TranscriptStore { async append(sessionId: string, msg: DaemonMessage, seq: number): Promise { if (!PERSISTED_TYPES.has(msg.type)) return; - const entry: TranscriptEntry = { - seq, - timestamp: "timestamp" in msg ? (msg as { timestamp: string }).timestamp : new Date().toISOString(), - message: capPersistedToolOutput(msg), - }; - - const line = `${JSON.stringify(entry)}\n`; + const line = transcriptLine(msg, seq); // True append. The previous implementation read the entire file // and rewrote it with the new line concatenated — O(n) per append, @@ -697,32 +747,77 @@ export class TranscriptStore { sessionId: string, opts: LoadTranscriptOptions = {}, ): Promise { - // Oldest → newest: [.N, …, .1, live]. - const candidates: string[] = []; - for (let i = this.#maxRotatedSegments; i >= 1; i--) { - candidates.push(this.#segmentPath(sessionId, i)); + const merged = new TranscriptMerge(); + let sinceDeadlineCheck = 0; + outer: for (const part of this.#transcriptWindow(sessionId, opts)) { + const lines = part.mark ? this.#readMarked(sessionId, part.mark, part.offset, opts) : readLines(part.path, part.offset); + for await (const line of lines) { + if (++sinceDeadlineCheck >= DEADLINE_CHECK_EVERY) { + sinceDeadlineCheck = 0; + if (deadlineHit(sessionId, opts, merged.size)) break outer; + } + merged.add(line); + } + } + return merged.entries(); + } + + /** + * `loadTranscript` read synchronously (same window, deadline and merge), + * for a session whose scrollback a synchronous path needs before anything + * awaited loading it (`Session#hydrateNow`). + */ + loadTranscriptSync(sessionId: string, opts: LoadTranscriptOptions = {}): TranscriptEntry[] { + const merged = new TranscriptMerge(); + let sinceDeadlineCheck = 0; + outer: for (const part of this.#transcriptWindow(sessionId, opts)) { + const lines = part.mark ? this.#readMarkedSync(sessionId, part.mark, part.offset, opts) : readLinesSync(part.path, part.offset); + for (const line of lines) { + if (++sinceDeadlineCheck >= DEADLINE_CHECK_EVERY) { + sinceDeadlineCheck = 0; + if (deadlineHit(sessionId, opts, merged.size)) break outer; + } + merged.add(line); + } } - candidates.push(this.transcriptPath(sessionId)); + return merged.entries(); + } + + /** + * The files (and start offsets) holding the newest `opts.maxBytes` of a + * session's log, oldest first: the files as they are, or as `opts.asOf` + * marked them. Sets `opts.stats.truncated` when older history is left out. + */ + #transcriptWindow( + sessionId: string, + opts: LoadTranscriptOptions, + ): Array<{ path: string; offset: number; mark?: TranscriptFileMark }> { + // Oldest → newest: [.N, …, .1, live]. + const candidates: Array<{ path: string; size: number; mark?: TranscriptFileMark }> = opts.asOf + ? opts.asOf.map((mark) => ({ path: "", size: mark.size, mark })) + : this.#transcriptFiles(sessionId).flatMap((path) => { + const size = fileSize(path); + return size === null ? [] : [{ path, size }]; + }); // Walk newest-first, keeping files while budget remains; the oldest // surviving file may enter mid-way (offset > 0, first partial line // dropped by the reader). - const chosen: Array<{ path: string; offset: number }> = []; + const chosen: Array<{ path: string; offset: number; mark?: TranscriptFileMark }> = []; let budget = opts.maxBytes ?? Number.POSITIVE_INFINITY; let skippedBytes = 0; for (let i = candidates.length - 1; i >= 0; i--) { - const f = Bun.file(candidates[i]!); - if (!(await f.exists())) continue; + const { path, size, mark } = candidates[i]!; if (budget <= 0) { - skippedBytes += f.size; + skippedBytes += size; continue; } - if (f.size <= budget) { - chosen.unshift({ path: candidates[i]!, offset: 0 }); - budget -= f.size; + if (size <= budget) { + chosen.unshift({ path, offset: 0, ...(mark ? { mark } : {}) }); + budget -= size; } else { - skippedBytes += f.size - budget; - chosen.unshift({ path: candidates[i]!, offset: f.size - budget }); + skippedBytes += size - budget; + chosen.unshift({ path, offset: size - budget, ...(mark ? { mark } : {}) }); budget = 0; } } @@ -732,84 +827,110 @@ export class TranscriptStore { `[codeoid] transcript ${sessionId}: replay window capped at ${opts.maxBytes} bytes — ${skippedBytes} bytes of older history left on disk (still exported by share.pack).`, ); } + return chosen; + } - // Merge entries by messageId — later entries update earlier ones. - const byMessageId = new Map(); - const order: string[] = []; - let sinceDeadlineCheck = 0; - - outer: for (const { path, offset } of chosen) { - for await (const line of readLines(path, offset)) { - if (++sinceDeadlineCheck >= DEADLINE_CHECK_EVERY) { - sinceDeadlineCheck = 0; - if (opts.deadlineAt !== undefined && Date.now() > opts.deadlineAt) { - if (opts.stats) opts.stats.truncated = true; - console.warn( - `[codeoid] transcript ${sessionId}: resume deadline hit mid-parse — replaying the ${order.length} message(s) merged so far.`, - ); - break outer; - } - } - if (!line.trim()) continue; - let entry: TranscriptEntry; - try { - entry = JSON.parse(line); - } catch { - continue; // Skip corrupted lines - } - // Valid JSON isn't necessarily a valid entry — `null`, arrays, or - // primitives would throw on the field accesses below. Skip them the - // same way as corrupted lines. - if ( - entry === null || - typeof entry !== "object" || - Array.isArray(entry) || - typeof entry.message !== "object" || - entry.message === null - ) { - continue; - } - const bytes = Buffer.byteLength(line, "utf-8"); - - const msg = entry.message; - const messageId = (msg as { messageId?: string }).messageId; + /** A session's log files, oldest first: [.N, …, .1, live]. */ + #transcriptFiles(sessionId: string): string[] { + const files: string[] = []; + for (let i = this.#maxRotatedSegments; i >= 1; i--) files.push(this.#segmentPath(sessionId, i)); + files.push(this.transcriptPath(sessionId)); + return files; + } - if (!messageId) { - // No messageId (e.g. status_change) — keep as-is with synthetic key - const key = `_seq_${entry.seq}`; - byMessageId.set(key, { ...entry, bytes }); - order.push(key); - continue; - } + /** + * Open the file `mark` names, wherever rotation has moved it since: null + * when it is gone (rotated out of retention; logged, and the read flagged + * as missing history). + */ + #openMarked(sessionId: string, mark: TranscriptFileMark, opts: LoadTranscriptOptions): number | null { + for (const path of this.#transcriptFiles(sessionId).reverse()) { + let fd: number; + try { + fd = openSync(path, "r"); + } catch (err) { + if (isMissing(err)) continue; + throw err; + } + const st = fstatSync(fd); + if (st.dev === mark.dev && st.ino === mark.ino) return fd; + closeSync(fd); + } + if (opts.stats) opts.stats.truncated = true; + console.warn(`[codeoid] transcript ${sessionId}: a file of the log was rotated out before it was read — ${mark.size} bytes of older history left out.`); + return null; + } - if (byMessageId.has(messageId)) { - // Update: merge the newer entry over the older one - const existing = byMessageId.get(messageId)!; - const existingMsg = existing.message as unknown as Record; - const newMsg = msg as unknown as Record; - - // Shallow merge — newer fields overwrite older, preserving what's not in the update - for (const [k, v] of Object.entries(newMsg)) { - if (v !== undefined) existingMsg[k] = v; - } - // Deep merge tool state specifically - if (newMsg.tool && existingMsg.tool) { - Object.assign(existingMsg.tool as Record, newMsg.tool as Record); - } - existing.seq = entry.seq; // update seq to latest - // The merged message is at least as large as its latest full line — - // good enough for scrollback's byte accounting. - existing.bytes = Math.max(existing.bytes ?? 0, bytes); - } else { - // First occurrence — insert - byMessageId.set(messageId, { ...entry, message: { ...msg as object } as typeof msg, bytes }); - order.push(messageId); - } + /** The lines of the file `mark` names, from `offset` up to its size then. */ + async *#readMarked( + sessionId: string, + mark: TranscriptFileMark, + offset: number, + opts: LoadTranscriptOptions, + ): AsyncGenerator { + const fd = this.#openMarked(sessionId, mark, opts); + if (fd === null) return; + try { + const lines = new LineSplitter(offset > 0); + const buf = Buffer.allocUnsafe(READ_CHUNK_BYTES); + for (let pos = offset; pos < mark.size; ) { + const n = await readAt(fd, buf, Math.min(buf.length, mark.size - pos), pos); + if (n <= 0) break; + pos += n; + yield* lines.push(buf.subarray(0, n)); } + yield* lines.end(); + } finally { + closeSync(fd); } + } - // "Went back a turn" (#355): hide what was taken back. - return applyRewinds(order.map((key) => byMessageId.get(key)!)); + /** `#readMarked`, synchronously. */ + *#readMarkedSync(sessionId: string, mark: TranscriptFileMark, offset: number, opts: LoadTranscriptOptions): Generator { + const fd = this.#openMarked(sessionId, mark, opts); + if (fd === null) return; + try { + yield* readFdLinesSync(fd, offset, mark.size); + } finally { + closeSync(fd); + } + } + + /** + * The newest end of a session's log, read from the tail of its newest + * file holding an entry, and a mark of each of its files as they are now + * (`LoadTranscriptOptions.asOf`): what a resumed session needs to keep + * appending before its scrollback is loaded, and to load it later exactly + * as it is now. `nextSeq` is one past the newest entry's seq: the highest + * in a log whose seqs only grow; one whose seqs went back down (an older + * daemon restarted them, an import) continues past the highest once its + * scrollback is loaded (`Session#installHistory`). `hasEntries`: the log + * holds any entry at all, with a seq or not. Throws when a file is there + * but can't be read. + */ + async transcriptTail(sessionId: string): Promise<{ nextSeq: number; hasEntries: boolean; files: TranscriptFileMark[] }> { + await this.#appendChain.get(sessionId); + const paths = this.#transcriptFiles(sessionId); + const files: TranscriptFileMark[] = []; + for (const path of paths) { + const st = fileStat(path); + if (st) files.push({ dev: st.dev, ino: st.ino, size: st.size }); + } + let hasEntries = false; + for (let i = paths.length - 1; i >= 0; i--) { + if (!fileSize(paths[i]!)) continue; + let newest: number | null = null; + tailLines(paths[i]!, (line) => { + const entry = transcriptEntryOf(line); + if (!entry) return false; + hasEntries = true; + if (typeof entry.seq !== "number" || !Number.isFinite(entry.seq)) return false; + newest = entry.seq; + return true; + }); + if (newest !== null) return { nextSeq: (newest as number) + 1, hasEntries, files }; + } + return { nextSeq: 0, hasEntries, files }; } /** @@ -879,6 +1000,302 @@ export function applyRewinds(rows: T[]): T return out; } +/** A turn index file's entries (torn lines left out). */ +function turnIndexOf(text: string): TurnIndexEntry[] { + const out: TurnIndexEntry[] = []; + for (const line of text.split("\n")) { + if (!line.trim()) continue; + try { + const e = JSON.parse(line) as TurnIndexEntry; + if (typeof e.turnId === "string" && (e.kind === "prompt" || e.kind === "background")) out.push(e); + } catch { + // torn line + } + } + return out; +} + +/** One transcript line for `msg` at `seq`, as `append` writes it. */ +function transcriptLine(msg: DaemonMessage, seq: number): string { + const entry: TranscriptEntry = { + seq, + timestamp: "timestamp" in msg ? (msg as { timestamp: string }).timestamp : new Date().toISOString(), + message: withoutBufferSeq(capPersistedToolOutput(msg)), + }; + return `${JSON.stringify(entry)}\n`; +} + +/** + * A message without the replay buffer's `seq` stamp: a cursor into one daemon + * run's scrollback, meaningless once it is read back after a restart (the + * replay frame's `maxSeq` is the client's cursor), and present or not + * depending on whether the session's scrollback was in memory when it was + * appended (lazy session loading). + */ +function withoutBufferSeq(msg: DaemonMessage): DaemonMessage { + if (msg.type !== "session.message" || (msg as { seq?: number }).seq === undefined) return msg; + const { seq: _seq, ...rest } = msg as SessionMessage; + return rest as DaemonMessage; +} + +/** A transcript line's entry, or null for a blank, torn or malformed line. */ +function transcriptEntryOf(line: string): TranscriptEntry | null { + if (!line.trim()) return null; + let entry: TranscriptEntry; + try { + entry = JSON.parse(line); + } catch { + return null; // Skip corrupted lines + } + // Valid JSON isn't necessarily a valid entry — `null`, arrays, or + // primitives would throw on the field accesses below. Skip them the + // same way as corrupted lines. + if ( + entry === null || + typeof entry !== "object" || + Array.isArray(entry) || + typeof entry.message !== "object" || + entry.message === null + ) { + return null; + } + return entry; +} + +/** + * A transcript's lines merged by messageId, in file order — later entries + * update earlier ones — with rewinds applied (`entries()`). + */ +class TranscriptMerge { + #byMessageId = new Map(); + #order: string[] = []; + + get size(): number { + return this.#order.length; + } + + add(line: string): void { + const entry = transcriptEntryOf(line); + if (!entry) return; + const bytes = Buffer.byteLength(line, "utf-8"); + + const msg = entry.message; + const messageId = (msg as { messageId?: string }).messageId; + + if (!messageId) { + // No messageId (e.g. status_change) — keep as-is with synthetic key + const key = `_seq_${entry.seq}`; + this.#byMessageId.set(key, { ...entry, bytes }); + this.#order.push(key); + return; + } + + const existing = this.#byMessageId.get(messageId); + if (existing) { + // Update: merge the newer entry over the older one + const existingMsg = existing.message as unknown as Record; + const newMsg = msg as unknown as Record; + + // Shallow merge — newer fields overwrite older, preserving what's not in the update + for (const [k, v] of Object.entries(newMsg)) { + if (v !== undefined) existingMsg[k] = v; + } + // Deep merge tool state specifically + if (newMsg.tool && existingMsg.tool) { + Object.assign(existingMsg.tool as Record, newMsg.tool as Record); + } + existing.seq = entry.seq; // update seq to latest + // The merged message is at least as large as its latest full line — + // good enough for scrollback's byte accounting. + existing.bytes = Math.max(existing.bytes ?? 0, bytes); + } else { + // First occurrence — insert + this.#byMessageId.set(messageId, { ...entry, message: { ...msg as object } as typeof msg, bytes }); + this.#order.push(messageId); + } + } + + entries(): TranscriptEntry[] { + // "Went back a turn" (#355): hide what was taken back. + return applyRewinds(this.#order.map((key) => this.#byMessageId.get(key)!)); + } +} + +/** Whether a transcript read ran past its deadline (logged, and flagged in `opts.stats`). */ +function deadlineHit(sessionId: string, opts: LoadTranscriptOptions, merged: number): boolean { + if (opts.deadlineAt === undefined || Date.now() <= opts.deadlineAt) return false; + if (opts.stats) opts.stats.truncated = true; + console.warn( + `[codeoid] transcript ${sessionId}: resume deadline hit mid-parse — replaying the ${merged} message(s) merged so far.`, + ); + return true; +} + +/** A canonical log line's turn, or null for a blank, torn or foreign line. */ +function canonicalTurnOf(line: string): CanonicalTurn | null { + if (!line.trim()) return null; + try { + const entry = JSON.parse(line) as { op?: string; turn?: CanonicalTurn }; + if (entry.op === "append" && entry.turn && (entry.turn.role === "user" || entry.turn.role === "assistant")) { + return entry.turn; + } + } catch { + // torn line + } + return null; +} + +/** Where a canonical read of the newest `maxBytes` starts. */ +function canonicalOffset(size: number, maxBytes: number | undefined): number { + return maxBytes !== undefined && size > maxBytes ? size - maxBytes : 0; +} + +/** The turns read from `offset` on, as a history: a tail starts at a prompt. */ +function canonicalWindow(turns: CanonicalTurn[], offset: number): { turns: CanonicalTurn[]; partial: boolean } { + const partial = offset > 0; + if (partial) { + // A tail read may start mid-turn: drop leading assistant turns. + const firstUser = turns.findIndex((t) => t.role === "user"); + turns.splice(0, firstUser === -1 ? turns.length : firstUser); + } + return { turns, partial }; +} + +/** Bytes read per step by the synchronous and marked readers. */ +const READ_CHUNK_BYTES = 1024 * 1024; + +/** A missing file (or a missing dir on its path): what the readers treat as no file. */ +function isMissing(err: unknown): boolean { + const code = (err as NodeJS.ErrnoException).code; + return code === "ENOENT" || code === "ENOTDIR"; +} + +/** A regular file's stat, or null when there is none at `path`. Throws when it can't be looked at. */ +function fileStat(path: string): import("node:fs").Stats | null { + let st: import("node:fs").Stats; + try { + st = statSync(path); + } catch (err) { + if (isMissing(err)) return null; + throw err; + } + return st.isFile() ? st : null; +} + +/** A regular file's size, or null when there is none at `path`. Throws when it can't be looked at. */ +function fileSize(path: string): number | null { + return fileStat(path)?.size ?? null; +} + +/** `pread`, awaited. */ +function readAt(fd: number, buf: Buffer, length: number, position: number): Promise { + return new Promise((resolve, reject) => { + read(fd, buf, 0, length, position, (err, n) => (err ? reject(err) : resolve(n))); + }); +} + +/** + * A byte stream cut into lines, as the readers yield them: with `dropFirst` + * (a read starting mid-file) the first, partial line is left out. + */ +class LineSplitter { + #decoder = new TextDecoder(); + #remainder = ""; + #dropFirst: boolean; + + constructor(dropFirst: boolean) { + this.#dropFirst = dropFirst; + } + + *push(bytes: Uint8Array): Generator { + this.#remainder += this.#decoder.decode(bytes, { stream: true }); + const lines = this.#remainder.split("\n"); + this.#remainder = lines.pop()!; + for (const line of lines) { + if (this.#dropFirst) { + this.#dropFirst = false; + continue; + } + yield line; + } + } + + *end(): Generator { + this.#remainder += this.#decoder.decode(); + if (this.#remainder && !this.#dropFirst) yield this.#remainder; + } +} + +/** `readLines`, synchronously. */ +function* readLinesSync(path: string, offset: number): Generator { + let fd: number; + try { + fd = openSync(path, "r"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") return; + throw err; + } + try { + yield* readFdLinesSync(fd, offset, Number.POSITIVE_INFINITY); + } finally { + closeSync(fd); + } +} + +/** An open file's lines from `offset` up to `end`. */ +function* readFdLinesSync(fd: number, offset: number, end: number): Generator { + const lines = new LineSplitter(offset > 0); + const buf = Buffer.allocUnsafe(READ_CHUNK_BYTES); + for (let pos = offset; pos < end; ) { + const n = readSync(fd, buf, 0, Math.min(buf.length, end - pos), pos); + if (n <= 0) break; + pos += n; + yield* lines.push(buf.subarray(0, n)); + } + yield* lines.end(); +} + +/** + * Hand a file's whole lines to `take`, newest first, reading backwards from + * its end in growing windows, until `take` returns true or the file's start + * is reached. For the few newest lines of a log too big to read whole. + */ +function tailLines(path: string, take: (line: string) => boolean): void { + let fd: number; + try { + fd = openSync(path, "r"); + } catch (err) { + // Gone since it was looked at: nothing to read. Anything else (a file + // that can't be read) is the caller's to fail on, never an empty log. + if (isMissing(err)) return; + throw err; + } + try { + const size = fstatSync(fd).size; + let window = 64 * 1024; + for (;;) { + const start = Math.max(0, size - window); + const buf = Buffer.allocUnsafe(size - start); + let read = 0; + while (read < buf.length) { + const n = readSync(fd, buf, read, buf.length - read, start + read); + if (n <= 0) break; + read += n; + } + const lines = buf.subarray(0, read).toString("utf-8").split("\n"); + // Mid-file, the first line is cut: the next, wider window reads it whole. + const from = start > 0 ? 1 : 0; + for (let i = lines.length - 1; i >= from; i--) { + if (take(lines[i]!)) return; + } + if (start === 0) return; + window *= 4; + } + } finally { + closeSync(fd); + } +} + /** * Stream a file's lines without materialising the whole file. With * `offset > 0` the read starts mid-file and the first (partial) line is diff --git a/src/tests/collaboration.test.ts b/src/tests/collaboration.test.ts index 9e0ff239..ba9e0471 100644 --- a/src/tests/collaboration.test.ts +++ b/src/tests/collaboration.test.ts @@ -489,8 +489,16 @@ let tmp: string; let workdir: string; let store: Store; let transcript: TranscriptStore; +/** Every transcript store the test made (a restart makes another), for the teardown to flush. */ +let transcripts: TranscriptStore[] = []; let manager: SessionManager; +function newTranscriptStore(): TranscriptStore { + const t = new TranscriptStore(join(tmp, "transcripts")); + transcripts.push(t); + return t; +} + function run(msg: ClientMessage): Promise { return manager.handle(msg, AUTH, CLIENT); } @@ -508,7 +516,8 @@ beforeEach(() => { workdir = join(tmp, "repo"); mkdirSync(workdir, { recursive: true }); store = new Store(join(tmp, "codeoid.db")); - transcript = new TranscriptStore(join(tmp, "transcripts")); + transcripts = []; + transcript = newTranscriptStore(); manager = new SessionManager(store, transcript, undefined, undefined, undefined, { config: mkConfig(), providers: makeRegistry(), @@ -522,8 +531,9 @@ afterEach(async () => { // Let in-flight fire-and-forget meta writes land before the tmp dir goes // away. Without this, tearing down a collaboration's children races their // own meta writes and floods the output with ENOENT rename warnings that - // would mask a real failure. - await Bun.sleep(150); + // would mask a real failure. drain flushes the live manager's store; the + // others (a restarted test's first manager's) are flushed here. + await Promise.all(transcripts.map((t) => t.flush())); rmSync(tmp, { recursive: true, force: true }); }); @@ -2142,7 +2152,7 @@ describe("collaboration survives a daemon restart", () => { await Bun.sleep(150); // let in-flight meta writes land const next = new SessionManager( new Store(join(tmp, "codeoid.db")), - new TranscriptStore(join(tmp, "transcripts")), + newTranscriptStore(), undefined, undefined, undefined, @@ -2214,7 +2224,7 @@ describe("collaboration survives a daemon restart", () => { claudeOnly.register({ id: "claude", displayName: "claude", create: () => new MockSessionProvider("claude", []) }); const next = new SessionManager( new Store(join(tmp, "codeoid.db")), - new TranscriptStore(join(tmp, "transcripts")), + newTranscriptStore(), undefined, undefined, undefined, { config: mkConfig(), providers: claudeOnly }, ); @@ -3105,7 +3115,7 @@ describe("the collaboration cost roll-up", () => { await Bun.sleep(150); const next = new SessionManager( new Store(join(tmp, "codeoid.db")), - new TranscriptStore(join(tmp, "transcripts")), + newTranscriptStore(), undefined, undefined, undefined, { config: mkConfig(), providers: makeRegistry() }, ); diff --git a/src/tests/dispatcher.test.ts b/src/tests/dispatcher.test.ts index 4d3ebc2a..c47cb371 100644 --- a/src/tests/dispatcher.test.ts +++ b/src/tests/dispatcher.test.ts @@ -14,6 +14,7 @@ import { NonRetryableDispatchError, type DispatcherHost, } from "../daemon/dispatch.js"; +import { DaemonStoppingError } from "../daemon/session.js"; import { Store, type DispatchEventRow, type DispatchTaskRow } from "../daemon/store.js"; import type { SessionStatus } from "../protocol/types.js"; @@ -634,3 +635,39 @@ describe("dispatch barrier — N-way join", () => { expect(members.map((m) => m.groupId)).toEqual([groupId, groupId, groupId]); }); }); + +describe("a tick in flight as the daemon stops", () => { + /** What Session.send throws once drain began: the real one, which dispatch.ts knows by name. */ + const stopping = () => new DaemonStoppingError(); + + test("a spawn or send refused because the daemon is stopping goes back to the queue, no attempt spent", async () => { + const spawn = enqueueSpawn(); + const send = enqueueSend(); + host.spawnError = stopping(); + host.sendError = stopping(); + await dispatcher.tick(); + for (const id of [spawn, send]) { + const row = store.dispatchGet(id)!; + expect(row.status).toBe("queued"); + expect(row.attempts).toBe(0); + expect(row.error).toBeNull(); + } + expect(host.audits.filter((a) => a.startsWith("dispatch.released"))).toHaveLength(2); + expect(host.audits.some((a) => a.startsWith("dispatch.failed"))).toBe(false); + }); + + test("any other failure still spends an attempt", async () => { + const spawn = enqueueSpawn(); + host.spawnError = new Error("boom"); + await dispatcher.tick(); + expect(store.dispatchGet(spawn)!.attempts).toBe(1); + }); + + test("a stopped dispatcher's tick claims nothing more", async () => { + const spawn = enqueueSpawn(); + dispatcher.stop(); + await dispatcher.tick(); + expect(store.dispatchGet(spawn)!.status).toBe("queued"); + expect(host.spawned).toHaveLength(0); + }); +}); diff --git a/src/tests/lazy-session-loading.test.ts b/src/tests/lazy-session-loading.test.ts new file mode 100644 index 00000000..0de02565 --- /dev/null +++ b/src/tests/lazy-session-loading.test.ts @@ -0,0 +1,1262 @@ +/** + * Lazy session loading: a resumed session holds its metadata only, and loads + * its scrollback and canonical history the first time something uses it. + * + * Every client-visible behaviour of a resumed session must be the one an + * eagerly loaded session has. So each test here runs the same scenario on two + * copies of one daemon's data — one resumed with `session.lazyLoad: false` + * (the history loaded at start, as before) and one lazily — and compares what + * a client sees: ids, timestamps and the data dirs normalized, nothing else. + */ + +import { afterAll, afterEach, beforeAll, describe, expect, it, setDefaultTimeout } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CodeoidConfig } from "../config.js"; +import type { CanonicalTurn } from "../daemon/providers/canonical.js"; +import type { ProviderEvent, SessionScopedEvent } from "../daemon/providers/interface.js"; +import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; +import { ProviderRegistry } from "../daemon/providers/registry.js"; +import { type AttachedClient, Session } from "../daemon/session.js"; +import { SessionManager } from "../daemon/session-manager.js"; +import { Store } from "../daemon/store.js"; +import { TranscriptStore } from "../daemon/transcript.js"; +import { ALL_SCOPES } from "../protocol/scopes.js"; +import { type AuthContext, CAPABILITIES, type DaemonMessage, type SessionInfo, type SessionMessage } from "../protocol/types.js"; + +// Two daemons per test, each resuming and shutting down: slower than the default under a loaded suite. +setDefaultTimeout(60_000); + +const AUTH: AuthContext = { + sub: "user:lazy", + scopes: [...ALL_SCOPES] as AuthContext["scopes"], + delegationDepth: 0, + accountId: "acc-lazy", + projectId: "proj-lazy", +}; + +const RESULT = { + providerId: "mock", + model: "mock", + inputTokens: 10, + outputTokens: 5, + cacheReadTokens: 0, + cacheCreationTokens: 0, + totalCostUsd: 0.01, + durationMs: 1, +}; + +function say(text: string): ProviderEvent[] { + return [{ type: "text_done", content: text } as ProviderEvent, { type: "turn_done", result: RESULT } as ProviderEvent]; +} + +/** A turn that reads a file (a safe tool: it runs unasked), then answers. */ +function readThenSay(n: number, text: string): ProviderEvent[] { + return [ + { type: "tool_start", toolId: `t${n}`, sdkToolUseId: `sdk-t${n}`, name: "Read", input: { file_path: "a.txt" }, approvalId: `ap-${n}` } as ProviderEvent, + { type: "tool_complete", sdkToolUseId: `sdk-t${n}`, output: `v0 (read ${n})`, success: true } as ProviderEvent, + ...say(text), + ]; +} + +/** Replies long enough that a 2 KiB attach tail and paging have work to do. */ +const long = (n: number): string => `answer ${n}: ${"lorem ipsum dolor sit amet ".repeat(12)}`; + +function mkConfig(root: string, lazyLoad?: boolean): CodeoidConfig { + return { + daemonUrl: "ws://127.0.0.1:7400", + dbPath: join(root, "codeoid.db"), + transcriptDir: join(root, "transcripts"), + auth: { baseUrl: "http://localhost:8899" }, + zeroidUrl: "http://localhost:8899", + workspaceIndex: { enabled: false, episodeThreshold: 5, timeThresholdMs: 60_000, debounceMs: 15_000 }, + compress: { enabled: false, excludeCommands: [], excludePatterns: [], compressPipes: false, minBytes: 1024 }, + labeling: {}, + telemetry: { osc8: "auto" }, + autoRotate: { enabled: false, warnPct: 0.6, rotatePct: 0.8, hardRotatePct: 0.9, minTurnsBeforeRotate: 3, strategy: "task-anchor" }, + session: { attachTailBytes: 2048, ...(lazyLoad !== undefined ? { lazyLoad } : {}) }, + conductor: { enabled: false, name: "conductor", provider: "claude" }, + } as CodeoidConfig; +} + +/** One daemon over one data dir. */ +interface Env { + label: string; + root: string; + repo: string; + store: Store; + transcript: TranscriptStore; + m: SessionManager; + /** Every provider the manager built, oldest first. */ + made: MockSessionProvider[]; +} + +/** Each provider replies to the next prompts with these, in order. */ +const REPLIES = (): ProviderEvent[][] => [say("r1"), say("r2"), say("r3"), say("r4")]; + +function openEnv(label: string, root: string, lazyLoad?: boolean, script: () => ProviderEvent[][] = REPLIES): Env { + const made: MockSessionProvider[] = []; + const registry = new ProviderRegistry("claude"); + for (const id of ["claude", "codex", "gemini-cli"]) { + registry.register({ + id, + displayName: id, + create: () => { + const p = new MockSessionProvider(id, script()); + made.push(p); + return p; + }, + }); + } + const store = new Store(join(root, "codeoid.db")); + const transcript = new TranscriptStore(join(root, "transcripts")); + const m = new SessionManager(store, transcript, undefined, undefined, undefined, { + config: mkConfig(root, lazyLoad), + providers: registry, + }); + return { label, root, repo: join(root, "repo"), store, transcript, m, made }; +} + +async function closeEnv(env: Env): Promise { + env.m.stopRoutines(); + env.m.stopLoops(); + await env.m.drain(3_000).catch(() => {}); + await env.transcript.flush(); + try { + env.store.close(); + } catch {} +} + +const client = (id: string, sink: DaemonMessage[], capabilities?: string[]): AttachedClient => ({ + id, + auth: AUTH, + send: (msg: DaemonMessage) => sink.push(msg), + ...(capabilities ? { capabilities } : {}), +}); + +async function call(env: Env, msg: Record, sink: DaemonMessage[] = []): Promise { + return env.m.handle({ id: "req", ...msg } as never, AUTH, client("cli", sink)); +} + +async function settle(env: Env, sessionId: string): Promise { + const s = env.m._sessionForTest(sessionId)!; + const busy = () => s.status !== "idle" || s.turnInFlight || s.preparingTurn; + for (let i = 0; i < 400 && busy(); i++) await Bun.sleep(5); + expect(s.status).toBe("idle"); + await Bun.sleep(20); +} + +async function sendAndSettle(env: Env, sessionId: string, text: string): Promise { + const r = await call(env, { type: "session.send", sessionId, text }); + await settle(env, sessionId); + return r; +} + +// ── The data both daemons resume ───────────────────────────────────────── + +let base: string; +/** Session ids in the fixture, by name. */ +const ids: Record = {}; +const roots: string[] = []; + +/** + * One daemon's data: a long session with tool calls and notices, a short + * one, an empty one, one on a backend that is re-seeded after a teardown, + * and one from before the canonical log (loaded at start either way). + */ +beforeAll(async () => { + base = mkdtempSync(join(tmpdir(), "codeoid-lazy-base-")); + const repo = join(base, "repo"); + mkdirSync(repo); + const git = (...args: string[]) => execFileSync("git", args, { cwd: repo, encoding: "utf8" }); + git("init", "-q"); + git("config", "user.email", "t@example.com"); + git("config", "user.name", "t"); + writeFileSync(join(repo, "a.txt"), "v0\n"); + git("add", "."); + git("commit", "-qm", "init"); + + const script = () => [ + say(long(1)), + readThenSay(2, long(2)), + say(long(3)), + readThenSay(4, long(4)), + say(long(5)), + say(long(6)), + ]; + const env = openEnv("base", base, undefined, script); + const create = async (name: string, providerId?: string): Promise => { + const r = await call(env, { type: "session.create", name, workdir: repo, ...(providerId ? { providerId } : {}) }); + if (r.type !== "response.ok") throw new Error(JSON.stringify(r)); + return (r.data as { id: string }).id; + }; + ids.rich = await create("rich"); + for (let i = 1; i <= 6; i++) await sendAndSettle(env, ids.rich, `prompt ${i}: ${"please ".repeat(20)}`); + await call(env, { type: "session.pin", sessionId: ids.rich, path: "a.txt" }); + await call(env, { type: "session.set_mode", sessionId: ids.rich, mode: "autonomous", maxTurns: 7 }); + + ids.short = await create("short"); + await sendAndSettle(env, ids.short, "hello"); + await call(env, { type: "session.rename", sessionId: ids.short, name: "short-renamed" }); + + ids.empty = await create("empty"); + + ids.gemini = await create("gemini", "gemini-cli"); + await sendAndSettle(env, ids.gemini, "hi gemini"); + + ids.legacy = await create("legacy"); + await sendAndSettle(env, ids.legacy, "old times"); + + // A routine that keeps its session across runs: its session is resumed. + const routine = await call(env, { + type: "routine.create", + spec: { name: "daily", prompt: "report", workdir: repo, approvals: "guarded", session: "persistent" }, + }); + if (routine.type !== "routine.snapshot") throw new Error(JSON.stringify(routine)); + ids.routine = routine.routine.id; + const run = await call(env, { type: "routine.run", routine: "daily" }); + if (run.type !== "routine.run.result") throw new Error(JSON.stringify(run)); + ids.routineSession = (await runEnds(env, ids.routine, run.run.id)).sessionId!; + + await closeEnv(env); + // From before the canonical log and the turn index (#354). + rmSync(join(base, "transcripts", `${ids.legacy}.canonical.jsonl`)); + rmSync(join(base, "transcripts", `${ids.legacy}.turns.jsonl`)); +}, 60_000); + +afterAll(() => { + rmSync(base, { recursive: true, force: true }); +}); + +const envs: Env[] = []; +afterEach(async () => { + for (const e of envs.splice(0)) await closeEnv(e); + for (const r of roots.splice(0)) rmSync(r, { recursive: true, force: true }); +}); + +/** A copy of the fixture, its sessions pointed at the copy's repo. */ +function cloneBase(label: string): string { + // Same-length names: a path inside an export counts toward its size. + const root = mkdtempSync(join(tmpdir(), `codeoid-lazy-${label.slice(0, 4)}-`)); + roots.push(root); + cpSync(base, root, { recursive: true }); + const dir = join(root, "transcripts"); + for (const f of readdirSync(dir)) { + if (!f.endsWith(".meta.json")) continue; + const p = join(dir, f); + writeFileSync(p, readFileSync(p, "utf8").split(base).join(root)); + } + // A copied repository's index has stale stat data, which makes a fork's + // `git stash create` fail: refresh it, as any git command would. + execFileSync("git", ["status", "--porcelain"], { cwd: join(root, "repo") }); + return root; +} + +/** The fixture resumed eagerly and lazily. */ +async function both(script?: () => ProviderEvent[][]): Promise<{ eager: Env; lazy: Env }> { + const eager = openEnv("eager", cloneBase("eager"), false, script); + const lazy = openEnv("lazy", cloneBase("lazy"), true, script); + envs.push(eager, lazy); + await eager.m.resumeSessions(); + await lazy.m.resumeSessions(); + return { eager, lazy }; +} + +const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g; +const ISO = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z/g; +/** A git object id (a checkpoint taken during the test embeds the clock). */ +const SHA = /\b[0-9a-f]{40}\b/g; +/** Epoch milliseconds (a turn's `createdAt`). */ +const EPOCH_MS = /\b1[7-9]\d{11}\b/g; + +/** A stable placeholder per distinct value, numbered by first appearance. */ +function token(seen: Map, value: string, kind: string): string { + let t = seen.get(value); + if (t === undefined) { + t = `<${kind}${seen.size}>`; + seen.set(value, t); + } + return t; +} + +/** What a client sees, with each daemon's own ids, clock and paths taken out. */ +function norm(env: Env, v: unknown): unknown { + const fixed = new Map(Object.values(ids).map((id) => [id, id])); + const fresh = new Map(); + const shas = new Map(); + const text = JSON.stringify(v ?? null) + .split(env.root) + .join("") + .split(base) + .join("") + .replace(ISO, "") + .replace(EPOCH_MS, "\"\"") + .replace(SHA, (sha) => token(shas, sha, "sha")) + .replace(UUID, (id) => fixed.get(id) ?? token(fresh, id, "id")); + return JSON.parse(text); +} + +/** Run `scenario` on both and expect the same result. */ +async function same(pair: { eager: Env; lazy: Env }, scenario: (env: Env) => Promise): Promise { + const before = Session.syncHydrations; + const a = await scenario(pair.eager); + const b = await scenario(pair.lazy); + expect(norm(pair.lazy, b)).toEqual(norm(pair.eager, a)); + // Every path above awaited the load: none had to block on it. + expect(Session.syncHydrations).toBe(before); + return b; +} + +const sessionOf = (env: Env, name: keyof typeof ids) => env.m._sessionForTest(ids[name]!)!; +const loaded = (env: Env, name: keyof typeof ids) => sessionOf(env, name).hydrated; + +/** Infos as a client lists them. */ +async function list(env: Env): Promise { + const r = await call(env, { type: "session.list" }); + if (r.type !== "session.list.result") throw new Error(JSON.stringify(r)); + return [...r.sessions].sort((x, y) => x.id.localeCompare(y.id)); +} + +/** What attaching shows. */ +async function attach(env: Env, sessionId: string, capabilities?: string[], resume?: { key: string; sinceSeq: number }): Promise { + const frames: DaemonMessage[] = []; + const r = await env.m.handle( + { type: "session.attach", id: "att", sessionId, ...(resume ? { resume } : {}) } as never, + AUTH, + client(`att-${Math.random()}`, frames, capabilities), + ); + return [r, ...frames.filter((f) => f.type === "scrollback.replay")]; +} + +async function runEnds(env: Env, routine: string, runId: string): Promise<{ status: string; sessionId?: string; summary?: string }> { + for (let i = 0; i < 400; i++) { + const r = await call(env, { type: "routine.runs", routine, runId }); + const run = r.type === "routine.runs.result" ? r.runs[0] : undefined; + if (run && !["queued", "checking", "running"].includes(run.status)) return run; + await Bun.sleep(25); + } + throw new Error("routine run never ended"); +} + +/** The transcript's seqs, in file order. */ +function seqsOnDisk(env: Env, sessionId: string): number[] { + const text = readFileSync(env.transcript.transcriptPath(sessionId), "utf8"); + return text + .split("\n") + .filter((l) => l.trim()) + .map((l) => (JSON.parse(l) as { seq: number }).seq); +} + +const PAGING = [CAPABILITIES.SCROLLBACK_PAGING]; + +// ── Tests ──────────────────────────────────────────────────────────────── + +describe("lazy session loading: a resumed session before it is used", () => { + it("lists with the same info, and loads nothing (a session from before the canonical log, or with none yet, loads at start)", async () => { + const pair = await both(); + await same(pair, list); + await same(pair, async (env) => Object.keys(ids).filter((k) => k !== "routine").map((k) => env.m._sessionForTest(ids[k]!)?.toInfo())); + for (const name of ["rich", "short", "gemini", "routineSession"] as const) { + expect([name, loaded(pair.eager, name)]).toEqual([name, true]); + expect([name, loaded(pair.lazy, name)]).toEqual([name, false]); + } + expect(loaded(pair.lazy, "legacy")).toBe(true); + // Never had a turn: no log to read lazily, and nothing to hold. + expect(loaded(pair.lazy, "empty")).toBe(true); + }); + + it("reports the same newest turn (what a routine reads before its run) without loading", async () => { + const pair = await both(); + await same(pair, async (env) => + (["rich", "short", "empty", "gemini"] as const).map((n) => [sessionOf(env, n).lastTurnId, sessionOf(env, n).lastTurnRole, sessionOf(env, n).currentTurnId]), + ); + expect(loaded(pair.lazy, "rich")).toBe(false); + }); + + it("the fleet board shows the same, without loading", async () => { + const pair = await both(); + await same(pair, async (env) => { + const r = await call(env, { type: "fleet.subscribe", scope: "tenant" }); + return r; + }); + expect(loaded(pair.lazy, "rich")).toBe(false); + }); + + it("the turn list is the same, without loading", async () => { + const pair = await both(); + await same(pair, async (env) => call(env, { type: "session.turns", sessionId: ids.rich })); + expect(loaded(pair.lazy, "rich")).toBe(false); + }); + + it("notices while unloaded (model, effort) are in the scrollback and on disk in order, their seqs never reused", async () => { + const pair = await both(); + const before = seqsOnDisk(pair.lazy, ids.short!).length; + await same(pair, async (env) => { + const results = [ + await call(env, { type: "session.set_mode", sessionId: ids.short, mode: "guarded" }), + await call(env, { type: "session.rename", sessionId: ids.short, name: "short-again" }), + await call(env, { type: "session.set_model", sessionId: ids.short, model: "sonnet" }), + await call(env, { type: "session.set_effort", sessionId: ids.short, effort: "high" }), + ]; + expect(results.map((r) => r.type)).toEqual(["response.ok", "response.ok", "response.ok", "response.ok"]); + await env.transcript.flush(); + return { results, info: sessionOf(env, "short").toInfo() }; + }); + // Appended while the history stayed on disk. + expect(loaded(pair.lazy, "short")).toBe(false); + expect(seqsOnDisk(pair.lazy, ids.short!).length).toBeGreaterThan(before); + await same(pair, async (env) => attach(env, ids.short!)); + await same(pair, async (env) => { + await sendAndSettle(env, ids.short!, "after the notices"); + await env.transcript.flush(); + return attach(env, ids.short!); + }); + for (const env of [pair.eager, pair.lazy]) { + const seqs = seqsOnDisk(env, ids.short!); + expect(new Set(seqs).size).toBe(seqs.length); + expect([...seqs].sort((x, y) => x - y)).toEqual(seqs); + } + expect(seqsOnDisk(pair.lazy, ids.short!)).toEqual(seqsOnDisk(pair.eager, ids.short!)); + }); +}); + +describe("lazy session loading: using a resumed session", () => { + it("attach replays the same: tail-first with paging, whole for a legacy client, and a stale cursor gets a snapshot", async () => { + const pair = await both(); + await same(pair, (env) => attach(env, ids.rich!, PAGING)); + await same(pair, (env) => attach(env, ids.rich!)); + await same(pair, (env) => attach(env, ids.rich!, PAGING, { key: "from-before-the-restart", sinceSeq: 3 })); + expect(loaded(pair.lazy, "rich")).toBe(true); + }); + + it("incremental resume by seq continues the same after a new turn", async () => { + const pair = await both(); + await same(pair, async (env) => { + const first = await attach(env, ids.rich!, PAGING); + const replay = first.find((f) => f.type === "scrollback.replay") as { resumeKey: string; maxSeq: number }; + await sendAndSettle(env, ids.rich!, "one more"); + const again = await attach(env, ids.rich!, PAGING, { key: replay.resumeKey, sinceSeq: replay.maxSeq }); + return { maxSeq: replay.maxSeq, again }; + }); + }); + + it("paging back through the whole history gives the same pages; MCP control's read_transcript page too", async () => { + const pair = await both(); + await same(pair, async (env) => { + const pages: DaemonMessage[] = []; + // read_transcript: the newest page, no anchor. + pages.push(await call(env, { type: "scrollback.page", sessionId: ids.rich, maxBytes: 4096 })); + const replay = (await attach(env, ids.rich!, PAGING)).find((f) => f.type === "scrollback.replay") as { messages: SessionMessage[] }; + let anchor = replay.messages[0]?.messageId; + for (let i = 0; anchor && i < 50; i++) { + const page = await call(env, { type: "scrollback.page", sessionId: ids.rich, beforeMessageId: anchor, maxBytes: 1500 }); + pages.push(page); + const p = page as { messages: SessionMessage[]; hasMore: boolean }; + if (!p.hasMore || p.messages.length === 0) break; + anchor = p.messages[0]!.messageId; + } + return pages; + }); + }); + + it("forking at a turn carries the same conversation, turns and scrollback", async () => { + const pair = await both(); + const forkId = await same(pair, async (env) => { + const turns = (await call(env, { type: "session.turns", sessionId: ids.rich })) as { turns: Array<{ turnId: string }> }; + const r = await call(env, { type: "session.fork", sessionId: ids.rich, name: "branch", afterTurnId: turns.turns[2]!.turnId, isolate: false }); + expect(r.type).toBe("response.ok"); + return r; + }); + const id = (forkId as { data?: { id: string } }).data?.id ?? (forkId as { session?: { id: string } }).session?.id; + expect(id).toBeString(); + await same(pair, async (env) => { + const r = (await call(env, { type: "session.list" })) as { sessions: SessionInfo[] }; + const fork = r.sessions.find((s) => s.name === "branch")!; + const p = env.made.at(-1)!; + return { + fork, + replay: await attach(env, fork.id, PAGING), + turns: await call(env, { type: "session.turns", sessionId: fork.id }), + history: env.m._sessionForTest(fork.id)!.canonicalHistory.map((t: CanonicalTurn) => [t.role, t.content, t.turnId]), + seeded: p.seededHistory?.map((t) => [t.role, t.content]), + }; + }); + const fork = (await list(pair.lazy)).find((s) => s.name === "branch")!; + expect(pair.lazy.m._sessionForTest(fork.id)!.canonicalHistory.length).toBe(6); + }); + + it("going back a turn removes the same, and the next prompt carries the same history", async () => { + const pair = await both(); + await same(pair, async (env) => { + const turns = (await call(env, { type: "session.turns", sessionId: ids.rich })) as { turns: Array<{ turnId: string }> }; + const r = await call(env, { type: "session.rewind", sessionId: ids.rich, turnId: turns.turns[4]!.turnId }); + expect(r.type).toBe("session.rewind.result"); + await settle(env, ids.rich!); + const after = await attach(env, ids.rich!, PAGING); + await sendAndSettle(env, ids.rich!, "instead"); + const opts = env.made.at(-1)!.capturedOpts.at(-1)!; + return { r, after, history: opts.history.map((t) => [t.role, t.content]), user: opts.userMessage }; + }); + }); + + it("a comparison runs from the same conversation and settles the same", async () => { + const pair = await both(); + await same(pair, async (env) => { + const r = (await call(env, { + type: "session.compare", + sessionId: ids.rich, + prompt: "do it two ways", + targets: [{ providerId: "claude" }, { providerId: "codex" }], + })) as { type: string; compare?: { compareId: string } }; + expect([r.type, (r as { error?: string }).error]).toEqual(["compare.state", undefined]); + if (r.type !== "compare.state") return r; + for (let i = 0; i < 400; i++) { + const g = (await call(env, { type: "compare.get", compareId: r.compare!.compareId })) as { compare: { targets: Array<{ done?: boolean }> } }; + if (g.compare.targets.every((t) => t.done)) { + const seeded = env.made.slice(-2).map((p) => p.seededHistory?.map((t) => [t.role, t.content])); + return { g, seeded }; + } + await Bun.sleep(10); + } + throw new Error("comparison never settled"); + }); + }); + + it("a backend switch hands the new backend the same history", async () => { + const pair = await both(); + await same(pair, async (env) => { + const r = await call(env, { type: "session.set_provider", sessionId: ids.rich, providerId: "codex" }); + expect(r.type).toBe("response.ok"); + const p = env.made.at(-1)!; + expect(p.id).toBe("codex"); + return { r, id: p.id, seeded: p.seededHistory?.map((t) => [t.role, t.content, t.turnId]), replay: await attach(env, ids.rich!, PAGING) }; + }); + }); + + it("export bundles the same", async () => { + const pair = await both(); + await same(pair, async (env) => { + const r = (await call(env, { type: "session.export", sessionId: ids.rich, includeMemory: false })) as unknown as Record; + expect(r.type).toBe("session.export.result"); + return r; + }); + }); + + it("a new prompt hands the backend the same history, and the transcript's seq continues past the log", async () => { + const pair = await both(); + await same(pair, async (env) => { + const before = seqsOnDisk(env, ids.rich!); + const r = await sendAndSettle(env, ids.rich!, "next"); + await env.transcript.flush(); + const after = seqsOnDisk(env, ids.rich!); + expect(after.slice(0, before.length)).toEqual(before); + expect(Math.min(...after.slice(before.length))).toBe(Math.max(...before) + 1); + expect(new Set(after).size).toBe(after.length); + const opts = env.made.at(-1)!.capturedOpts.at(-1)!; + return { r, history: opts.history.map((t) => [t.role, t.content, t.turnId]), hasQueried: env.made.at(-1)!.hasQueried, info: sessionOf(env, "rich").toInfo() }; + }); + }); + + it("a background task settling on a resumed session wakes it on the same history", async () => { + const pair = await both(); + await same(pair, async (env) => { + const s = sessionOf(env, "short"); + const p = env.made.find((x) => x.onSessionEvent && env.m._sessionForTest(ids.short!)?.providerId === x.id && s.toInfo().rotation?.claudeCodeSessionId === x.backingSessionId)!; + p.onSessionEvent!({ type: "background_tasks", tasks: [{ id: "bg1", kind: "local_bash", description: "build", status: "running" }] } as SessionScopedEvent); + p.onSessionEvent!({ type: "background_task_settled", taskId: "bg1", status: "completed", summary: "built" } as SessionScopedEvent); + for (let i = 0; i < 200 && p.capturedOpts.length === 0; i++) await Bun.sleep(5); + await settle(env, ids.short!); + expect(p.capturedOpts.length).toBe(1); + return { turns: p.capturedOpts.map((o) => ({ user: o.userMessage, history: o.history.map((t) => [t.role, t.content]) })), replay: await attach(env, ids.short!, PAGING) }; + }); + }); + + it("a persistent routine's run on its resumed session ends the same", async () => { + const pair = await both(); + await same(pair, async (env) => { + const r = await call(env, { type: "routine.run", routine: "daily" }); + if (r.type !== "routine.run.result") throw new Error(JSON.stringify(r)); + const run = await runEnds(env, ids.routine!, r.run.id); + expect(run.status).toBe("succeeded"); + expect(run.sessionId).toBe(ids.routineSession); + const p = env.made.find((x) => x.capturedOpts.length > 0)!; + expect(p.capturedOpts.at(-1)!.history.length).toBeGreaterThan(2); + return { run, history: p.capturedOpts.at(-1)?.history.map((t) => [t.role, t.content]) }; + }); + }); + + it("an idle release before first use re-seeds a backend that needs it once the history loads", async () => { + const pair = await both(); + await same(pair, async (env) => { + const s = sessionOf(env, "gemini"); + s.releaseIdleProvider({ idle: true }); + await Bun.sleep(50); + const p = env.made.find((x) => x.id === "gemini-cli")!; + // Before anything uses it: the lazy one owes the seed until it loads. + if (env.label === "lazy") expect(s.hydrated).toBe(false); + await sendAndSettle(env, ids.gemini!, "back again"); + expect(p.seededHistory?.length).toBe(2); + return { seeded: p.seededHistory?.map((t) => [t.role, t.content]), teardowns: p.teardownCount, history: p.capturedOpts.at(-1)?.history.map((t) => [t.role, t.content]) }; + }); + }); +}); + +describe("lazy session loading: restarts", () => { + it("restarting twice, with notices (some while unloaded) and a turn in between, keeps everything the same", async () => { + const pair = await both(); + const restart = async (env: Env): Promise => { + await closeEnv(env); + envs.splice(envs.indexOf(env), 1); + const next = openEnv(env.label, env.root, env.label === "lazy"); + envs.push(next); + await next.m.resumeSessions(); + return next; + }; + let { eager, lazy } = pair; + const models: Record = {}; + const step = async (env: Env) => { + await call(env, { type: "session.set_mode", sessionId: ids.short, mode: "autonomous", maxTurns: 2 }); + // A notice appended while "short" is unloaded (lazy), persisted across the restarts. + const n = (models[env.label] ?? 0) + 1; + models[env.label] = n; + const r = await call(env, { type: "session.set_model", sessionId: ids.short, model: n === 1 ? "sonnet" : "haiku" }); + expect([r.type, (r as { error?: string }).error]).toEqual(["response.ok", undefined]); + if (env.label === "lazy") expect(sessionOf(env, "short").hydrated).toBe(false); + await sendAndSettle(env, ids.rich!, "between restarts"); + return list(env); + }; + await same({ eager, lazy }, step); + eager = await restart(eager); + lazy = await restart(lazy); + await same({ eager, lazy }, list); + await same({ eager, lazy }, step); + eager = await restart(eager); + lazy = await restart(lazy); + await same({ eager, lazy }, list); + await same({ eager, lazy }, async (env) => ({ + short: await attach(env, ids.short!), + rich: await attach(env, ids.rich!, PAGING), + turns: await call(env, { type: "session.turns", sessionId: ids.rich }), + })); + for (const env of [eager, lazy]) { + const seqs = seqsOnDisk(env, ids.short!); + expect(new Set(seqs).size).toBe(seqs.length); + const replay = (await attach(env, ids.short!)).find((f) => f.type === "scrollback.replay") as { messages: SessionMessage[] }; + expect(replay.messages.filter((m) => m.metadata?.event === "session.model_changed").length).toBe(2); + } + }); +}); + +/** + * A session from a long-lived daemon: a transcript past the 24 MiB the + * scrollback is read from, and a canonical log past the 4 MiB the history + * is, written into both copies. `seqOf` gives line i of n its seq (none: undefined). + */ +function writeBig(root: string, seqOf: (i: number, n: number) => number | undefined = (i) => i): string { + const id = "0b16b16b-0000-4000-8000-00000000b16b"; + const dir = join(root, "transcripts"); + const at = (i: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, i)).toISOString(); + const lines: string[] = []; + const canon: string[] = []; + const index: string[] = []; + const entries: Array<{ timestamp: string; message: object }> = []; + for (let t = 0; t < 1100; t++) { + const turnId = `turn-${String(t).padStart(5, "0")}`; + const prompt = `prompt ${t} ${"x".repeat(200)}`; + const reply = `reply ${t} ${"y".repeat(24_000)}`; + const user = { type: "session.message", sessionId: id, messageId: `u-${t}`, role: "user", content: prompt, identity: { sub: AUTH.sub, type: "human" }, timestamp: at(2 * t), turnId }; + const asst = { type: "session.message", sessionId: id, messageId: `a-${t}`, role: "assistant", content: reply, identity: { sub: `agent:${id}`, type: "agent", name: "big (Claude)" }, timestamp: at(2 * t + 1), turnId }; + entries.push({ timestamp: user.timestamp, message: user }, { timestamp: asst.timestamp, message: asst }); + canon.push(JSON.stringify({ op: "append", turn: { role: "user", content: prompt, turnId, prompt, at: user.timestamp } })); + canon.push(JSON.stringify({ op: "append", turn: { role: "assistant", content: reply.slice(0, 4_000), turnId } })); + index.push(JSON.stringify({ turnId, kind: "prompt", preview: `prompt ${t}`, startedAt: user.timestamp })); + } + for (let i = 0; i < entries.length; i++) lines.push(JSON.stringify({ seq: seqOf(i, entries.length), ...entries[i]! })); + writeFileSync(join(dir, `${id}.jsonl`), `${lines.join("\n")}\n`); + writeFileSync(join(dir, `${id}.canonical.jsonl`), `${canon.join("\n")}\n`); + writeFileSync(join(dir, `${id}.turns.jsonl`), `${index.join("\n")}\n`); + writeFileSync( + join(dir, `${id}.meta.json`), + JSON.stringify({ + sessionId: id, + sessionName: "big", + workdir: join(root, "repo"), + createdBy: AUTH.sub, + createdAt: at(0), + lastStatus: "idle", + lastActivityAt: at(2200), + accountId: AUTH.accountId, + projectId: AUTH.projectId, + providerId: "claude", + }), + ); + return id; +} + +describe("lazy session loading: a history past the resume budgets", () => { + it("loads the same windows: a partial scrollback paged from disk, a partial history read whole on demand", async () => { + const eager = openEnv("eager", cloneBase("eager"), false); + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(eager, lazy); + const id = writeBig(eager.root); + writeBig(lazy.root); + ids.big = id; + try { + await eager.m.resumeSessions(); + await lazy.m.resumeSessions(); + const pair = { eager, lazy }; + expect(lazy.m._sessionForTest(id)!.hydrated).toBe(false); + // A notice while unloaded: the load's window still starts where a resume's did. + await same(pair, async (env) => call(env, { type: "session.set_mode", sessionId: id, mode: "guarded" })); + await same(pair, async (env) => { + const replay = await attach(env, id, PAGING); + const s = env.m._sessionForTest(id)!; + const full = await s.fullCanonicalHistory(); + const pages: DaemonMessage[] = []; + let anchor = (replay.find((f) => f.type === "scrollback.replay") as { messages: SessionMessage[] }).messages[0]?.messageId; + // Back past the buffer's floor, into the transcript on disk. + for (let i = 0; anchor && i < 400; i++) { + const page = (await call(env, { type: "scrollback.page", sessionId: id, beforeMessageId: anchor, maxBytes: 2_000_000 })) as { messages: SessionMessage[]; hasMore: boolean; source?: string }; + pages.push({ ...page, messages: page.messages.map((m) => m.messageId) } as never); + if (!page.hasMore || page.messages.length === 0) break; + anchor = page.messages[0]!.messageId; + } + return { + replay, + partial: s.historyIsPartial, + inMemory: s.historyLength, + full: full.length, + last: [s.lastTurnId, s.lastTurnRole, s.lastAssistantText?.slice(0, 20)], + pages, + }; + }); + expect(lazy.m._sessionForTest(id)!.historyIsPartial).toBe(true); + } finally { + delete ids.big; + } + }); +}); + +describe("lazy session loading: the load itself", () => { + it("concurrent first uses share one load", async () => { + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(lazy); + await lazy.m.resumeSessions(); + const reads: string[] = []; + const load = lazy.transcript.loadTranscript.bind(lazy.transcript); + lazy.transcript.loadTranscript = (sessionId, opts) => { + reads.push(sessionId); + return load(sessionId, opts); + }; + const results = await Promise.all([ + attach(lazy, ids.rich!, PAGING), + call(lazy, { type: "scrollback.page", sessionId: ids.rich, maxBytes: 4096 }), + attach(lazy, ids.rich!), + call(lazy, { type: "session.fork", sessionId: ids.rich, name: "f", isolate: false }), + ]); + expect(results[1]!.type).toBe("scrollback.page.result"); + expect(results[3]!.type).toBe("response.ok"); + // One load; the fork reads the parent's transcript itself, as it always has. + expect(reads.filter((r) => r === ids.rich).length).toBe(2); + expect(sessionOf(lazy, "rich").hydrated).toBe(true); + }); + + it("an unreadable transcript is an error for the request that needed it, and the next one loads it", async () => { + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(lazy); + await lazy.m.resumeSessions(); + const path = lazy.transcript.transcriptPath(ids.short!); + const { chmodSync } = await import("node:fs"); + chmodSync(path, 0o000); + try { + const r = await call(lazy, { type: "session.attach", sessionId: ids.short }); + expect(r.type).toBe("response.error"); + expect(sessionOf(lazy, "short").hydrated).toBe(false); + const send = await call(lazy, { type: "session.send", sessionId: ids.short, text: "hi" }); + expect(send.type).toBe("response.error"); + } finally { + chmodSync(path, 0o600); + } + const ok = await attach(lazy, ids.short!, PAGING); + expect(ok[0]!.type).toBe("response.ok"); + expect(sessionOf(lazy, "short").hydrated).toBe(true); + expect((ok[1] as { messages: unknown[] }).messages.length).toBeGreaterThan(0); + }); +}); + +describe("lazy session loading: a log that can't be read at start", () => { + it("a transcript that can't be read fails the session's resume, as an eager resume did, and leaves it as it was for the next start", async () => { + const { chmodSync } = await import("node:fs"); + const roots = { eager: cloneBase("eager"), lazy: cloneBase("lazy") }; + const paths = Object.values(roots).map((r) => join(r, "transcripts", `${ids.short}.jsonl`)); + const before = readFileSync(paths[0]!, "utf8"); + for (const p of paths) chmodSync(p, 0o000); + try { + const pair = { eager: openEnv("eager", roots.eager, false), lazy: openEnv("lazy", roots.lazy, true) }; + envs.push(pair.eager, pair.lazy); + await pair.eager.m.resumeSessions(); + await pair.lazy.m.resumeSessions(); + // Not listed as a session with no history (its seq at 0, its backend starting over): not listed. + await same(pair, async (env) => (await list(env)).map((i) => i.name)); + expect(pair.lazy.m._sessionForTest(ids.short!)).toBeUndefined(); + await same(pair, (env) => call(env, { type: "session.set_model", sessionId: ids.short, model: "sonnet" })); + for (const env of [pair.eager, pair.lazy]) await closeEnv(env); + envs.splice(0); + } finally { + for (const p of paths) chmodSync(p, 0o600); + } + for (const p of paths) expect(readFileSync(p, "utf8")).toBe(before); + // Readable again: the next start resumes it whole. + const pair = { eager: openEnv("eager", roots.eager, false), lazy: openEnv("lazy", roots.lazy, true) }; + envs.push(pair.eager, pair.lazy); + await pair.eager.m.resumeSessions(); + await pair.lazy.m.resumeSessions(); + const queried = await same(pair, async (env) => env.made.map((p) => p.hasQueried)); + expect(queried.filter((q) => !q).length).toBe(1); // the empty session's + await same(pair, async (env) => attach(env, ids.short!, PAGING)); + await same(pair, async (env) => { + await sendAndSettle(env, ids.short!, "again"); + return attach(env, ids.short!, PAGING); + }); + }); + + it("a canonical log or turn index that can't be read: the session resumes without its history, as an eager resume did, and reads it on its next use once it can", async () => { + const { chmodSync } = await import("node:fs"); + const roots = { eager: cloneBase("eager"), lazy: cloneBase("lazy") }; + const paths = Object.values(roots).map((r) => join(r, "transcripts", `${ids.short}.canonical.jsonl`)); + for (const p of paths) chmodSync(p, 0o000); + const pair = { eager: openEnv("eager", roots.eager, false), lazy: openEnv("lazy", roots.lazy, true) }; + envs.push(pair.eager, pair.lazy); + try { + await pair.eager.m.resumeSessions(); + await pair.lazy.m.resumeSessions(); + await same(pair, async (env) => { + const s = sessionOf(env, "short"); + return { names: (await list(env)).map((i) => i.name), history: s.historyLength, last: s.lastTurnId, replay: await attach(env, ids.short!, PAGING) }; + }); + } finally { + for (const p of paths) chmodSync(p, 0o600); + } + // Readable again: the next use reads it. + await same(pair, async (env) => { + await attach(env, ids.short!, PAGING); + const s = sessionOf(env, "short"); + return { history: s.historyLength, last: s.lastTurnId }; + }); + expect(sessionOf(pair.lazy, "short").historyLength).toBe(2); + }); +}); + +describe("lazy session loading: logs whose seqs don't grow along the file", () => { + /** Line i of n's seq in each shape the audits found on disk (undefined: the line has none). */ + const SHAPES: Record number | undefined> = { + // A daemon from before 2026-07-03 started every run's seqs at 0. + "seqs restarted at 0 by an older daemon": (i, n) => (i < n - 2 ? i : i - (n - 2)), + "the newest lines' seqs back down": (i, n) => (i < n - 4 ? i : i - (n - 4)), + // An import writes 1..N; the imported session's own prompt then started at 0. + "an import, then a prompt": (i, n) => (i < n - 2 ? i + 1 : i - (n - 2)), + "no seqs at all": () => undefined, + "the newest lines without a seq": (i, n) => (i < n - 2 ? i : undefined), + }; + + /** Rewrite a session's transcript with `shape`'s seqs, its lines unchanged otherwise. */ + function reshape(root: string, sessionId: string, shape: (i: number, n: number) => number | undefined): void { + const path = join(root, "transcripts", `${sessionId}.jsonl`); + const lines = readFileSync(path, "utf8").split("\n").filter((l) => l.trim()); + const out = lines.map((l, i) => { + const { seq: _seq, ...rest } = JSON.parse(l) as { seq?: number }; + const seq = shape(i, lines.length); + return JSON.stringify(seq === undefined ? rest : { seq, ...rest }); + }); + writeFileSync(path, `${out.join("\n")}\n`); + } + + /** What a resume read before lazy loading (origin/main's loader): the log's newest 24 MiB, merged, as it is now. */ + async function eagerRead(root: string, sessionId: string): Promise<{ messages: unknown[]; truncated: boolean }> { + const stats: { truncated?: boolean } = {}; + const entries = await new TranscriptStore(join(root, "transcripts")).loadTranscript(sessionId, { maxBytes: 24 * 1024 * 1024, stats }); + return { messages: entries.map((e) => e.message), truncated: stats.truncated === true }; + } + + /** A replayed message as the log holds it (the buffer stamps its own seq; a resumed tool call is reconciled). */ + const shown = (m: unknown) => { + const { seq: _seq, ...rest } = m as SessionMessage; + return { messageId: rest.messageId, role: rest.role, content: rest.content }; + }; + + for (const [label, shape] of Object.entries(SHAPES)) { + it(`${label}: every line loads, as an eager resume read it, with and without lazy loading`, async () => { + const roots = { eager: cloneBase("eager"), lazy: cloneBase("lazy") }; + for (const root of Object.values(roots)) { + reshape(root, ids.rich!, shape); + reshape(root, ids.legacy!, shape); + } + const want = await eagerRead(roots.eager, ids.rich!); + const wantLegacy = await eagerRead(roots.eager, ids.legacy!); + expect(want.messages.length).toBeGreaterThan(10); + const pair = { eager: openEnv("eager", roots.eager, false), lazy: openEnv("lazy", roots.lazy, true) }; + envs.push(pair.eager, pair.lazy); + await pair.eager.m.resumeSessions(); + await pair.lazy.m.resumeSessions(); + expect(loaded(pair.lazy, "rich")).toBe(false); + // Every backend with a conversation on disk is told to resume it, seqs or none. + expect(pair.eager.made.length).toBeGreaterThan(0); + const queried = await same(pair, async (env) => env.made.map((p) => p.hasQueried)); + // All but the empty session's. + expect(queried.filter((q) => !q).length).toBe(1); + // A notice while unloaded is appended past the log, whatever its seqs. + await same(pair, (env) => call(env, { type: "session.set_model", sessionId: ids.rich, model: "sonnet" })); + const replays = await same(pair, async (env) => { + const replay = (await attach(env, ids.rich!))[1] as { messages: SessionMessage[] }; + const s = sessionOf(env, "rich"); + return { messages: replay.messages, partial: s.historyIsPartial }; + }); + // What the log held, in its order, then the notice. + expect(replays.messages.slice(0, -1).map(shown)).toEqual(want.messages.map(shown)); + expect(replays.messages.at(-1)!.content).toContain("Model switched"); + expect(replays.partial).toBe(want.truncated); + // A session from before the canonical log rebuilds its history from every line, and persists it. + await same(pair, async (env) => { + const s = sessionOf(env, "legacy"); + const replay = (await attach(env, ids.legacy!))[1] as { messages: SessionMessage[] }; + expect(replay.messages.map(shown)).toEqual(wantLegacy.messages.map(shown)); + return { history: s.historyLength, turns: (await call(env, { type: "session.turns", sessionId: ids.legacy })) }; + }); + // A new prompt's lines continue past every seq the log holds. + await same(pair, (env) => sendAndSettle(env, ids.rich!, "after the restart")); + for (const env of [pair.eager, pair.lazy]) { + const seqs = seqsOnDisk(env, ids.rich!); + const before = seqs.slice(0, want.messages.length).filter((q) => q !== undefined); + const newest = seqs.slice(-2); + if (before.length > 0) for (const q of newest) expect(q).toBeGreaterThan(Math.max(...before)); + } + // And the next restart reads the same log from both. + const after = await Promise.all([eagerRead(roots.eager, ids.rich!), eagerRead(roots.lazy, ids.rich!)]); + expect(norm(pair.lazy, after[1].messages.map(shown))).toEqual(norm(pair.eager, after[0].messages.map(shown))); + }); + } + + it("a log past the read budget whose seqs restarted: the same window, the same partial flag", async () => { + const pair = { eager: openEnv("eager", cloneBase("eager"), false), lazy: openEnv("lazy", cloneBase("lazy"), true) }; + envs.push(pair.eager, pair.lazy); + // Seqs restarted at 0 for the newest 20 turns. + const restart = (i: number, n: number) => (i < n - 40 ? i : i - (n - 40)); + const id = writeBig(pair.eager.root, restart); + writeBig(pair.lazy.root, restart); + const want = await eagerRead(pair.eager.root, id); + expect(want.truncated).toBe(true); + ids.big = id; + try { + await pair.eager.m.resumeSessions(); + await pair.lazy.m.resumeSessions(); + await same(pair, (env) => call(env, { type: "session.set_model", sessionId: id, model: "sonnet" })); + const got = await same(pair, async (env) => { + const s = env.m._sessionForTest(id)!; + const page = (await call(env, { type: "scrollback.page", sessionId: id, maxBytes: 2_000_000 })) as { messages: SessionMessage[] }; + return { ids: page.messages.map((m) => m.messageId), partial: s.historyIsPartial }; + }); + // The newest page is what was read, in its order, up to its newest line, then the notice. + const read = want.messages.map((m) => (m as SessionMessage).messageId); + const kept = got.ids.slice(0, -1); + expect(kept.length).toBeGreaterThan(50); + const at = kept.map((k) => read.indexOf(k!)); + expect(at.every((i, n) => i >= 0 && (n === 0 || i > at[n - 1]!))).toBe(true); + expect(kept.at(-1)).toBe(read.at(-1)); + // It reaches back past where the seqs restarted (turn 1080). + expect(kept).toContain("a-1070"); + expect(got.partial).toBe(true); + } finally { + delete ids.big; + } + }); +}); + +describe("lazy session loading: messages while a load is in flight", () => { + /** The fixture resumed eagerly and lazily, every transcript read held `ms` (a large log). */ + async function slowPair(ms: number): Promise<{ eager: Env; lazy: Env }> { + const pair = await both(); + for (const env of [pair.eager, pair.lazy]) { + const load = env.transcript.loadTranscript.bind(env.transcript); + env.transcript.loadTranscript = async (sessionId, opts) => { + await Bun.sleep(ms); + return load(sessionId, opts); + }; + } + return pair; + } + + /** A turn's worth of provider sends the session's backends were handed. */ + const prompts = (env: Env) => env.made.reduce((n, p) => n + p.capturedOpts.length, 0); + + it("are handled in the order they came: an interrupt stops the prompt sent before it, a detach follows its attach", async () => { + const pair = await slowPair(50); + await same(pair, async (env) => { + const sent = env.m.handle({ type: "session.send", id: "s", sessionId: ids.rich, text: "go" } as never, AUTH, client("c", [])); + const stop = env.m.handle({ type: "session.interrupt", id: "i", sessionId: ids.rich } as never, AUTH, client("c", [])); + const responses = await Promise.all([sent, stop]); + await settle(env, ids.rich!); + return { responses, prompts: prompts(env), turns: sessionOf(env, "rich").toInfo().usage?.numTurns }; + }); + expect(prompts(pair.lazy)).toBe(0); + await same(pair, async (env) => { + const frames: DaemonMessage[] = []; + const c = client("ws", frames); + const attached = env.m.handle({ type: "session.attach", id: "a", sessionId: ids.short } as never, AUTH, c); + const detached = env.m.handle({ type: "session.detach", id: "d", sessionId: ids.short } as never, AUTH, c); + const responses = await Promise.all([attached, detached]); + return { responses: responses.map((r) => r.type), clients: sessionOf(env, "short").attachedClientCount }; + }); + expect(sessionOf(pair.lazy, "short").attachedClientCount).toBe(0); + }); + + it("an attach whose socket closed while it waited leaves no client attached", async () => { + const pair = await slowPair(50); + await same(pair, async (env) => { + const attached = env.m.handle({ type: "session.attach", id: "a", sessionId: ids.rich } as never, AUTH, client("ws1", [])); + env.m.disconnectClient("ws1"); // the socket's close handler + await attached; + return sessionOf(env, "rich").attachedClientCount; + }); + expect(sessionOf(pair.lazy, "rich").attachedClientCount).toBe(0); + // Another client of the same session is unaffected. + const frames: DaemonMessage[] = []; + await pair.lazy.m.handle({ type: "session.attach", id: "b", sessionId: ids.rich } as never, AUTH, client("ws2", frames)); + expect(sessionOf(pair.lazy, "rich").attachedClientCount).toBe(1); + }); + + it("a prompt waiting on the load is work the shutdown drain sees: no turn starts after it", async () => { + const pair = await slowPair(300); + for (const env of [pair.eager, pair.lazy]) { + const sent = env.m.handle({ type: "session.send", id: "s", sessionId: ids.rich, text: "go" } as never, AUTH, client("c", [])); + expect(sessionOf(env, "rich").preparingTurn).toBe(true); + await env.m.drain(5_000); + const drained = prompts(env); + const reply = (await sent) as { type: string; error?: string }; + if (env === pair.eager) { + // It was already preparing when the drain began: accepted, then stopped before it went out. + expect(reply.type).toBe("response.ok"); + } else { + // Still waiting on the load when the drain began: refused, saying so (no turn starts once a shutdown has). + expect(reply.type).toBe("response.error"); + expect(reply.error).toContain("shutting down"); + } + await Bun.sleep(100); + expect(prompts(env)).toBe(drained); + expect(sessionOf(env, "rich").preparingTurn).toBe(false); + expect(sessionOf(env, "rich").status).toBe("idle"); + } + expect(prompts(pair.lazy)).toBe(0); + }); + + it("the drain waits for a message waiting on a load", async () => { + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(lazy); + await lazy.m.resumeSessions(); + const load = lazy.transcript.loadTranscript.bind(lazy.transcript); + lazy.transcript.loadTranscript = async (sessionId, opts) => { + await Bun.sleep(300); + return load(sessionId, opts); + }; + let answered = false; + const attached = lazy.m.handle({ type: "session.attach", id: "a", sessionId: ids.rich } as never, AUTH, client("c", [])).then((r) => { + answered = true; + return r; + }); + await lazy.m.drain(5_000); + expect(answered).toBe(true); + expect((await attached).type).toBe("response.ok"); + }); + + it("a request its handler refuses on scope loads nothing", async () => { + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(lazy); + await lazy.m.resumeSessions(); + const asked: Array<[string, Record]> = [ + ["session.attach", {}], + ["scrollback.page", { maxBytes: 4096 }], + ["session.send", { text: "hi" }], + ["session.part_action", { messageId: "m", action: "a" }], + ["session.rotate", {}], + ["session.rewind", { turnId: "t" }], + ["session.approve", { approvalId: "x", decision: "allow" }], + ["session.fork", { name: "f" }], + ["session.set_provider", { providerId: "codex" }], + ]; + for (const scopes of [[], ["session:watch"]] as const) { + const auth: AuthContext = { ...AUTH, scopes: [...scopes] as AuthContext["scopes"] }; + for (const [type, rest] of asked) { + if (type === "session.attach" || type === "scrollback.page") { + if (scopes.length > 0) continue; // a watcher may read the history + } + const r = await lazy.m.handle({ type, id: "r", sessionId: ids.rich, ...rest } as never, auth, client("c", [])); + expect([type, r.type, (r as { error?: string }).error?.startsWith("Missing scope")]).toEqual([type, "response.error", true]); + expect(sessionOf(lazy, "rich").hydrated).toBe(false); + } + } + // A watcher's attach does load it. + const watcher: AuthContext = { ...AUTH, scopes: ["session:watch"] as AuthContext["scopes"] }; + const r = await lazy.m.handle({ type: "session.attach", id: "w", sessionId: ids.rich } as never, watcher, client("w", [])); + expect(r.type).toBe("response.ok"); + expect(sessionOf(lazy, "rich").hydrated).toBe(true); + }); + + it("a request its handler refuses for another reason, or that never reads the history, loads nothing", async () => { + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(lazy); + await lazy.m.resumeSessions(); + const delegated: AuthContext = { ...AUTH, delegationDepth: 1 }; + const ask = async (msg: Record, auth: AuthContext = AUTH) => { + const r = await lazy.m.handle({ id: "r", sessionId: ids.rich, ...msg } as never, auth, client("c", [])); + expect(sessionOf(lazy, "rich").hydrated).toBe(false); + return r; + }; + // mcp.adopt only reads the session's workdir; compare.keep its status. + expect(((await ask({ type: "mcp.adopt", server: "srv" }, delegated)) as { error?: string }).error).toContain("delegated token"); + expect(((await ask({ type: "compare.keep", compareId: "nope" })) as { error?: string }).error).toBe("Comparison not found"); + // A running loop's worker: what its guard refuses never reaches the history. + const loops = lazy.m.loops!; + const runOfWorker = loops.runOfWorker.bind(loops); + loops.runOfWorker = (id) => (id === ids.rich ? "loop_test" : runOfWorker(id)); + try { + for (const msg of [ + { type: "session.rotate" }, + { type: "session.rewind", turnId: "t" }, + { type: "session.set_provider", providerId: "codex" }, + ]) { + expect(((await ask(msg)) as { error?: string }).error).toContain("running loop's worker"); + } + const approve = await ask({ type: "session.approve", approvalId: "x", decision: "allow" }, delegated); + expect((approve as { error?: string }).error).toContain("loop worker"); + } finally { + loops.runOfWorker = runOfWorker; + } + }); +}); + +describe("lazy session loading: a canonical log that can't be read", () => { + /** The canonical log's lines on disk. */ + const canonicalLines = (env: Env, sessionId: string) => + readFileSync(env.transcript.canonicalPath(sessionId), "utf8").split("\n").filter((l) => l.trim()).length; + const newestTurn = async (env: Env, sessionId: string) => + ((await call(env, { type: "session.turns", sessionId })) as { turns: Array<{ turnId: string }> }).turns.at(-1)!.turnId; + + it("at the first use: that request fails, nothing is lost, and the next use loads the whole conversation", async () => { + const pair = await both(); + const { chmodSync } = await import("node:fs"); + const path = pair.lazy.transcript.canonicalPath(ids.rich!); + chmodSync(path, 0o000); + try { + const r = await call(pair.lazy, { type: "session.attach", sessionId: ids.rich }); + expect(r.type).toBe("response.error"); + expect(sessionOf(pair.lazy, "rich").hydrated).toBe(false); + const send = await call(pair.lazy, { type: "session.send", sessionId: ids.rich, text: "hi" }); + expect(send.type).toBe("response.error"); + } finally { + chmodSync(path, 0o600); + } + expect(pair.lazy.made.reduce((n, p) => n + p.capturedOpts.length, 0)).toBe(0); + await same(pair, async (env) => { + const replay = await attach(env, ids.rich!, PAGING); + const s = sessionOf(env, "rich"); + return { replay, history: s.historyLength, last: s.lastTurnId, text: s.lastAssistantText }; + }); + expect(sessionOf(pair.lazy, "rich").historyLength).toBe(12); + // Going back the newest turn keeps the rest on disk. + await same(pair, async (env) => { + const r = await call(env, { type: "session.rewind", sessionId: ids.rich, turnId: await newestTurn(env, ids.rich!) }); + expect(r.type).toBe("session.rewind.result"); + await settle(env, ids.rich!); + return canonicalLines(env, ids.rich!); + }); + expect(canonicalLines(pair.lazy, ids.rich!)).toBe(10); + }); + + it("a synchronous load that can't read it throws, and leaves the session unloaded for the next use", async () => { + const lazy = openEnv("lazy", cloneBase("lazy"), true); + envs.push(lazy); + await lazy.m.resumeSessions(); + const { chmodSync } = await import("node:fs"); + const path = lazy.transcript.canonicalPath(ids.rich!); + const s = sessionOf(lazy, "rich"); + chmodSync(path, 0o000); + try { + expect(() => s.canonicalHistory).toThrow(); + expect(s.hydrated).toBe(false); + } finally { + chmodSync(path, 0o600); + } + expect(s.canonicalHistory.length).toBe(12); + expect(s.hydrated).toBe(true); + }); + + it("at resume: the session runs without it, as before, never writes over it, and reads it again on its next use", async () => { + const pair = { eager: openEnv("eager", cloneBase("eager"), false), lazy: openEnv("lazy", cloneBase("lazy"), true) }; + envs.push(pair.eager, pair.lazy); + let failing = true; + for (const env of [pair.eager, pair.lazy]) { + const load = env.transcript.loadCanonical.bind(env.transcript); + env.transcript.loadCanonical = (sessionId, opts) => { + if (failing && sessionId === ids.rich) return Promise.reject(new Error("EIO: i/o error, read")); + return load(sessionId, opts); + }; + } + await pair.eager.m.resumeSessions(); + await pair.lazy.m.resumeSessions(); + const env = pair.eager; + expect(sessionOf(env, "rich").historyLength).toBe(0); + const turnId = await newestTurn(env, ids.rich!); + // Going back or forking would rewrite the log from what's in memory: refused, the log untouched. + const rewind = await call(env, { type: "session.rewind", sessionId: ids.rich, turnId }); + expect([rewind.type, (rewind as { error?: string }).error]).toEqual(["response.error", expect.stringContaining("couldn't be read")]); + const preview = await call(env, { type: "session.rewind", sessionId: ids.rich, turnId, dryRun: true }); + expect(preview.type).toBe("response.error"); + for (const fork of [{}, { afterTurnId: turnId }]) { + const r = await call(env, { type: "session.fork", sessionId: ids.rich, name: "f", isolate: false, ...fork }); + expect([r.type, (r as { error?: string }).error]).toEqual(["response.error", expect.stringContaining("couldn't be read")]); + } + expect(canonicalLines(env, ids.rich!)).toBe(12); + // A turn still runs, and is appended to the log. + expect((await sendAndSettle(env, ids.rich!, "more")).type).toBe("response.ok"); + expect(canonicalLines(env, ids.rich!)).toBe(14); + expect(sessionOf(env, "rich").historyLength).toBe(2); + // Readable again: the next use reads it, the turn since included. + failing = false; + await attach(env, ids.rich!, PAGING); + expect(sessionOf(env, "rich").historyLength).toBe(14); + const r = await call(env, { type: "session.rewind", sessionId: ids.rich, turnId: await newestTurn(env, ids.rich!) }); + expect(r.type).toBe("session.rewind.result"); + await settle(env, ids.rich!); + expect(canonicalLines(env, ids.rich!)).toBe(12); + // The lazy daemon read it at the first use, failing or not. + expect(sessionOf(pair.lazy, "rich").hydrated).toBe(false); + await attach(pair.lazy, ids.rich!, PAGING); + expect(sessionOf(pair.lazy, "rich").historyLength).toBe(12); + }); + + it("at resume: a rotation empties it, as it always does, and nothing is read back after it", async () => { + const env = openEnv("eager", cloneBase("eager"), false); + envs.push(env); + let failing = true; + const load = env.transcript.loadCanonical.bind(env.transcript); + env.transcript.loadCanonical = (sessionId, opts) => + failing && sessionId === ids.rich ? Promise.reject(new Error("EIO: i/o error, read")) : load(sessionId, opts); + await env.m.resumeSessions(); + // Past the turns a rotation needs, each appended to the log. + for (const text of ["one", "two", "three"]) await sendAndSettle(env, ids.rich!, text); + await env.transcript.flush(); + expect(canonicalLines(env, ids.rich!)).toBe(18); + expect((await call(env, { type: "session.rotate", sessionId: ids.rich })).type).toBe("response.ok"); + await settle(env, ids.rich!); + expect(sessionOf(env, "rich").toInfo().rotation?.count).toBe(1); + await env.transcript.flush(); + expect(canonicalLines(env, ids.rich!)).toBe(0); + failing = false; + await attach(env, ids.rich!, PAGING); + expect(sessionOf(env, "rich").historyLength).toBe(0); + expect((await sendAndSettle(env, ids.rich!, "after")).type).toBe("response.ok"); + expect(sessionOf(env, "rich").historyLength).toBe(2); + expect(canonicalLines(env, ids.rich!)).toBe(2); + }); +}); diff --git a/src/tests/provider-opencode.integration.test.ts b/src/tests/provider-opencode.integration.test.ts index 87f1771b..7e0ee71a 100644 --- a/src/tests/provider-opencode.integration.test.ts +++ b/src/tests/provider-opencode.integration.test.ts @@ -641,6 +641,55 @@ describe("opencode (real binary)", () => { await p.teardown(); }, 180_000); + maybeScripted("I71 an interrupted turn ends, its spend read, within the settle bound the provider declares for a daemon's drain", async () => { + const p = provider(); + const took: number[] = []; + // Mid-reply, at a tool's gate, and before the prompt has started: where a shutdown finds a turn. + for (let i = 0; i < 9; i++) { + let at = 0; + let release = () => {}; + const parked = new Promise((r) => { + release = r; + }); + const atGate = i % 3 === 1; + const atStart = i % 3 === 2; + const run = p.runTurn( + atGate + ? opts('[[tool:shell {"command":"echo held","description":"Held at the gate"}]]', { + canUseTool: async () => { + release(); + await Bun.sleep(60_000); + return { behavior: "deny", message: "no" }; + }, + }) + : opts("[[slow:300]] [[text:a b c d e f g h i j k l m n o p q r s t]]"), + ); + if (atStart) { + at = Date.now(); + void run.interrupt(); + } else if (atGate) { + void parked.then(() => { + at = Date.now(); + void run.interrupt(); + }); + } + for await (const e of run.events) { + if (!atGate && !atStart && !at && e.type === "text_delta") { + at = Date.now(); + void run.interrupt(); + } + if (e.type === "turn_done" || e.type === "error") { + if (e.type === "turn_done") expect(e.result.stopReason).toBe("interrupted"); + break; + } + } + took.push(Date.now() - at); + } + console.log(`[opencode integration] I71: interrupt to turn end ${took.join(", ")} ms (bound ${p.interruptSettleMs} ms)`); + for (const ms of took) expect(ms).toBeLessThan(p.interruptSettleMs); + await p.teardown(); + }, 180_000); + maybeScripted("I12 Code Mode is hidden: a script can't run, so its fetch can't get past the gate", async () => { // On 2.2.2 a Code Mode script's fetch runs without asking anything. The // session rules deny `execute`, which takes the tool away on 2.2.x. diff --git a/src/tests/provider-pi.test.ts b/src/tests/provider-pi.test.ts index 74bec29e..adf083aa 100644 --- a/src/tests/provider-pi.test.ts +++ b/src/tests/provider-pi.test.ts @@ -375,6 +375,22 @@ describe("PiProvider", () => { } }); + it("T12: an interrupt before pi is up stops the turn: the prompt never goes out, the history seed waits for the next", async () => { + const p = makeProvider(); + p.seedFromHistory([{ role: "user", content: "earlier question" }]); + // pi's process is still starting: there is nothing to send an abort to. + const run = p.runTurn(turnOpts("hello")); + await run.interrupt(); + const events = await collect(run.events); + expect(events.some((e) => e.type === "text_delta")).toBe(false); + expect(events.find((e) => e.type === "turn_done")).toMatchObject({ result: { stopReason: "interrupted", inputTokens: 0, totalCostUsd: 0 } }); + + const next = await collect(p.runTurn(turnOpts("echo-prompt")).events); + const done = next.find((e) => e.type === "text_done"); + expect(done?.type === "text_done" && done.content).toContain("earlier question"); + expect(next.find((e) => e.type === "turn_done")).toMatchObject({ result: { inputTokens: 100 } }); + }); + it("T10: second turn reports usage as a DELTA, not the cumulative total", async () => { const p = makeProvider(); await collect(p.runTurn(turnOpts("hello")).events); diff --git a/src/tests/routine-manager.test.ts b/src/tests/routine-manager.test.ts index 23471c55..6a71af98 100644 --- a/src/tests/routine-manager.test.ts +++ b/src/tests/routine-manager.test.ts @@ -12,11 +12,13 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CodeoidConfig } from "../config.js"; import { HookBus } from "../daemon/hooks/bus.js"; -import type { ProviderEvent } from "../daemon/providers/interface.js"; +import { AsyncQueue } from "../daemon/async-queue.js"; +import type { ProviderEvent, TurnOpts, TurnRun } from "../daemon/providers/interface.js"; import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; import { ProviderRegistry } from "../daemon/providers/registry.js"; import { handleRoutineFire } from "../daemon/routines/http.js"; import { SessionManager } from "../daemon/session-manager.js"; +import { DRAIN_SETTLE_MS } from "../daemon/session.js"; import { readSetupRecipe } from "../daemon/workspace-setup.js"; import { Store } from "../daemon/store.js"; import { TranscriptStore } from "../daemon/transcript.js"; @@ -83,6 +85,38 @@ let nextScript: ProviderEvent[][]; let nextStall = false; let hookFile: string; let registry: ProviderRegistry; +/** Set: the next backend works until interrupted, its result (spend included) following that many ms after. */ +let nextLateResult: number | null = null; + +/** Works until interrupted; acknowledges at once and ends the turn, $0.42 spent, `late` ms after. */ +class LateResultProvider extends MockSessionProvider { + constructor(private readonly late: number) { + super("claude"); + } + + override runTurn(opts: TurnOpts): TurnRun { + this.capturedOpts.push(opts); + const queue = new AsyncQueue(); + void Promise.resolve().then(() => queue.push({ type: "text_delta", content: "working" } as ProviderEvent)); + return { + events: queue, + interrupt: async () => { + setTimeout(() => { + try { + queue.push({ + type: "turn_done", + result: { providerId: "claude", model: "mock", inputTokens: 1000, outputTokens: 200, cacheReadTokens: 0, cacheCreationTokens: 0, totalCostUsd: 0.42, costSource: "backend", durationMs: 1, isError: true, errorMessage: "interrupted" }, + } as ProviderEvent); + } catch { + // closed + } + queue.close(); + }, this.late); + }, + endTurn: () => queue.close(), + }; + } +} const CLIENT = (auth: AuthContext) => ({ id: `c-${auth.sub}`, auth, send: () => {} }); const call = (msg: Record, auth: AuthContext = OWNER): Promise => @@ -113,13 +147,14 @@ beforeEach(() => { providers = []; nextScript = [reply("Found 2 stale PRs: #12, #15.")]; nextStall = false; + nextLateResult = null; hookFile = join(tmp, "hook.json"); registry = new ProviderRegistry("claude"); registry.register({ id: "claude", displayName: "claude", create: () => { - const p = new MockSessionProvider("claude", nextScript.map((t) => [...t]), { stall: nextStall }); + const p = nextLateResult !== null ? new LateResultProvider(nextLateResult) : new MockSessionProvider("claude", nextScript.map((t) => [...t]), { stall: nextStall }); providers.push(p); return p; }, @@ -293,6 +328,50 @@ describe("routine lifecycle through the protocol", () => { }); }); +describe("a routine run at shutdown", () => { + /** A routine run mid-turn; then the daemon's order: routines stop, the dispatcher stops, sessions drain, the store closes. */ + async function shutDownMidRun(late: number) { + nextLateResult = late; + await create({ name: "long" }); + const fired = (await call({ type: "routine.run", routine: "long" })) as RoutineRunResultMsg; + const svc = manager.routines!; + let sessionId: string | null = null; + for (let i = 0; i < 200; i++) { + sessionId = svc.store.getRun(fired.run.id)?.sessionId ?? null; + if (sessionId && manager._sessionForTest(sessionId)?.status === "thinking") break; + await Bun.sleep(10); + } + expect(manager._sessionForTest(sessionId!)?.status).toBe("thinking"); + const started = performance.now(); + manager.stopRoutines(); + manager.stopDispatcher(); + await manager.drain(10_000); + const took = performance.now() - started; + // What the store holds now is what it keeps. + const row = svc.store.getRun(fired.run.id)!; + return { row, took }; + } + + // 1300 and 1800: past the ~1.1 s a run used to wait, within the 2 s drain waits for the turn. + for (const late of [0, 100, 300, 1300, 1800]) { + it(`records its end and its spend before the store closes (the turn's result ${late} ms after the interrupt)`, async () => { + const { row, took } = await shutDownMidRun(late); + expect(row.status).toBe("interrupted"); + expect(row.costUsd).toBeCloseTo(0.42); + expect(row.costUnknown).toBeFalsy(); + expect(took).toBeLessThan(late + 1_000); + }); + } + + it("records its end at once, the cost unknown, when drain gives up on its turn", async () => { + // Past the backend's settle bound (DRAIN_SETTLE_MS): the turn is left, the run isn't. + const { row, took } = await shutDownMidRun(DRAIN_SETTLE_MS + 2_000); + expect(row.status).toBe("interrupted"); + expect(row.costUnknown).toBe(true); + expect(took).toBeLessThan(DRAIN_SETTLE_MS + 1_000); + }); +}); + describe("authority", () => { it("no scope of its own: anyone who can create and prompt sessions can keep routines", async () => { // The operator preset can, as the operator themselves... diff --git a/src/tests/sandbox-dirs-manager.test.ts b/src/tests/sandbox-dirs-manager.test.ts index 61a53d76..d7052b02 100644 --- a/src/tests/sandbox-dirs-manager.test.ts +++ b/src/tests/sandbox-dirs-manager.test.ts @@ -183,6 +183,8 @@ describe("project roots and added directories through the SessionManager", () => expect(await m.handle({ type: "session.ui_response", id: "u", sessionId: id, requestId, confirmed: true } as never, AUTH, CLIENT)).toMatchObject({ type: "response.ok" }); await until(() => audits.some((a) => a.action === "session.sandbox_dir_added")); expect(audits.find((a) => a.action === "session.sandbox_dir_added")?.detail).toBe(`${sibling} by=owner`); + // The audit row is written as the setting changes; the session's info follows once its sandbox is re-resolved. + await until(() => (m.findByName(info?.name as string, AUTH)?.toInfo()?.sandbox?.addedDirs?.length ?? 0) > 0); const after = m.findByName(info?.name as string, AUTH)?.toInfo(); expect(after?.sandbox?.addedDirs).toEqual([sibling]); expect(sandboxSummary(after?.sandbox)).toContain(`also writes ${sibling}`); diff --git a/src/tests/sandbox-manager.test.ts b/src/tests/sandbox-manager.test.ts index e7d35e1d..9f995152 100644 --- a/src/tests/sandbox-manager.test.ts +++ b/src/tests/sandbox-manager.test.ts @@ -367,6 +367,8 @@ describe("Docker access (#448 follow-up): an opt-in widening, the owner's own", expect(await m.handle({ type: "session.ui_response", id: "u", sessionId: id, requestId, confirmed: true } as never, AUTH, CLIENT)).toMatchObject({ type: "response.ok" }); for (let i = 0; i < 100 && !audits.some((a) => a.action === "session.set_sandbox"); i++) await Bun.sleep(10); expect(audits.filter((a) => a.action === "session.set_sandbox").at(-1)?.detail).toContain("→ auto, network on, Docker access"); + // The session's info follows the audited change once its sandbox is re-resolved. + for (let i = 0; i < 100 && m.findByName(info?.name as string, AUTH)?.toInfo().sandbox?.dockerSocket !== true; i++) await Bun.sleep(10); expect(m.findByName(info?.name as string, AUTH)?.toInfo().sandbox?.dockerSocket).toBe(true); // Off again: narrowing, applied without asking, by a delegated token too. const narrowed = await m.handle({ type: "session.set_sandbox", id: "x", sessionId: id, sandbox: { dockerSocket: false } }, DELEGATED, { ...CLIENT, auth: DELEGATED }); diff --git a/src/tests/transcript.test.ts b/src/tests/transcript.test.ts index 8a032b37..a3671b75 100644 --- a/src/tests/transcript.test.ts +++ b/src/tests/transcript.test.ts @@ -352,3 +352,43 @@ describe("TranscriptStore — rotation, bounded load, output cap (#85)", () => { expect(await Bun.file(`${small.transcriptPath("gone")}.2`).exists()).toBe(false); }); }); + +describe("TranscriptStore — the log as it was (lazy session loading)", () => { + test("a read as of a tail's marks leaves out what was appended since, whatever its seqs, rotated or not", async () => { + const small = new TranscriptStore(tmpDir, { segmentMaxBytes: 1500, maxRotatedSegments: 3 }); + const ids: string[] = []; + // Seqs that go back down, as an older daemon's restarts left them. + for (let i = 0; i < 6; i++) { + const msg = makeMsg(`before ${i} ${"x".repeat(120)}`); + ids.push(msg.messageId); + await small.append("asof", msg, i < 4 ? i : i - 4); + } + await small.flush(); + const want = (await small.loadTranscript("asof")).map((e) => (e.message as SessionMessage).messageId); + expect(want).toEqual(ids); + const tail = await small.transcriptTail("asof"); + expect(tail).toMatchObject({ nextSeq: 2, hasEntries: true }); + // Appended since, at seqs the log already holds, and enough to rotate the files. + for (let i = 0; i < 4; i++) await small.append("asof", makeMsg(`after ${i} ${"y".repeat(200)}`), 2 + i); + await small.flush(); + expect(await Bun.file(`${small.transcriptPath("asof")}.2`).exists()).toBe(true); + const got = await small.loadTranscript("asof", { asOf: tail.files }); + expect(got.map((e) => (e.message as SessionMessage).messageId)).toEqual(want); + const gotSync = small.loadTranscriptSync("asof", { asOf: tail.files }); + expect(gotSync.map((e) => (e.message as SessionMessage).messageId)).toEqual(want); + }); + + test("a log with entries but no seqs has entries; an unreadable one is an error, never an empty log", async () => { + const { appendFileSync, chmodSync } = await import("node:fs"); + const path = store.transcriptPath("noseq"); + appendFileSync(path, `${JSON.stringify({ timestamp: "t", message: makeMsg("no seq") })}\n`); + expect(await store.transcriptTail("noseq")).toMatchObject({ nextSeq: 0, hasEntries: true }); + chmodSync(path, 0o000); + try { + await expect(store.transcriptTail("noseq")).rejects.toThrow(); + } finally { + chmodSync(path, 0o600); + } + expect(await store.transcriptTail("missing")).toEqual({ nextSeq: 0, hasEntries: false, files: [] }); + }); +}); diff --git a/src/tests/turns.test.ts b/src/tests/turns.test.ts index 4d33eaa8..4f1552ff 100644 --- a/src/tests/turns.test.ts +++ b/src/tests/turns.test.ts @@ -8,17 +8,23 @@ */ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { Database } from "bun:sqlite"; import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CodeoidConfig } from "../config.js"; +import { AsyncQueue } from "../daemon/async-queue.js"; import { canonicalFromTranscript } from "../daemon/canonical-restore.js"; +import { HookBus } from "../daemon/hooks/bus.js"; import { listCheckpoints, shadowRepoPath } from "../daemon/checkpoints.js"; -import { SendStoppedError } from "../daemon/session.js"; -import type { ProviderEvent } from "../daemon/providers/interface.js"; +import { DaemonStoppingError, SendStoppedError } from "../daemon/session.js"; +import type { ProviderEvent, TurnOpts, TurnRun } from "../daemon/providers/interface.js"; import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; -import { SessionManager } from "../daemon/session-manager.js"; +import { LocalSandboxHandle } from "../daemon/sandbox/local-handle.js"; +import { SandboxRegistry } from "../daemon/sandbox/registry.js"; +import type { SandboxProvider, SandboxSpec } from "../daemon/sandbox/types.js"; +import { SessionManager, TURN_END_SETTLE_MS } from "../daemon/session-manager.js"; import { Store } from "../daemon/store.js"; import { TranscriptStore } from "../daemon/transcript.js"; import { ALL_SCOPES } from "../protocol/scopes.js"; @@ -366,6 +372,464 @@ describe("a Stop while a send prepares its turn", () => { expect(providers.at(-1)!.capturedOpts).toHaveLength(0); expect(s.status).toBe("idle"); }); + + it("drain() returns as soon as the work it stopped has settled, not after a fixed half second", async () => { + const m = newManager(); + const id = await create(m); + const s = m._sessionForTest(id)!; + const sending = s.send("go", AUTH).catch((e) => e); + // When the work itself has settled: the stopped send, then the writes it made. + const settled = sending.then(() => transcript.flush()).then(() => performance.now()); + await m.drain(2_000); + const returned = performance.now(); + // drain used to wait 500 ms before looking again, however soon the stop + // settled; it looks again within 50 ms now. Measured from the work's own + // end, so a slow stop under load doesn't count against it. + expect(returned - (await settled)).toBeLessThan(300); + expect(await sending).toBeInstanceOf(SendStoppedError); + }); +}); + +/** + * A backend whose interrupt is acknowledged at once while the turn's result + * follows `late` ms after it, as real ones do: Claude's result after the + * interrupt receipt, Codex's turn/completed after turn/interrupt, opencode's + * feed or read-back after its interrupt call. + */ +class LateResultProvider extends MockSessionProvider { + /** `clean`: the stopped turn's result is no error (the session ends idle, where a wake is delivered). */ + constructor( + private readonly late: number, + private readonly clean = false, + ) { + super("claude"); + } + + override runTurn(opts: TurnOpts): TurnRun { + this.capturedOpts.push(opts); + const queue = new AsyncQueue(); + void Promise.resolve().then(() => queue.push({ type: "text_delta", content: "working on it" } as ProviderEvent)); + let interrupted = false; + return { + events: queue, + interrupt: async () => { + if (interrupted) return; + interrupted = true; + setTimeout(() => { + try { + queue.push({ type: "text_done", content: "working on it, cut short" } as ProviderEvent); + queue.push({ + type: "turn_done", + result: { + providerId: "mock", + model: "mock", + inputTokens: 1000, + outputTokens: 200, + cacheReadTokens: 0, + cacheCreationTokens: 0, + totalCostUsd: 0.42, + costSource: "backend", + durationMs: 1, + ...(this.clean ? {} : { isError: true, errorMessage: "interrupted" }), + }, + } as ProviderEvent); + } catch { + // closed + } + queue.close(); + }, this.late); + }, + endTurn: () => queue.close(), + }; + } +} + +describe("drain() at shutdown", () => { + for (const late of [0, 20, 100, 300, 600]) { + it(`records an interrupted turn whose result comes ${late} ms after the interrupt, before it returns`, async () => { + const m = new SessionManager(store, transcript, undefined, undefined, undefined, { + config: mkConfig(tmp), + _testProviderFactory: () => new LateResultProvider(late), + }); + const id = await create(m); + const s = m._sessionForTest(id)!; + const sending = s.send("go", AUTH).catch((e) => e); + for (let i = 0; i < 200 && s.status !== "thinking"; i++) await Bun.sleep(5); + expect(s.status).toBe("thinking"); + + const errors: string[] = []; + const consoleError = console.error; + console.error = (...args: unknown[]) => { + errors.push(args.map((a) => (a instanceof Error ? a.message : String(a))).join(" ")); + }; + try { + const started = performance.now(); + await m.drain(10_000); + const took = performance.now() - started; + // What is on disk now is what the daemon leaves: the store closes next, then the process exits. + const persisted = store.getSession(id)?.status; + store.close(); + const canonical = readFileSync(transcript.canonicalPath(id), "utf8"); + const lines = readFileSync(join(tmp, "transcripts", `${id}.jsonl`), "utf8"); + const usage = s.toInfo().usage!; + + expect(s.turnInFlight).toBe(false); + expect(canonical.split("\n").filter((l) => l.includes('"assistant"'))).toHaveLength(1); + expect(lines).toContain("working on it, cut short"); + expect(usage.numTurns).toBe(1); + expect(usage.totalCostUsd).toBeCloseTo(0.42); + expect(persisted).toBe(s.status); + // As soon as the turn has ended, not after a fixed wait. + expect(took).toBeLessThan(late + 400); + + await Bun.sleep(50); + expect(errors.filter((e) => /closed database|persist failed/i.test(e))).toEqual([]); + } finally { + console.error = consoleError; + } + await sending; + }); + } + + /** A manager on `provider`, its session mid-turn. */ + async function midTurn(provider: () => MockSessionProvider): Promise<{ m: SessionManager; id: string; sending: Promise }> { + const m = new SessionManager(store, transcript, undefined, undefined, undefined, { + config: mkConfig(tmp), + _testProviderFactory: () => { + const p = provider(); + providers.push(p); + return p; + }, + }); + const id = await create(m); + const s = m._sessionForTest(id)!; + const sending = s.send("go", AUTH).catch((e) => e); + for (let i = 0; i < 200 && s.status !== "thinking"; i++) await Bun.sleep(5); + expect(s.status).toBe("thinking"); + return { m, id, sending }; + } + + /** Errors logged while `fn` runs, and a moment after (a write that lands once the store closed). */ + async function errorsDuring(fn: () => Promise): Promise { + const errors: string[] = []; + const consoleError = console.error; + console.error = (...args: unknown[]) => { + errors.push(args.map((a) => (a instanceof Error ? a.message : String(a))).join(" ")); + }; + try { + await fn(); + await Bun.sleep(50); + } finally { + console.error = consoleError; + } + return errors; + } + + it("waits for a turn stopped just before shutdown began, whose end is still on its way", async () => { + const { m, id, sending } = await midTurn(() => new LateResultProvider(300)); + const s = m._sessionForTest(id)!; + // The owner's Stop: acknowledged at once, so the session reads idle, its result 300 ms away. + await s.interrupt(AUTH); + expect(s.status).toBe("idle"); + expect(s.turnInFlight).toBe(true); + + const errors = await errorsDuring(async () => { + const started = performance.now(); + await m.drain(10_000); + const took = performance.now() - started; + store.close(); + const canonical = readFileSync(transcript.canonicalPath(id), "utf8"); + expect(s.turnInFlight).toBe(false); + expect(canonical.split("\n").filter((l) => l.includes('"assistant"'))).toHaveLength(1); + expect(s.toInfo().usage!.totalCostUsd).toBeCloseTo(0.42); + expect(took).toBeGreaterThanOrEqual(250); + expect(took).toBeLessThan(300 + 1_000); + }); + expect(errors.filter((e) => /closed database|persist failed/i.test(e))).toEqual([]); + await sending; + }); + + it("interrupts a turn the backend starts on its own after drain's interrupt, and waits for it", async () => { + // The interrupted turn ends 50 ms after the interrupt; as the session lets + // go of it, the backend starts one of its own (the Claude CLI answering a + // background agent that settled). + class SelfStarting extends LateResultProvider { + own = 0; + override runTurn(opts: TurnOpts): TurnRun { + const run = super.runTurn(opts); + return { + ...run, + endTurn: () => { + run.endTurn?.(); + if (this.own++ > 0) return; + queueMicrotask(() => this.startOwnTurn([{ type: "text_delta", content: "carrying on" } as ProviderEvent])); + }, + }; + } + } + const { m, id, sending } = await midTurn(() => new SelfStarting(50)); + const s = m._sessionForTest(id)!; + const provider = providers.at(-1) as SelfStarting; + await m.drain(10_000); + expect(provider.own).toBe(1); + // Stopped as it started, and its end recorded before drain returned. + expect(provider.ownTurnInterrupts).toBe(1); + expect(s.status).toBe("idle"); + expect(s.turnInFlight).toBe(false); + await sending; + }); + + it("starts no background wake once drain has begun", async () => { + // Background work settles as the interrupted turn ends: the session would wake itself. + class SettlesOnStop extends LateResultProvider { + override runTurn(opts: TurnOpts): TurnRun { + const run = super.runTurn(opts); + return { + ...run, + interrupt: async () => { + this.onSessionEvent?.({ type: "background_task_settled", taskId: "task-1", status: "completed", summary: "done" } as never); + await run.interrupt(); + }, + }; + } + } + const { m, id, sending } = await midTurn(() => new SettlesOnStop(20, true)); + const s = m._sessionForTest(id)!; + await m.drain(10_000); + await Bun.sleep(100); + // Only the prompted turn ever reached the backend. + expect(providers.at(-1)!.capturedOpts).toHaveLength(1); + expect(s.status).toBe("idle"); + expect(s.preparingTurn).toBe(false); + expect(s.turnInFlight).toBe(false); + await sending; + }); + + it("refuses a send once drain has begun", async () => { + const m = newManager(); + const id = await create(m); + const s = m._sessionForTest(id)!; + await m.drain(2_000); + await expect(s.send("late", AUTH)).rejects.toBeInstanceOf(DaemonStoppingError); + const r = await m.handle({ type: "session.send", id: "s1", sessionId: id, text: "late", clientMsgId: "cm-late" } as never, AUTH, { + id: "cli", + auth: AUTH, + send: () => {}, + }); + expect(r.type).toBe("response.error"); + // Refused, so its id isn't marked as seen: this is its first mark. + expect(s.markClientMsgSeen("cm-late")).toBe(false); + expect(providers.at(-1)!.capturedOpts).toHaveLength(0); + }); + + it("waits for a turn that never ends only its backend's settle bound, for every session at once", async () => { + // Acknowledges the interrupt, or never answers it, and never ends the turn. + class Wedged extends MockSessionProvider { + readonly interruptSettleMs = 300; + constructor(private readonly ack: boolean) { + super("claude"); + } + override runTurn(opts: TurnOpts): TurnRun { + this.capturedOpts.push(opts); + const queue = new AsyncQueue(); + void Promise.resolve().then(() => queue.push({ type: "text_delta", content: "working" } as ProviderEvent)); + return { + events: queue, + interrupt: () => (this.ack ? Promise.resolve() : new Promise(() => {})), + endTurn: () => queue.close(), + }; + } + } + const m = new SessionManager(store, transcript, undefined, undefined, undefined, { + config: mkConfig(tmp), + // Every other one never answers the interrupt. + _testProviderFactory: () => new Wedged(providers.push(new MockSessionProvider("claude")) % 2 === 0), + }); + managers.push(m); + const sends: Promise[] = []; + const ids: string[] = []; + for (let i = 0; i < 4; i++) { + const id = await create(m); + ids.push(id); + const s = m._sessionForTest(id)!; + sends.push(s.send("go", AUTH).catch((e) => e)); + for (let j = 0; j < 200 && s.status !== "thinking"; j++) await Bun.sleep(5); + } + const warned: string[] = []; + const consoleWarn = console.warn; + console.warn = (...args: unknown[]) => { + warned.push(args.map(String).join(" ")); + }; + let took = 0; + try { + const started = performance.now(); + await m.drain(10_000); + took = performance.now() - started; + } finally { + console.warn = consoleWarn; + } + // One bound, not four in a row, and nowhere near the deadline. + expect(took).toBeGreaterThanOrEqual(290); + expect(took).toBeLessThan(300 + 1_500); + for (const id of ids) expect(warned.some((w) => w.includes(id.slice(0, 8)) && /didn't end its turn/.test(w))).toBe(true); + }); +}); + +describe("drain() and the work a turn's end starts", () => { + /** A sandbox provider that wraps nothing: enough for the session's #448 git check to run. */ + function fakeSandboxes(): SandboxRegistry { + const open = (spec: SandboxSpec) => new LocalSandboxHandle("bwrap", spec, (argv, o) => ({ argv: [...argv], env: o.env })); + const reg = new SandboxRegistry("linux", (n) => `/usr/bin/${n}`); + reg.register({ + id: "bwrap", + displayName: "bwrap", + capabilities: { isolation: "os", placement: "local", workspace: "mount", stdioProcess: true, commands: true, wrapsArgv: true, network: ["on", "off"], platforms: ["linux"] }, + probe: async () => ({ available: true }), + open: async (spec) => open(spec), + openSync: open, + } as SandboxProvider); + return reg; + } + + /** The agent commits a project settings file during its turn (what #checkSettingsCommits exists to catch). */ + class SettingsCommitter extends LateResultProvider { + override runTurn(opts: TurnOpts): TurnRun { + mkdirSync(join(repo, ".claude"), { recursive: true }); + writeFileSync(join(repo, ".claude", "settings.json"), "{}\n"); + git("add", ".claude"); + git("commit", "-qm", "settings"); + return super.runTurn(opts); + } + } + + /** A manager on `provider`, its session mid-turn; drained, then the store closed: what is on disk is what the daemon leaves. */ + async function drainMidTurn(provider: () => MockSessionProvider, extra: { sandboxRegistry?: SandboxRegistry; hooks?: HookBus } = {}) { + const m = new SessionManager(store, transcript, undefined, undefined, undefined, { + config: mkConfig(tmp), + _testProviderFactory: provider, + ...extra, + }); + const id = await create(m); + const s = m._sessionForTest(id)!; + const sending = s.send("go", AUTH).catch((e) => e); + // A large workspace's start snapshot holds the turn back up to its 2 s wait. + for (let i = 0; i < 1_000 && s.status !== "thinking"; i++) await Bun.sleep(5); + expect(s.status).toBe("thinking"); + const errors: string[] = []; + const warned: string[] = []; + const consoleError = console.error; + const consoleWarn = console.warn; + console.error = (...args: unknown[]) => void errors.push(args.map((a) => (a instanceof Error ? a.message : String(a))).join(" ")); + console.warn = (...args: unknown[]) => void warned.push(args.map(String).join(" ")); + let took = 0; + // Read synchronously right after the store closes: nothing still running gets a turn first. + let snapshots: string[] = []; + let lines = ""; + try { + const started = performance.now(); + await m.drain(10_000); + took = performance.now() - started; + store.close(); + lines = readFileSync(join(tmp, "transcripts", `${id}.jsonl`), "utf8"); + snapshots = existsSync(shadowRepoPath(ckptRoot(), id)) + ? execFileSync("git", ["--git-dir", shadowRepoPath(ckptRoot(), id), "for-each-ref", "--format=%(refname)", "refs/turns/"], { encoding: "utf8" }) + .split("\n") + .filter(Boolean) + : []; + await Bun.sleep(50); + } finally { + console.error = consoleError; + console.warn = consoleWarn; + } + await sending; + const db = new Database(join(tmp, "codeoid.db"), { readonly: true }); + const audit = (db.query("SELECT action FROM audit_log WHERE session_id = ?").all(id) as Array<{ action: string }>).map((r) => r.action); + db.close(); + return { id, took, errors, warned, lines, snapshots, audit }; + } + + const closedStore = (errors: string[]) => errors.filter((e) => /closed database|persist failed|audit write failed/i.test(e)); + + /** A repository with a 3000-commit history, a tag on each: git takes longer than drain's poll slack. */ + function deepHistory(): void { + const head = git("rev-parse", "HEAD").trim(); + let script = ""; + for (let i = 0; i < 3000; i++) { + script += `commit refs/heads/main\nmark :${i + 1}\ncommitter t ${1_700_000_000 + i} +0000\ndata 2\nc\n${i === 0 ? `from ${head}\n` : ""}M 644 inline f${i % 50}.txt\ndata 2\n${i % 10}\n\nreset refs/tags/t${i}\nfrom :${i + 1}\n\n`; + } + execFileSync("git", ["fast-import", "--quiet"], { cwd: repo, input: script }); + git("checkout", "-q", "-f", "main"); + } + + it("records the turn end's settings-commit check (audit row, warning line) before the store closes", async () => { + const r = await drainMidTurn(() => new SettingsCommitter(20), { sandboxRegistry: fakeSandboxes() }); + expect(r.audit).toContain("session.settings_committed"); + expect(r.lines).toContain("sandbox.settings_committed"); + expect(closedStore(r.errors)).toEqual([]); + }); + + for (const late of [20, 100, 300]) { + it(`records it in a repository with a 3000-commit history, the result ${late} ms after the interrupt`, async () => { + deepHistory(); + const r = await drainMidTurn(() => new SettingsCommitter(late), { sandboxRegistry: fakeSandboxes() }); + expect(r.audit).toContain("session.settings_committed"); + expect(r.lines).toContain("sandbox.settings_committed"); + expect(closedStore(r.errors)).toEqual([]); + }, 30_000); + } + + it("takes the interrupted turn's end snapshot of a large workspace before it returns", async () => { + mkdirSync(join(repo, "src")); + for (let i = 0; i < 20_000; i++) writeFileSync(join(repo, "src", `f${i}.txt`), `file ${i}\n`.repeat(20)); + const r = await drainMidTurn(() => new LateResultProvider(20)); + // The start snapshot and the end one: a rewind past this turn has the files it ended with. + expect(r.snapshots).toHaveLength(2); + expect(r.snapshots.filter((ref) => ref.endsWith("-end"))).toHaveLength(1); + expect(closedStore(r.errors)).toEqual([]); + }, 30_000); + + it("lets an after_turn hook finish", async () => { + const out = join(tmp, "after-turn.json"); + const hooks = new HookBus([{ event: "after_turn", type: "command", command: `sleep 0.3; cat > "${out}"` }]); + const r = await drainMidTurn(() => new LateResultProvider(20), { hooks }); + expect(existsSync(out)).toBe(true); + expect(r.took).toBeGreaterThanOrEqual(250); + }); + + it("waits for a turn end's work at most its bound, and names the session", async () => { + const out = join(tmp, "never.json"); + // A hook that outlives the bound: shutdown goes on without it. + const hooks = new HookBus([{ event: "after_turn", type: "command", command: `sleep 7; cat > "${out}"`, timeoutMs: 10_000 }]); + const r = await drainMidTurn(() => new LateResultProvider(20), { hooks }); + expect(r.took).toBeGreaterThanOrEqual(TURN_END_SETTLE_MS - 100); + expect(r.took).toBeLessThan(TURN_END_SETTLE_MS + 1_500); + expect(r.warned.some((w) => w.includes(r.id.slice(0, 8)) && /didn't finish its turn's end/.test(w))).toBe(true); + }, 15_000); +}); + +describe("a stopped turn's last reply", () => { + it("is shown whole, not typed out frame by frame, so the turn ends as its result arrives", async () => { + const m = new SessionManager(store, transcript, undefined, undefined, undefined, { + config: mkConfig(tmp), + _testProviderFactory: () => new LateResultProvider(0), + }); + managers.push(m); + const id = await create(m); + const s = m._sessionForTest(id)!; + const seen: DaemonMessage[] = []; + s.attach({ id: "w", auth: AUTH, send: (msg: DaemonMessage) => { seen.push(msg); } }); + const sending = s.send("go", AUTH).catch((e) => e); + for (let i = 0; i < 200 && s.status !== "thinking"; i++) await Bun.sleep(5); + await s.interrupt(AUTH); + await s.turnDrained(5_000); + await sending; + + const last = seen.filter( + (x): x is SessionMessage & DaemonMessage => x.type === "session.message" && x.role === "assistant" && x.content === "working on it, cut short", + ); + expect(last).toHaveLength(1); + expect(seen.some((x) => x.type === "session.message.delta" && x.messageId === last[0]!.messageId)).toBe(false); + }); }); describe("turn attribution across mid-turn messages, rotation and forks", () => { diff --git a/tsconfig.json b/tsconfig.json index b8819348..85facb24 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,7 @@ { "compilerOptions": { + "incremental": true, + "tsBuildInfoFile": ".cache/tsc/root.tsbuildinfo", "target": "ESNext", "module": "ESNext", "moduleResolution": "bundler",